diff --git a/.github/workflows/crawl-native-privacy.yml b/.github/workflows/crawl-native-privacy.yml new file mode 100644 index 0000000000..4e639e30ff --- /dev/null +++ b/.github/workflows/crawl-native-privacy.yml @@ -0,0 +1,84 @@ +name: Crawl native privacy + +on: + pull_request: + push: + branches: [main] + +permissions: + contents: read + +jobs: + crawl-native-privacy: + runs-on: ubuntu-latest + timeout-minutes: 20 + env: + LOOM_ENV_FILE: "" + LOOM_CH_CONTAINER: docsrc-crawl-${{ github.run_id }}-${{ github.run_attempt }} + LOOM_CH_HOST: 127.0.0.1 + LOOM_CH_PORT: "18123" + LOOM_CH_HOST_PORT: "18123" + LOOM_CH_NATIVE_HOST_PORT: "19000" + LOOM_CH_USER: loom + LOOM_CH_PASSWORD: loomdev + LOOM_CH_DATABASE: loom + LOOM_DEPLOYMENT: "1" + LOOM_INSTANCE_LABEL: ci-crawl-${{ github.run_id }}-${{ github.run_attempt }} + LOOM_BUILD_COMMIT: ${{ github.sha }} + LOOM_AUTH_REQUIRED: "true" + LOOM_STORAGE_AUTHORITY: clickhouse + LOOM_WORKERS_PROCESS: inline + LOOM_BACKGROUND_WORKERS: auto + LOOM_EMBEDDING_MODEL: fake-ngram-1024 + LOOM_LLM_PROVIDER: fake + LOOM_KNOWLEDGE_MIRRORED_SOURCES_ENABLED: "true" + LOOM_KNOWLEDGE_SOURCE_SYNC_ENABLED: "true" + LOOM_KNOWLEDGE_SOURCE_SYNC_INTERVAL_SECONDS: "60" + LOOM_KNOWLEDGE_SOURCE_SYNC_IDLE_INTERVAL_SECONDS: "60" + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + with: + persist-credentials: false + - uses: astral-sh/setup-uv@d0cc045d04ccac9d8b7881df0226f9e82c39688e # v6 + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 + with: + node-version: "22" + - name: Install locked dependencies and build dashboard assets + run: | + uv sync --locked --python 3.12 --extra dev --extra code + npm --prefix frontend ci + npm --prefix frontend run build + - name: Configure the test signer + run: | + crawl_signer_material="$(openssl rand -hex 32)" + printf '::add-mask::%s\n' "$crawl_signer_material" + printf 'LOOM_JWT_SECRET=%s\n' "$crawl_signer_material" >> "$GITHUB_ENV" + - name: Start the owned backend + id: clickhouse + run: | + if docker container inspect "$LOOM_CH_CONTAINER" >/dev/null 2>&1; then + echo "refusing an existing backend container" >&2 + exit 1 + fi + LOOM_CLICKHOUSE_IMAGE="$(bash dev/clickhouse_release.sh)" + export LOOM_CLICKHOUSE_IMAGE + echo "admitted=true" >> "$GITHUB_OUTPUT" + bash dev/ci_start_clickhouse.sh + - name: Run the native privacy witness + run: >- + .venv/bin/python -m pytest tests/integration/crawl_native_privacy.py + -q --tb=short --junitxml=crawl-native-privacy.xml + - name: Preserve backend diagnostics + if: always() && steps.clickhouse.outputs.admitted == 'true' + run: docker logs "$LOOM_CH_CONTAINER" > crawl-native-clickhouse.log 2>&1 + - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + if: always() + with: + name: crawl-native-privacy + path: | + crawl-native-privacy.xml + crawl-native-clickhouse.log + if-no-files-found: ignore + - name: Remove the owned backend + if: always() && steps.clickhouse.outputs.admitted == 'true' + run: docker rm -f "$LOOM_CH_CONTAINER" diff --git a/dev/restore_connected_accounts.py b/dev/restore_connected_accounts.py index c7fb13b1fc..8537007b19 100644 --- a/dev/restore_connected_accounts.py +++ b/dev/restore_connected_accounts.py @@ -17,6 +17,7 @@ from server.cli._apply import add_apply_flags from server.cli._ch import reads, run_connected, writes from server.knowledge.app.connected_restore import restore_all +from server.knowledge.app.crawl_restore import restore_all as restore_crawls def main(argv: list[str] | None = None) -> int: @@ -26,6 +27,7 @@ def main(argv: list[str] | None = None) -> int: async def run(ch: Any) -> int: await restore_all(apply=not args.dry_run, ch=ch) + await restore_crawls(apply=not args.dry_run, ch=ch) print( "connected-account restore cleanup completed; connections must be established again" if not args.dry_run diff --git a/docs/architecture/clickhouse.md b/docs/architecture/clickhouse.md index 652a7ec27f..c02776461e 100644 --- a/docs/architecture/clickhouse.md +++ b/docs/architecture/clickhouse.md @@ -694,6 +694,25 @@ with `purge=true`, runner state, and the supersession rows `knowledge.restore` r refused because an erasure removed the document it was built from deletes the chunk rows it inserted (lightweight `DELETE FROM loom.chunks` by source and version, `chunks.delete_versions`), so its text and author do not outlive the erasure. +- **Source credential resealing** conditionally updates only the matching old + ciphertext for an exact org, namespace, name and source id, under the source + lease. It rewrites raw versions and checks for old-ciphertext residue before + the previous root is retired. A full-row replacement could revive selectors + or identity metadata erased during a delayed write; the mutation preserves + every other column, including the version and unrelated sources. +- **Canonical-person metadata publishers** bound every source and SCIM INSERT + under their claim's receiver deadline. An unanswered send retains its Keeper + key until expiry; the receiver's latest possible completion precedes that + expiry. Nested claims retain the earliest deadline and every unanswered send. + Cleanup cannot acquire the same claim and report raw history erased while an + older publisher may still append it. Scoped statement bounds also limit the + guarded receiver; an insufficient claim window refuses before publishing. +- **SCIM cleanup capsule resealing** updates matching historical `scim_users` rows + in place under the identity lease, by exact SCIM row id and old ciphertext. The + replacement retains the captured canonical/provider identity and cleanup phase. + It waits for mutation completion and checks raw-history residue before the old + sealing root can be retired. Appending a version would leave older capsules + encrypted by the retired root (`server/auth/person_erasure.py`). - **Remaining `ALTER … UPDATE/DELETE` call sites** are listed in `tests/unit/test_clickhouse_mutation_inventory.py`, which fails on a new one until it is classified there. diff --git a/docs/architecture/document-sources.md b/docs/architecture/document-sources.md index 717e95a535..8798ed9229 100644 --- a/docs/architecture/document-sources.md +++ b/docs/architecture/document-sources.md @@ -373,7 +373,7 @@ directory, so it expands no group ([below](#who-may-read)). A `crawl` source's credential is a service-account key whose client id the org's workspace administrator granted domain-wide delegation for `drive.readonly`, -`admin.directory.group.member.readonly` and `admin.directory.user.readonly`. Its config +`admin.directory.group.member.readonly` and `admin.directory.user.readonly`. Its `crawl_options` field holds `directory_subject` (a workspace administrator the directory reads impersonate) and `subjects`: people (emails) and groups (group emails), whose members the directory read lists at each turn, at most 2000 people (not a setting). For each person the sync @@ -385,8 +385,9 @@ included, leaves out the people an erasure excluded ([erasure](#erasure-and-audi The probe checks that the delegation grants no write on what the crawl reads: it asks for each of the three scopes, which must be granted, and for -`https://www.googleapis.com/auth/drive`, `admin.directory.user` and -`admin.directory.group`, which the store must refuse; a credential the store grants any +`https://www.googleapis.com/auth/drive`, `admin.directory.user`, +`admin.directory.group` and the distinct `admin.directory.group.member` write scope, +which the store must refuse; a credential the store grants any of them is refused (`CREDENTIAL_REFUSED`). It cannot prove the delegation grants nothing else (a mail scope, say), and the page says so to the admin who registers it. The audit rows of `add`, `update` and `rotate` record the configured subjects as keyed hashes (an @@ -395,6 +396,24 @@ addresses does not reverse them). Audit rows also record the scopes granted and number of members resolved: the audit trail outlives erasure and keeps no crawled person's address. +Registration, update and rotation require a current org administrator who also +administers the source namespace. Each approval binds the signing-key lifecycle, +canonical registrant, exact selectors, items, credential hash, verified Workspace +customer and source generation. Workers recheck that authority after provider and +model waits and under the publication leases. An indexed read uses the same local +approval and exclusion fence without opening a credential or contacting Google. +Only a complete union listing stamped with both the source generation and current +exclusion epoch admits copies; updating a registration cannot reuse its old listing. +A check reports delegation standing without document titles or counts. + +`knowledge.source.update.crawl_readmit` explicitly readmits named, Directory-verified +people after SCIM reactivation. It requires the same current org and namespace +administration, refuses a still-deactivated contact and advances the durable refusal +epoch. It never republishes old copies: a complete subsequent crawl must establish +their permissions and freshness. Ordinary selector edits do not readmit anyone. +A dry run writes no registry or approval; an org whose refusal key has not been +initialized must complete its first real registration before a dry-run probe. + ## Access follows the source A source's `access`: @@ -1242,8 +1261,8 @@ statements and census keys against these, and its tests pin each one: - A person's connected accounts, the sources they own with every mirrored copy those wrote, and their entries in every crawl are erased when an org admin asks; afterwards no source row, list or members row names them, and a crawl reads them again only once - an org admin readmits them. The pull request that builds connected accounts and crawls - designs that erasure. + an org admin readmits them. Canonical-person deletion runs from the required SCIM + lifecycle; it never infers that a memory `user_id` names a sign-in subject. - Private canonical-owner cleanup uses the exact `(org, subject, kind)` key independently of namespace and end-user erasure. Disconnect and deactivation fence the generation, clear the current credential, queue every owned source for retirement, then physically @@ -1254,6 +1273,20 @@ statements and census keys against these, and its tests pin each one: mean an end-user principal and never guesses a sign-in subject from an id or email. Source retirement remains queued until the existing mirror retirement worker finishes; account cleanup alone does not certify that every source copy or crawl entry is erased. +- Canonical-person deletion captures verified sign-in subject aliases and contact + ties before removing them. A bounded Directory lookup separately proves Workspace + customer and provider user ids; neither an email nor an end-user `user_id` is a + canonical subject. A required cleanup capsule fences crawls until raw document ACL, + group membership, source selector, approval and capsule histories have been scrubbed + and their censuses pass. Other writers' content and the registering administrator's + service-account key remain intact. The capsule is physically erased; only keyed + refusal intent survives until explicit readmission. Reactivation cannot clear it. +- The canonical-person control registry is cross-namespace and uses its real identity + axes rather than inventing a namespace or memory `user_id`. SCIM capsules have an + identity-lease-protected history reseal; exclusion keys and pending manifests reseal + without changing the keyed identities, cleanup phase or Keeper fence. An unknown + historical deletion remains a refusal pending verified identity repair, rather than + guessing whom an old email denoted. - `erasure.request` refuses an `acl-` scope for every caller but a platform admin with auth on, and `erasure.export` refuses one for every caller (an export would read the mirror's chunks with no permission check); with auth off both refuse it for everyone. diff --git a/docs/architecture/storage-layout.md b/docs/architecture/storage-layout.md index 6a4285189e..85025174da 100644 --- a/docs/architecture/storage-layout.md +++ b/docs/architecture/storage-layout.md @@ -900,29 +900,31 @@ copies of tables listed here, owned and removed by that run, and erasure reaches ### Identity, tokens and access -| Table | Today | Target | Kind | Erasure | Wave | Why, and code that names it | -| -------------------- | ------------------------------------------------------- | --------------------------------------------------------------------------------- | ---------------------------------------- | ------------------------------------------------------- | ---- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -| `revoked_tokens` | RMT(version) `jti`, TTL | Postgres `revoked_tokens` | SoR | no content | 1 | read on every request through a cache that must fail closed; keyed `(kind, key)`, kind `subject_or_jti`, `cutoff` or `scim` (today's `jti` string holds a token id or a subject, either with an `issued_before` that stays a column, plus subject cutoffs and SCIM keys); named in `server/auth/revocation.py` | -| `signing_keys` | RMT(version) `kid` | Postgres `signing_keys` | SoR | no content | 1 | few rows, read by `kid` on every verify; rotation is a compare-and-set on `activate_at`; named in `server/auth/keyring.py`, `server/cli/secrets.py` | -| `keyring_readers` | MT `(seen_at, process)`, TTL | merge into Postgres `instances.keyring_refresh_s` | Coord | no content | 1 | one value per live process; the instance row already exists; named in `server/auth/keyring.py` | -| `service_accounts` | RMT(version) `(org, name)`, by `org` | Postgres `service_accounts` | SoR | no content | 1 | point read by `(org, name)` on token mint and on every delegated request; named in `server/auth/service_accounts.py`, `server/identity/app/me_service_accounts.py` | -| `machine_identities` | RMT(version) `name` | Postgres `machine_identities` | SoR | no content | 1 | workload trusts, read by name on token exchange; named in `server/auth/machines.py`, `server/observability/otel_probes.py` | -| `scim_users` | RMT(version) `id` | Postgres `scim_users` | SoR | no content | 1 | deprovisioning revokes tokens: a stale read keeps access; named in `server/auth/scim.py` | -| `sso_links` | RMT(version) `(email, org, namespace)` | Postgres `sso_links` | SoR | no content | 1 | account ownership decided at sign-in by `(email, org, namespace)`; named in `server/auth/sso_links.py`, `server/identity/store/namespace_census.py` | -| `principal_groups` | RMT(version) `(org, group_id)` | Postgres `principal_groups` | SoR | step `groups.remove_member` | 1 | group membership expands grants at check time; named in `server/identity/app/scim_groups.py`, `server/erasure/store/steps_principals.py` | -| `oauth_state` | RMT(version) `(record_type, id)`, by `record_type`, TTL | Postgres `oauth_state` | SoR | no content | 1 | registered clients, refresh grants including optional OIDC-verified email metadata (written before the token is handed out) and owed revocations (`revocation_owed`); what a sign-in spends once is in `oauth_signin_state`; today's 90-day TTL becomes a sweep; named in `server/auth/oauth_store.py`, `server/auth/revocation.py` | -| `oauth_claims` | KeeperMap `claim_key` | delete | Coord | no content | 1 | no code path writes it (its census entry says so); no module outside the census names the table | -| `oauth_signin_state` | KeeperMap `claim_key` | Postgres `oauth_signin_state` | SoR | no content | 1 | strict compare-and-set per key with an expiry and optional verified sign-in email metadata: an `UPDATE … WHERE value = $old`; the KeeperMap expiry becomes `expires_at`, read by a sweep; named in `server/auth/claims.py`, `server/cli/secrets.py` | -| `grant_claims` | KeeperMap `claim_key` | delete | Coord | no content | 1 | no code path writes it (its census entry says so); no module outside the census names the table (`server/auth/grant_claims.py` is the claim module, whose keys are in `_migration_lock`) | -| `namespace_grants` | RMT(version) `(org, namespace, subject)`, by `org` | Postgres `namespace_grants` | SoR | step `grants.revoke` | 1 | every request's role check; a revoke takes effect within the grant cache's invalidation bound; named in `server/identity/store/grants.py`, `server/auth/self_signup.py` | -| `namespaces` | RMT(updated_at) `(org, namespace)`, by `org` | Postgres `namespaces` + ClickHouse mirror | SoR (head) | step `personal.retire`; step `collections.forget_owner` | 1 | the registry with its per-namespace settings, read by grants, sign-in and request paths through a cache; the mirror serves ClickHouse worker scans and the SQL surface views; named in `server/auth/self_signup.py`, `server/cli/hygiene.py` | -| `collections` | RMT(version) `(org, namespace)` | Postgres `collections` | SoR | step `collections.forget_owner` | 1 | the collection tree grants inherit through; named in `server/identity/store/collections.py`, `server/identity/store/namespace_census.py` | -| `acl_versions` | SMT(changes) `(org, kind)` | merge into Postgres `cache_generations` (kinds `grants`, `groups`, `collections`) | Coord | no content | 1 | a change counter caches poll; a generation row plus `NOTIFY` replaces it; named in `server/core/grant_cache.py` | -| `auth_attempts` | MT `(kind, key_sha256, at)`, TTL | Postgres `auth_attempts` | Coord | no content | 1 | attempt counts per kind and hashed key (sign-in, sign-up, OIDC start, workload-token exchange; `server/auth/attempts.py`), read before any org is known, so one count per cell, not per org; it admits no more per key than today's sliding window and keeps the releases of successful attempts; named in `server/auth/attempts.py`, `server/cli/doctor.py` | -| `org_model_calls` | RMT(version) `org` | Postgres `org_model_calls` | SoR | no content | 1 | the org's switch for every model call, read at model-call time; one per org; named in `server/identity/store/model_calls.py`, `server/llm/customer.py` | -| `user_state` | RMT(version) `(org, subject)`, by `org` | Postgres `user_state` | SoR | no content | 3 | per-user dashboard state, read and written by key; named in `server/identity/app/me_routes.py`, `server/identity/store/user_state.py` | -| `namespace_visits` | RMT(version) `(org, subject, namespace)`, by `org` | Postgres `namespace_visits` | SoR | no content | 3 | written by `namespace.mark_seen` and read by key, like `user_state`; named in `server/identity/app/namespace_membership.py` | -| `subject_last_seen` | AMT `(org, subject)` | stays | Derived, SoR past its source's retention | step `subject_last_seen.delete` | — | `max` aggregate per subject, filled by a refreshable view over `agent_events`, without a model; past the source's retention the only copy; named in `server/tasks/store/repo.py`, `server/erasure/store/steps_memory.py` | +| Table | Today | Target | Kind | Erasure | Wave | Why, and code that names it | +| -------------------------------- | ------------------------------------------------------- | --------------------------------------------------------------------------------- | ---------------------------------------- | ------------------------------------------------------- | ---- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `knowledge_crawl_exclusion_keys` | RMT(version) `org` | Postgres `crawl_exclusion_keys` | SoR | canonical-person census, `crawl_person.exclude` | 3 | Stable sealed refusal key and pending cleanup manifest; namespaced memory erasure never resets it; named in `server/knowledge/store/crawl_exclusions.py`. | +| `knowledge_crawl_exclusions` | RMT(version) `(org, digest)` | Postgres `crawl_exclusions` | SoR | canonical-person census, `crawl_person.exclude` | 3 | Keyed person refusal intent survives physical contact cleanup until explicit readmission; named in `server/knowledge/store/crawl_exclusions.py`. | +| `revoked_tokens` | RMT(version) `jti`, TTL | Postgres `revoked_tokens` | SoR | no content | 1 | read on every request through a cache that must fail closed; keyed `(kind, key)`, kind `subject_or_jti`, `cutoff` or `scim` (today's `jti` string holds a token id or a subject, either with an `issued_before` that stays a column, plus subject cutoffs and SCIM keys); named in `server/auth/revocation.py` | +| `signing_keys` | RMT(version) `kid` | Postgres `signing_keys` | SoR | no content | 1 | few rows, read by `kid` on every verify; rotation is a compare-and-set on `activate_at`; named in `server/auth/keyring.py`, `server/cli/secrets.py` | +| `keyring_readers` | MT `(seen_at, process)`, TTL | merge into Postgres `instances.keyring_refresh_s` | Coord | no content | 1 | one value per live process; the instance row already exists; named in `server/auth/keyring.py` | +| `service_accounts` | RMT(version) `(org, name)`, by `org` | Postgres `service_accounts` | SoR | no content | 1 | point read by `(org, name)` on token mint and on every delegated request; named in `server/auth/service_accounts.py`, `server/identity/app/me_service_accounts.py` | +| `machine_identities` | RMT(version) `name` | Postgres `machine_identities` | SoR | no content | 1 | workload trusts, read by name on token exchange; named in `server/auth/machines.py`, `server/observability/otel_probes.py` | +| `scim_users` | RMT(version) `id` | Postgres `scim_users` | SoR | canonical-person census, `crawl_person.finalize` | 1 | deprovisioning revokes tokens: a stale read keeps access; named in `server/auth/scim.py` | +| `sso_links` | RMT(version) `(email, org, namespace)` | Postgres `sso_links` | SoR | no content | 1 | account ownership decided at sign-in by `(email, org, namespace)`; named in `server/auth/sso_links.py`, `server/identity/store/namespace_census.py` | +| `principal_groups` | RMT(version) `(org, group_id)` | Postgres `principal_groups` | SoR | step `groups.remove_member` | 1 | group membership expands grants at check time; named in `server/identity/app/scim_groups.py`, `server/erasure/store/steps_principals.py` | +| `oauth_state` | RMT(version) `(record_type, id)`, by `record_type`, TTL | Postgres `oauth_state` | SoR | no content | 1 | registered clients, refresh grants including optional OIDC-verified email metadata (written before the token is handed out) and owed revocations (`revocation_owed`); what a sign-in spends once is in `oauth_signin_state`; today's 90-day TTL becomes a sweep; named in `server/auth/oauth_store.py`, `server/auth/revocation.py` | +| `oauth_claims` | KeeperMap `claim_key` | delete | Coord | no content | 1 | no code path writes it (its census entry says so); no module outside the census names the table | +| `oauth_signin_state` | KeeperMap `claim_key` | Postgres `oauth_signin_state` | SoR | no content | 1 | strict compare-and-set per key with an expiry and optional verified sign-in email metadata: an `UPDATE … WHERE value = $old`; the KeeperMap expiry becomes `expires_at`, read by a sweep; named in `server/auth/claims.py`, `server/cli/secrets.py` | +| `grant_claims` | KeeperMap `claim_key` | delete | Coord | no content | 1 | no code path writes it (its census entry says so); no module outside the census names the table (`server/auth/grant_claims.py` is the claim module, whose keys are in `_migration_lock`) | +| `namespace_grants` | RMT(version) `(org, namespace, subject)`, by `org` | Postgres `namespace_grants` | SoR | step `grants.revoke` | 1 | every request's role check; a revoke takes effect within the grant cache's invalidation bound; named in `server/identity/store/grants.py`, `server/auth/self_signup.py` | +| `namespaces` | RMT(updated_at) `(org, namespace)`, by `org` | Postgres `namespaces` + ClickHouse mirror | SoR (head) | step `personal.retire`; step `collections.forget_owner` | 1 | the registry with its per-namespace settings, read by grants, sign-in and request paths through a cache; the mirror serves ClickHouse worker scans and the SQL surface views; named in `server/auth/self_signup.py`, `server/cli/hygiene.py` | +| `collections` | RMT(version) `(org, namespace)` | Postgres `collections` | SoR | step `collections.forget_owner` | 1 | the collection tree grants inherit through; named in `server/identity/store/collections.py`, `server/identity/store/namespace_census.py` | +| `acl_versions` | SMT(changes) `(org, kind)` | merge into Postgres `cache_generations` (kinds `grants`, `groups`, `collections`) | Coord | no content | 1 | a change counter caches poll; a generation row plus `NOTIFY` replaces it; named in `server/core/grant_cache.py` | +| `auth_attempts` | MT `(kind, key_sha256, at)`, TTL | Postgres `auth_attempts` | Coord | no content | 1 | attempt counts per kind and hashed key (sign-in, sign-up, OIDC start, workload-token exchange; `server/auth/attempts.py`), read before any org is known, so one count per cell, not per org; it admits no more per key than today's sliding window and keeps the releases of successful attempts; named in `server/auth/attempts.py`, `server/cli/doctor.py` | +| `org_model_calls` | RMT(version) `org` | Postgres `org_model_calls` | SoR | no content | 1 | the org's switch for every model call, read at model-call time; one per org; named in `server/identity/store/model_calls.py`, `server/llm/customer.py` | +| `user_state` | RMT(version) `(org, subject)`, by `org` | Postgres `user_state` | SoR | no content | 3 | per-user dashboard state, read and written by key; named in `server/identity/app/me_routes.py`, `server/identity/store/user_state.py` | +| `namespace_visits` | RMT(version) `(org, subject, namespace)`, by `org` | Postgres `namespace_visits` | SoR | no content | 3 | written by `namespace.mark_seen` and read by key, like `user_state`; named in `server/identity/app/namespace_membership.py` | +| `subject_last_seen` | AMT `(org, subject)` | stays | Derived, SoR past its source's retention | step `subject_last_seen.delete` | — | `max` aggregate per subject, filled by a refreshable view over `agent_events`, without a model; past the source's retention the only copy; named in `server/tasks/store/repo.py`, `server/erasure/store/steps_memory.py` | ### Deployment configuration and secrets diff --git a/docs/reference/configuration.md b/docs/reference/configuration.md index 1e6e899d1c..89eea4314c 100644 --- a/docs/reference/configuration.md +++ b/docs/reference/configuration.md @@ -1005,6 +1005,16 @@ Knowledge documents and conversations: chunking, search floor, thread titles. | `LOOM_KNOWLEDGE_CONNECTED_LEASE_WAIT_S` | `float` | `5.0` | | Seconds a connection change waits for another holder's account-generation lease. | | `LOOM_KNOWLEDGE_CONNECTED_REGISTRY_SCAN_LIMIT` | `int` | `4096` | | Maximum complete owner scan; exceeding it refuses instead of stopping a subset. | | `LOOM_KNOWLEDGE_CONNECTED_REGISTRY_PAGE_SIZE` | `int` | `256` | | Registrations per page during root-secret resealing. | +| `LOOM_KNOWLEDGE_CRAWL_DIRECTORY_MAX_RESPONSE_BYTES` | `int` | `1048576` | | Bytes read from one delegated Directory or OAuth response; larger responses refuse. | +| `LOOM_KNOWLEDGE_CRAWL_DIRECTORY_MAX_PAGES` | `int` | `256` | | Directory membership pages across the entire expansion, including nested groups. | +| `LOOM_KNOWLEDGE_CRAWL_MAX_GROUPS` | `int` | `2000` | | Groups across one complete crawl expansion; the total-person ceiling stays fixed. | +| `LOOM_KNOWLEDGE_CRAWL_MAX_FILES` | `int` | `10000` | | Distinct files across every person's complete listing; a larger union publishes nothing. | +| `LOOM_KNOWLEDGE_CRAWL_REGISTRY_SCAN_LIMIT` | `int` | `4096` | | Maximum full control-registry scan for cleanup; a larger scan refuses partial work. | +| `LOOM_KNOWLEDGE_CRAWL_REGISTRY_PAGE_SIZE` | `int` | `256` | | Org control rows per root-secret reseal page. | +| `LOOM_KNOWLEDGE_CRAWL_LEASE_TTL_S` | `float` | `60.0` | | Seconds the renewed org exclusion lease lives; namespace publication leases follow it. | +| `LOOM_KNOWLEDGE_CRAWL_LEASE_WAIT_S` | `float` | `5.0` | | Seconds a crawl control change waits for an existing org exclusion lease. | +| `LOOM_KNOWLEDGE_CRAWL_IDENTITY_CAPTURE_TIMEOUT_S` | `float` | `30.0` | | Whole pre-delete Directory identity capture across a person's existing crawl sources. | +| `LOOM_KNOWLEDGE_CRAWL_PERSON_CLEANUP_TIMEOUT_S` | `float` | `60.0` | | Whole local canonical-person crawl cleanup, including publisher fencing and residue. | | `LOOM_KNOWLEDGE_CONNECTED_OAUTH_TIMEOUT_S` | `float` | `30.0` | | Seconds one connected-account OAuth request may take, including its response. The caller's remaining deadline can shorten this bound. | | `LOOM_KNOWLEDGE_CONNECTED_OAUTH_RESPONSE_MAX_BYTES` | `int` | `65536` | | Bytes read from a connected-account token or identity response; larger metadata replies are refused before parsing. | | `LOOM_KNOWLEDGE_SOURCES_MAX_PER_NAMESPACE` | `int` | `8` | live | Active document sources (knowledge.source.add) one namespace may register; an add past it is refused with SOURCE_QUOTA. | diff --git a/docs/reference/errors.md b/docs/reference/errors.md index 003aff192d..61ae337ba6 100644 --- a/docs/reference/errors.md +++ b/docs/reference/errors.md @@ -450,7 +450,7 @@ Raised in `server/tracker/app/sources.py`. #### `CREDENTIAL_REFUSED` -Raised in `server/knowledge/app/connected_sources.py`, `server/knowledge/app/sources.py`. +Raised in `server/knowledge/app/connected_sources.py`, `server/knowledge/app/crawl_sources.py`, `server/knowledge/app/sources.py`. #### `CROSS_SITE_REQUEST` diff --git a/docs/reference/operations/knowledge.md b/docs/reference/operations/knowledge.md index ce11319251..306111b3f2 100644 --- a/docs/reference/operations/knowledge.md +++ b/docs/reference/operations/knowledge.md @@ -624,12 +624,14 @@ Register a document source on a namespace: a kind, what it reads (items and link | `links` | `list[str] \| None` | `None` | Links to folders, files, documents, slides, pages, databases or projects, read as items (`knowledge.source.parse_links`); an unreadable link refuses the call (BAD_LINK). | | `config` | `dict[str, Any] \| None` | `None` | What the source reads in the form that names ids per kind, instead of items or links; gdrive: {folder_ids: [1-20 folder ids], include: [docs, slides, pdf, pptx, docx] (default all but docx)}; linear: {project_ids: [1-20 project ids… | | `include` | `list[str] \| None` | `None` | gdrive with items or links: the file kinds read, of docs, slides, pdf, pptx, docx (default all but docx). | -| `credential_mode` | `Literal['shared', 'connected']` | `'shared'` | Use the supplied shared credential, or your own mirrored connected account. | +| `credential_mode` | `Literal['shared', 'connected', 'crawl']` | `'shared'` | Use shared credentials, your connected account, or approved Workspace delegation. | +| `crawl_options` | `KnowledgeSourceCrawlOptions \| None` | `None` | Crawl only: delegation identity and explicit people or groups; default none. | | `credential` | `str` | `''` | The source's read-only credential (gdrive: a service account's JSON key, with the folders shared to the account read-only; linear: a read-only API key of an account that sees the projects; notion: an integration token with read content… | | `dry_run` | `bool` | `False` | Probe and write nothing. | | Output field | Type | Meaning | | --- | --- | --- | +| `crawl_proof` | `KnowledgeSourceCrawlProof \| None` | Crawl only: checked scope and keyed-subject proof; None for another credential mode. | | `name` | `str` | The registered source's name, normalised. | | `kind` | `str` | The source kind, normalised. | | `namespace` | `str` | The namespace the source was registered on. | @@ -681,6 +683,7 @@ One registered document source: its items, its sync state and the documents it w | `kind` | `str` | The source's kind. | | `credential_mode` | `str` | The registration's credential class; never the credential or account identity. | | `access` | `str` | Namespace-wide or checked mirrored document access. | +| `crawl_options` | `dict[str, Any]` | Crawl selectors, only for a current org and namespace administrator. | | `owner_subject` | `str` | Canonical connected owner, only in the owner's permitted detail projection. | | `items` | `list[dict[str, Any]]` | The things the source reads; [] for a recording source. | | `include` | `list[str]` | The file kinds a gdrive source reads; [] for another kind. | @@ -840,6 +843,7 @@ Replace a registered document source's credential. | Output field | Type | Meaning | | --- | --- | --- | +| `crawl_proof` | `KnowledgeSourceCrawlProof \| None` | Crawl only: checked scope and keyed-subject proof; None for another credential mode. | | `name` | `str` | The source whose credential was replaced. | | `probe` | `KnowledgeSourceProbe` | What the probe with the new credential found. | | `rotated_at` | `str` | When the credential was replaced, ISO UTC. | @@ -877,10 +881,13 @@ Add items to a registered document source or remove them, or change a gdrive sou | `add_links` | `list[str]` | *(computed)* | Links to add, read as items; a link to something the source holds names it as held (a recursive folder, a drive at its root), and an add_items entry for the same thing wins. | | `remove_items` | `list[dict[str, Any]]` | *(computed)* | Items to remove, matched by type and id (a filter by its fields); a folder entry also removes a shared drive of the same id the source holds. | | `remove_links` | `list[str]` | *(computed)* | Links to remove, read as items; a folder link also removes a shared drive of the same id the source holds. | +| `crawl_readmit` | `list[str]` | *(computed)* | Crawl only: readmit these Directory-verified people after SCIM reactivation; default none. | +| `crawl_options` | `KnowledgeSourceCrawlOptions \| None` | `None` | Crawl only: replace the explicit delegation selectors; default unchanged. | | `include` | `list[str] \| None` | `None` | gdrive only: the new file kinds read, of docs, slides, pdf, pptx, docx; default unchanged. | | Output field | Type | Meaning | | --- | --- | --- | +| `crawl_proof` | `KnowledgeSourceCrawlProof \| None` | Crawl only: checked scope and keyed-subject proof; None for another credential mode. | | `name` | `str` | The source updated. | | `items` | `list[dict[str, Any]]` | The source's items as now stored. | | `include` | `list[str]` | The file kinds a gdrive source reads; [] for another kind. | diff --git a/docs/reference/schema.md b/docs/reference/schema.md index 7d94cdf67d..8d3977105b 100644 --- a/docs/reference/schema.md +++ b/docs/reference/schema.md @@ -142,7 +142,7 @@ the `loom` tables each view reads, and, for each materialized view, its target, -196 tables, 5 views and 19 materialized views in the `loom` and `loom_telemetry` databases; a `loom_telemetry` object keeps its database prefix. +198 tables, 5 views and 19 materialized views in the `loom` and `loom_telemetry` databases; a `loom_telemetry` object keeps its database prefix. #### Tables @@ -242,6 +242,8 @@ the `loom` tables each view reads, and, for each materialized view, its target, | `idempotency` | `ReplacingMergeTree(created_at)` | `idempotency_key` | | `expires_at` | | | `instances` | `KeeperMap('/loom/instances')` | | | | | | `keyring_readers` | `MergeTree` | `(seen_at, process)` | | `toDateTime(seen_at) + toIntervalDay(1)` | | +| `knowledge_crawl_exclusion_keys` | `ReplacingMergeTree(version)` | `org` | | | | +| `knowledge_crawl_exclusions` | `ReplacingMergeTree(version)` | `(org, digest)` | | | | | `knowledge_gaps` | `MergeTree` | `(org, namespace, build, gap_id)` | | `built_at + toIntervalDay(7)` | | | `knowledge_group_members` | `ReplacingMergeTree(version)` | `(org, namespace, source_id, group)` | | | | | `knowledge_ingest_cancels` | `MergeTree` | `(org, namespace, ingestion_id)` | | `toDateTime(requested_at) + toIntervalDay(30)` | | diff --git a/loom.example.yaml b/loom.example.yaml index 2bc42fdfdd..89b62433fd 100644 --- a/loom.example.yaml +++ b/loom.example.yaml @@ -2960,6 +2960,44 @@ settings: # Registrations per page during root-secret resealing. [restart] # knowledge_connected_registry_page_size: 256 + # Bytes read from one delegated Directory or OAuth response; larger responses refuse. + # [restart] + # knowledge_crawl_directory_max_response_bytes: 1048576 + + # Directory membership pages across the entire expansion, including nested groups. + # [restart] + # knowledge_crawl_directory_max_pages: 256 + + # Groups across one complete crawl expansion; the total-person ceiling stays fixed. + # [restart] + # knowledge_crawl_max_groups: 2000 + + # Distinct files across every person's complete listing; a larger union publishes + # nothing. [restart] + # knowledge_crawl_max_files: 10000 + + # Maximum full control-registry scan for cleanup; a larger scan refuses partial work. + # [restart] + # knowledge_crawl_registry_scan_limit: 4096 + + # Org control rows per root-secret reseal page. [restart] + # knowledge_crawl_registry_page_size: 256 + + # Seconds the renewed org exclusion lease lives; namespace publication leases follow + # it. [restart] + # knowledge_crawl_lease_ttl_s: 60.0 + + # Seconds a crawl control change waits for an existing org exclusion lease. [restart] + # knowledge_crawl_lease_wait_s: 5.0 + + # Whole pre-delete Directory identity capture across a person's existing crawl + # sources. [restart] + # knowledge_crawl_identity_capture_timeout_s: 30.0 + + # Whole local canonical-person crawl cleanup, including publisher fencing and residue. + # [restart] + # knowledge_crawl_person_cleanup_timeout_s: 60.0 + # Seconds one connected-account OAuth request may take, including its response. The # caller's remaining deadline can shorten this bound. [restart] # knowledge_connected_oauth_timeout_s: 30.0 diff --git a/schemas/loom-settings.schema.json b/schemas/loom-settings.schema.json index 75c2c45c1f..90e13fcdab 100644 --- a/schemas/loom-settings.schema.json +++ b/schemas/loom-settings.schema.json @@ -3197,6 +3197,67 @@ "type": "number", "description": "Seconds for local connection stops, including their source retirement writes." }, + "knowledge_crawl_directory_max_pages": { + "default": 256, + "minimum": 1, + "type": "integer", + "description": "Directory membership pages across the entire expansion, including nested groups." + }, + "knowledge_crawl_directory_max_response_bytes": { + "default": 1048576, + "minimum": 1024, + "type": "integer", + "description": "Bytes read from one delegated Directory or OAuth response; larger responses refuse." + }, + "knowledge_crawl_identity_capture_timeout_s": { + "default": 30.0, + "exclusiveMinimum": 0, + "type": "number", + "description": "Whole pre-delete Directory identity capture across a person's existing crawl sources." + }, + "knowledge_crawl_lease_ttl_s": { + "default": 60.0, + "exclusiveMinimum": 0, + "type": "number", + "description": "Seconds the renewed org exclusion lease lives; namespace publication leases follow it." + }, + "knowledge_crawl_lease_wait_s": { + "default": 5.0, + "minimum": 0, + "type": "number", + "description": "Seconds a crawl control change waits for an existing org exclusion lease." + }, + "knowledge_crawl_max_files": { + "default": 10000, + "minimum": 1, + "type": "integer", + "description": "Distinct files across every person's complete listing; a larger union publishes nothing." + }, + "knowledge_crawl_max_groups": { + "default": 2000, + "minimum": 1, + "type": "integer", + "description": "Groups across one complete crawl expansion; the total-person ceiling stays fixed." + }, + "knowledge_crawl_person_cleanup_timeout_s": { + "default": 60.0, + "exclusiveMinimum": 0, + "type": "number", + "description": "Whole local canonical-person crawl cleanup, including publisher fencing and residue." + }, + "knowledge_crawl_registry_page_size": { + "default": 256, + "maximum": 4096, + "minimum": 1, + "type": "integer", + "description": "Org control rows per root-secret reseal page." + }, + "knowledge_crawl_registry_scan_limit": { + "default": 4096, + "minimum": 1, + "type": "integer", + "description": "Maximum full control-registry scan for cleanup; a larger scan refuses partial work." + }, "knowledge_gaps_enabled": { "default": false, "type": "boolean", diff --git a/schemas/migrations/114_workspace_crawl.sql b/schemas/migrations/114_workspace_crawl.sql new file mode 100644 index 0000000000..8a2eb30900 --- /dev/null +++ b/schemas/migrations/114_workspace_crawl.sql @@ -0,0 +1,44 @@ +-- Crawl admission stays closed until every process validates its authority and exclusions. +ALTER TABLE loom.knowledge_sources ADD COLUMN IF NOT EXISTS options_json String DEFAULT '{}'; +ALTER TABLE loom.knowledge_sources ADD COLUMN IF NOT EXISTS registrant_subject String DEFAULT ''; +ALTER TABLE loom.knowledge_sources ADD COLUMN IF NOT EXISTS approval_enc String DEFAULT ''; +ALTER TABLE loom.knowledge_sources ADD COLUMN IF NOT EXISTS crawl_generation String DEFAULT ''; +ALTER TABLE loom.knowledge_sources ADD COLUMN IF NOT EXISTS crawl_customer_id String DEFAULT ''; +ALTER TABLE loom.knowledge_sources ADD COLUMN IF NOT EXISTS crawl_credential_hash String DEFAULT ''; +-- A deletion records verified canonical identity before discarding its contact ties. +ALTER TABLE loom.scim_users ADD COLUMN IF NOT EXISTS crawl_erasure_enc String DEFAULT ''; +ALTER TABLE loom.scim_users ADD COLUMN IF NOT EXISTS crawl_erasure_done Bool DEFAULT false; +ALTER TABLE loom.scim_users ADD COLUMN IF NOT EXISTS crawl_erasure_required Bool DEFAULT false; +-- A pre-erasure full listing never proves a post-erasure crawl union. +ALTER TABLE loom.knowledge_source_listings ADD COLUMN IF NOT EXISTS crawl_exclusion_generation String DEFAULT ''; + +-- Only a complete listing of this exact registration generation admits its copies. +ALTER TABLE loom.knowledge_source_listings ADD COLUMN IF NOT EXISTS crawl_generation String DEFAULT ''; + +-- One stable, sealed refusal key per org survives rotation of the deployment root. +CREATE TABLE IF NOT EXISTS loom.knowledge_crawl_exclusion_keys +( + org String, + key_id String, + key_enc String, + generation String, + pending_id String DEFAULT '', + pending_digests Array(String) DEFAULT [], + pending_exclude Bool DEFAULT false, + version UInt64 DEFAULT toUnixTimestamp64Nano(now64(9)) +) +ENGINE = ReplacingMergeTree(version) +ORDER BY org +SETTINGS non_replicated_deduplication_window = 1000; + +-- These explicit refusal tombstones contain no raw provider id or address. +CREATE TABLE IF NOT EXISTS loom.knowledge_crawl_exclusions +( + org String, + digest String, + excluded Bool, + version UInt64 DEFAULT toUnixTimestamp64Nano(now64(9)) +) +ENGINE = ReplacingMergeTree(version) +ORDER BY (org, digest) +SETTINGS non_replicated_deduplication_window = 1000; diff --git a/schemas/schema.sql b/schemas/schema.sql index 6496900e97..90f139cc01 100644 --- a/schemas/schema.sql +++ b/schemas/schema.sql @@ -16,13 +16,13 @@ -- Do NOT hand-edit. Edit schemas/migrations/*.sql and re-dump. -- CI verifies this file matches what migrations would produce. -- --- Counts: 196 tables + 5 views + 19 materialized views. +-- Counts: 198 tables + 5 views + 19 materialized views. CREATE DATABASE IF NOT EXISTS `loom`; CREATE DATABASE IF NOT EXISTS `loom_telemetry`; -- ====================================================================== --- Tables (196) +-- Tables (198) -- ====================================================================== -- loom.acl_versions @@ -2136,6 +2136,34 @@ ORDER BY (seen_at, process) TTL toDateTime(seen_at) + toIntervalDay(1) SETTINGS non_replicated_deduplication_window = 1000, index_granularity = 8192; +-- loom.knowledge_crawl_exclusion_keys +CREATE TABLE IF NOT EXISTS loom.knowledge_crawl_exclusion_keys +( + `org` String, + `key_id` String, + `key_enc` String, + `generation` String, + `pending_id` String DEFAULT '', + `pending_digests` Array(String) DEFAULT [], + `pending_exclude` Bool DEFAULT false, + `version` UInt64 DEFAULT toUnixTimestamp64Nano(now64(9)) +) +ENGINE = ReplacingMergeTree(version) +ORDER BY org +SETTINGS non_replicated_deduplication_window = 1000, index_granularity = 8192; + +-- loom.knowledge_crawl_exclusions +CREATE TABLE IF NOT EXISTS loom.knowledge_crawl_exclusions +( + `org` String, + `digest` String, + `excluded` Bool, + `version` UInt64 DEFAULT toUnixTimestamp64Nano(now64(9)) +) +ENGINE = ReplacingMergeTree(version) +ORDER BY (org, digest) +SETTINGS non_replicated_deduplication_window = 1000, index_granularity = 8192; + -- loom.knowledge_gaps CREATE TABLE IF NOT EXISTS loom.knowledge_gaps ( @@ -2265,7 +2293,9 @@ CREATE TABLE IF NOT EXISTS loom.knowledge_source_listings `source_id` String, `name` String, `started_at` DateTime64(3), - `version` UInt64 DEFAULT toUInt64(toUnixTimestamp64Nano(now64(9))) + `version` UInt64 DEFAULT toUInt64(toUnixTimestamp64Nano(now64(9))), + `crawl_exclusion_generation` String DEFAULT '', + `crawl_generation` String DEFAULT '' ) ENGINE = ReplacingMergeTree(version) ORDER BY (org, namespace, source_id) @@ -2318,7 +2348,13 @@ CREATE TABLE IF NOT EXISTS loom.knowledge_sources `access` LowCardinality(String) DEFAULT 'namespace', `credential_mode` LowCardinality(String) DEFAULT 'shared', `owner_subject` String DEFAULT '', - `connection_id` String DEFAULT '' + `connection_id` String DEFAULT '', + `options_json` String DEFAULT '{}', + `registrant_subject` String DEFAULT '', + `approval_enc` String DEFAULT '', + `crawl_generation` String DEFAULT '', + `crawl_customer_id` String DEFAULT '', + `crawl_credential_hash` String DEFAULT '' ) ENGINE = ReplacingMergeTree(version) ORDER BY (org, namespace, name) @@ -3949,7 +3985,10 @@ CREATE TABLE IF NOT EXISTS loom.scim_users `subjects` Array(String), `created_at` DateTime64(3, 'UTC'), `updated_at` DateTime64(3, 'UTC'), - `version` UInt64 + `version` UInt64, + `crawl_erasure_enc` String DEFAULT '', + `crawl_erasure_done` Bool DEFAULT false, + `crawl_erasure_required` Bool DEFAULT false ) ENGINE = ReplacingMergeTree(version) ORDER BY id diff --git a/sdk/python/loom/ops.py b/sdk/python/loom/ops.py index fd1913dc09..e105e011f4 100644 --- a/sdk/python/loom/ops.py +++ b/sdk/python/loom/ops.py @@ -11716,11 +11716,26 @@ class KnowledgeSourceAddInput(TypedDict, total=False): links: NotRequired[Optional[list[str]]] config: NotRequired[Optional[dict[str, Any]]] include: NotRequired[Optional[list[str]]] - credential_mode: NotRequired[Literal["shared", "connected"]] + credential_mode: NotRequired[Literal["shared", "connected", "crawl"]] + crawl_options: NotRequired[Optional["KnowledgeSourceCrawlOptions"]] credential: NotRequired[str] dry_run: NotRequired[bool] +class KnowledgeSourceCrawlOptions(TypedDict, total=False): + "Delegation identity and the complete bounded union of selected people." + + directory_subject: Required[str] + subjects: Required[list["KnowledgeSourceCrawlSubject"]] + + +class KnowledgeSourceCrawlSubject(TypedDict, total=False): + "An explicitly selected Workspace person or group, resolved through Directory." + + type: Required[Literal["person", "group"]] + email: Required[str] + + class KnowledgeSourceAddBody(TypedDict, total=False): "Register a document source on a namespace: a kind, what it reads (items" @@ -11732,12 +11747,14 @@ class KnowledgeSourceAddBody(TypedDict, total=False): links: NotRequired[Optional[list[str]]] config: NotRequired[Optional[dict[str, Any]]] include: NotRequired[Optional[list[str]]] - credential_mode: NotRequired[Literal["shared", "connected"]] + credential_mode: NotRequired[Literal["shared", "connected", "crawl"]] + crawl_options: NotRequired[Optional["KnowledgeSourceCrawlOptions"]] credential: NotRequired[str] dry_run: NotRequired[bool] class KnowledgeSourceAddOutput(TypedDict, total=False): + crawl_proof: NotRequired[Optional["KnowledgeSourceCrawlProof"]] name: Required[str] kind: Required[str] namespace: Required[str] @@ -11748,6 +11765,15 @@ class KnowledgeSourceAddOutput(TypedDict, total=False): dry_run: NotRequired[bool] +class KnowledgeSourceCrawlProof(TypedDict, total=False): + "The read-only delegation proof, without person addresses or credentials." + + subject_hashes: Required[list[str]] + read_scopes: Required[list[str]] + write_denials: Required[list[str]] + members: Required[int] + + class KnowledgeSourceProbe(TypedDict, total=False): "What a probe that succeeded found; a probe that fails refuses the call." @@ -11807,6 +11833,7 @@ class KnowledgeSourceInspectOutput(TypedDict, total=False): kind: Required[str] credential_mode: NotRequired[str] access: NotRequired[str] + crawl_options: NotRequired[dict[str, Any]] owner_subject: NotRequired[str] items: NotRequired[list[dict[str, Any]]] include: NotRequired[list[str]] @@ -11870,6 +11897,7 @@ class KnowledgeSourceEntry(TypedDict, total=False): kind: Required[str] credential_mode: NotRequired[str] access: NotRequired[str] + crawl_options: NotRequired[dict[str, Any]] owner_subject: NotRequired[str] items: NotRequired[list[dict[str, Any]]] include: NotRequired[list[str]] @@ -12027,6 +12055,7 @@ class KnowledgeSourceRotateBody(TypedDict, total=False): class KnowledgeSourceRotateOutput(TypedDict, total=False): + crawl_proof: NotRequired[Optional["KnowledgeSourceCrawlProof"]] name: Required[str] probe: NotRequired["KnowledgeSourceProbe"] rotated_at: NotRequired[str] @@ -12065,6 +12094,8 @@ class KnowledgeSourceUpdateInput(TypedDict, total=False): add_links: NotRequired[list[str]] remove_items: NotRequired[list[dict[str, Any]]] remove_links: NotRequired[list[str]] + crawl_readmit: NotRequired[list[str]] + crawl_options: NotRequired[Optional["KnowledgeSourceCrawlOptions"]] include: NotRequired[Optional[list[str]]] @@ -12078,10 +12109,13 @@ class KnowledgeSourceUpdateBody(TypedDict, total=False): add_links: NotRequired[list[str]] remove_items: NotRequired[list[dict[str, Any]]] remove_links: NotRequired[list[str]] + crawl_readmit: NotRequired[list[str]] + crawl_options: NotRequired[Optional["KnowledgeSourceCrawlOptions"]] include: NotRequired[Optional[list[str]]] class KnowledgeSourceUpdateOutput(TypedDict, total=False): + crawl_proof: NotRequired[Optional["KnowledgeSourceCrawlProof"]] name: Required[str] items: NotRequired[list[dict[str, Any]]] include: NotRequired[list[str]] diff --git a/sdk/typescript/src/generated/ops.ts b/sdk/typescript/src/generated/ops.ts index ff302c5c8a..455bd62579 100644 --- a/sdk/typescript/src/generated/ops.ts +++ b/sdk/typescript/src/generated/ops.ts @@ -4278,6 +4278,19 @@ export interface KnowledgeSearchOutput { stale_dropped?: number; } +export interface KnowledgeSourceCrawlOptions { + /** The delegated Directory reader's address. */ + directory_subject: string; + /** The people and groups to expand, at most 2000 people in total. */ + subjects: KnowledgeSourceCrawlSubject[]; +} + +export interface KnowledgeSourceCrawlSubject { + type: "person" | "group"; + /** The selected person's or group's address. */ + email: string; +} + export interface KnowledgeSourceAddInput { /** The org. */ org: string; @@ -4295,14 +4308,23 @@ export interface KnowledgeSourceAddInput { config?: Record | null; /** gdrive with items or links: the file kinds read, of docs, slides, pdf, pptx, docx (default all but docx). */ include?: string[] | null; - /** Use the supplied shared credential, or your own mirrored connected account. */ - credential_mode?: "shared" | "connected"; + /** Use shared credentials, your connected account, or approved Workspace delegation. */ + credential_mode?: "shared" | "connected" | "crawl"; + /** Crawl only: delegation identity and explicit people or groups; default none. */ + crawl_options?: KnowledgeSourceCrawlOptions | null; /** The source's read-only credential (gdrive: a service account's JSON key, with the folders shared to the account read-only; linear: a read-only API key of an acc */ credential?: string; /** Probe and write nothing. */ dry_run?: boolean; } +export interface KnowledgeSourceCrawlProof { + subject_hashes: string[]; + read_scopes: string[]; + write_denials: string[]; + members: number; +} + export interface KnowledgeSourceProbe { documents?: number; more?: boolean; @@ -4312,6 +4334,7 @@ export interface KnowledgeSourceProbe { } export interface KnowledgeSourceAddOutput { + crawl_proof?: KnowledgeSourceCrawlProof | null; name: string; kind: string; namespace: string; @@ -4383,6 +4406,7 @@ export interface KnowledgeSourceInspectOutput { kind: string; credential_mode?: string; access?: string; + crawl_options?: Record; owner_subject?: string; items?: Array>; include?: string[]; @@ -4405,6 +4429,7 @@ export interface KnowledgeSourceEntry { kind: string; credential_mode?: string; access?: string; + crawl_options?: Record; owner_subject?: string; items?: Array>; include?: string[]; @@ -4526,6 +4551,7 @@ export interface KnowledgeSourceRotateInput { } export interface KnowledgeSourceRotateOutput { + crawl_proof?: KnowledgeSourceCrawlProof | null; name: string; probe?: KnowledgeSourceProbe; rotated_at?: string; @@ -4562,11 +4588,16 @@ export interface KnowledgeSourceUpdateInput { remove_items?: Array>; /** Links to remove, read as items; a folder link also removes a shared drive of the same id the source holds. */ remove_links?: string[]; + /** Crawl only: readmit these Directory-verified people after SCIM reactivation; default none. */ + crawl_readmit?: string[]; + /** Crawl only: replace the explicit delegation selectors; default unchanged. */ + crawl_options?: KnowledgeSourceCrawlOptions | null; /** gdrive only: the new file kinds read, of docs, slides, pdf, pptx, docx; default unchanged. */ include?: string[] | null; } export interface KnowledgeSourceUpdateOutput { + crawl_proof?: KnowledgeSourceCrawlProof | null; name: string; items?: Array>; include?: string[]; diff --git a/server/auth/authz.py b/server/auth/authz.py index d998e4a40b..570455e8c4 100644 --- a/server/auth/authz.py +++ b/server/auth/authz.py @@ -39,7 +39,7 @@ from server.operations.base import GRANT_ROLES, Role, grant_rank, rank_role if TYPE_CHECKING: - from server.auth.jwt import AuthInfo + from server.auth.jwt import AuthInfo, AuthorizationClaims from server.core.grant_cache import GrantMap from server.operations.base import Operation @@ -66,7 +66,7 @@ class _Unknown(Enum): _UNKNOWN_GRANT = _Unknown.GRANT -def may_act_in_org(auth: AuthInfo, org: str) -> bool: +def may_act_in_org(auth: AuthorizationClaims, org: str) -> bool: """Whether `auth` may act in `org` at all: a token of that org, an org-wide (`*`) token, or a platform admin. The one cross-org rule — the transports and every handler's same-org check ask this; a restricted @@ -84,7 +84,7 @@ class ActedForRefused(PermissionError): access to an index, so a transport adds no index reason to it.""" -async def _check_provisioning(auth: AuthInfo | None, org: str | None) -> None: +async def _check_provisioning(auth: AuthorizationClaims | None, org: str | None) -> None: from server.auth import login, scim from server.core.errors import BackendUnavailableError, LoomServerError @@ -103,7 +103,7 @@ async def _check_provisioning(auth: AuthInfo | None, org: str | None) -> None: raise ActedForRefused(_ACTED_FOR_REVOKED) -async def check_acted_for(auth: AuthInfo | None, org: str | None) -> None: +async def check_acted_for(auth: AuthorizationClaims | None, org: str | None) -> None: """Refuse a trusted caller acting for a revoked or SCIM-deactivated subject under any spelling it holds, or for an `sa:` subject that is not an active bot or one of its agents: a token's own subject, whose grants are the token's and end with it however @@ -219,6 +219,36 @@ async def check_authorized( row_scoped: bool = False, known_grant: Role | None | _Unknown = _UNKNOWN_GRANT, grants: GrantMap | None = None, +) -> None: + """Authorize an actual request principal with the common claim and live-grant rules.""" + from server.auth.jwt import AuthInfo + + if namespace is not None and is_mirror_name(namespace): + raise PermissionError(MIRROR_REFUSED) + if auth is not None and not isinstance(auth, AuthInfo): + raise PermissionError("invalid auth principal") + await _check_authorized_claims( + auth, + required_role, + org, + namespace, + deployment_scoped=deployment_scoped, + row_scoped=row_scoped, + known_grant=known_grant, + grants=grants, + ) + + +async def _check_authorized_claims( + auth: AuthorizationClaims | None, + required_role: Role, + org: str | None = None, + namespace: str | None = None, + *, + deployment_scoped: bool = False, + row_scoped: bool = False, + known_grant: Role | None | _Unknown = _UNKNOWN_GRANT, + grants: GrantMap | None = None, ) -> None: """Raise PermissionError if `auth` lacks `required_role` or access to `(org, namespace)`. Consults `loom.namespace_grants` as a fallback when the JWT role alone @@ -231,7 +261,7 @@ async def check_authorized( `org` and is decided there. `known_grant` (the grant the gate would read, the creator's for a delegated token) and `grants` (the subject's cached grant map, for the restricted check): what a caller that already read them hands in.""" - from server.auth.jwt import AuthInfo # circular import sidestep + from server.auth.jwt import AuthorizationClaims # circular import sidestep # A mirror opens to no token, with auth off too: only the stores' own workers reach one. if namespace is not None and is_mirror_name(namespace): @@ -239,7 +269,7 @@ async def check_authorized( if auth is None: return - if not isinstance(auth, AuthInfo): # defensive + if not isinstance(auth, AuthorizationClaims): # defensive raise PermissionError("invalid auth principal") # A cell serves only the orgs it holds: the routes outside the op registry @@ -376,7 +406,11 @@ async def check_authorized( def _check_scoped_without_target( - auth: AuthInfo, required_role: Role, org: str | None, deployment_scoped: bool, row_scoped: bool + auth: AuthorizationClaims, + required_role: Role, + org: str | None, + deployment_scoped: bool, + row_scoped: bool, ) -> None: """A scoped token on an op that names no (org, namespace): admitted on its best role anywhere for a row- or deployment-scoped op, else 400.""" @@ -403,7 +437,11 @@ def _check_scoped_without_target( async def _grant_role( - auth: AuthInfo, org: str, namespace: str, cap: int | None, known_grant: Role | None | _Unknown + auth: AuthorizationClaims, + org: str, + namespace: str, + cap: int | None, + known_grant: Role | None | _Unknown, ) -> Role | None: """The grant the subject holds there. Where its claim is capped, the cap's read decides for the subject itself: it covers every spelling consistently, @@ -424,7 +462,7 @@ async def _grant_role( async def _check_creator( - auth: AuthInfo, required_role: Role, org: str | None, namespace: str | None + auth: AuthorizationClaims, required_role: Role, org: str | None, namespace: str | None ) -> None: """A delegated token stops when its creator's account is disabled. An op with no namespace has no grant to recheck, so a write there needs the @@ -449,7 +487,7 @@ async def _check_creator( async def _check_delegated( - auth: AuthInfo, + auth: AuthorizationClaims, required_role: Role, org: str, namespace: str, @@ -513,7 +551,7 @@ async def may_be_restricted(org: str, namespace: str) -> bool: return True -def names_namespace(auth: AuthInfo, org: str, namespace: str) -> bool: +def names_namespace(auth: AuthorizationClaims, org: str, namespace: str) -> bool: """True when the token names `namespace` literally — in its `namespaces` claim or its `ns_roles` map. A wildcard (`*`) never counts: a restricted namespace must be asked for by name at mint time.""" @@ -525,7 +563,7 @@ def names_namespace(auth: AuthInfo, org: str, namespace: str) -> bool: def _scoped_role( - auth: AuthInfo, required_role: Role, org: str, namespace: str, *, cap: int | None + auth: AuthorizationClaims, required_role: Role, org: str, namespace: str, *, cap: int | None ) -> Role | None: """The role a scoped token holds on (org, namespace) for this op: its own, capped at the subject's live grant where it holds one. None for a write @@ -573,7 +611,7 @@ def _capped_message(org: str, namespace: str, required_role: Role) -> str: ) -async def claim_cap(auth: AuthInfo, org: str, namespace: str) -> int | None: +async def claim_cap(auth: AuthorizationClaims, org: str, namespace: str) -> int | None: """The live grant rank this token's claim on `namespace` is capped at (0: none live), where the token names it and its subject holds a grant row there of any status. None where the subject never held one (an operator's @@ -585,7 +623,7 @@ async def claim_cap(auth: AuthInfo, org: str, namespace: str) -> int | None: return (await grant_map(org, auth.subject)).rank(namespace) -async def live_claims(auth: AuthInfo, org: str, namespaces: Iterable[str]) -> list[str]: +async def live_claims(auth: AuthorizationClaims, org: str, namespaces: Iterable[str]) -> list[str]: """The named `namespaces` (no `*`), in order, that the claim still reaches: all but those capped at a grant that is no longer live.""" return [ns for ns in namespaces if ns and ns != "*" and await claim_cap(auth, org, ns) != 0] @@ -593,7 +631,7 @@ async def live_claims(auth: AuthInfo, org: str, namespaces: Iterable[str]) -> li @marks_unverified async def _check_restricted( - auth: AuthInfo, + auth: AuthorizationClaims, required_role: Role, org: str, namespace: str, @@ -645,7 +683,7 @@ async def _check_restricted( async def restricted_allows( - auth: AuthInfo, + auth: AuthorizationClaims, required_role: Role, org: str, namespace: str, diff --git a/server/auth/delegation.py b/server/auth/delegation.py index 036413fad9..7d1554177d 100644 --- a/server/auth/delegation.py +++ b/server/auth/delegation.py @@ -23,7 +23,7 @@ from server.auth import jwt as jwt_mod from server.auth.authz import grant_role from server.auth.fail_closed import note_unreadable, unreadable -from server.auth.jwt import AuthInfo, TokenError +from server.auth.jwt import AuthInfo, AuthorizationClaims, TokenError from server.core.grant_sql import GRANT_RANK_SQL, LIVE_GRANT from server.observability.metrics import record_auth_refusal from server.operations.base import GRANT_ROLES, Role, grant_rank @@ -369,7 +369,9 @@ def _lower(a: Role | None, b: Role | None) -> Role | None: return min(a, b) -def ceiling_role(token: AuthInfo, org: str, namespace: str, *, write: bool = False) -> Role | None: +def ceiling_role( + token: AuthorizationClaims, org: str, namespace: str, *, write: bool = False +) -> Role | None: """The most `token` itself allows on `namespace`, at most admin. A scoped token's `ns_roles` entry; a delegated flat token's role where its claim covers the namespace (`write`: its write claim); a primary flat token's diff --git a/server/auth/gateway.py b/server/auth/gateway.py index 8ff604836d..77c7111c26 100644 --- a/server/auth/gateway.py +++ b/server/auth/gateway.py @@ -17,13 +17,13 @@ import dataclasses -from server.auth.jwt import AuthInfo +from server.auth.jwt import AuthInfo, AuthorizationClaims from server.core.principals import USER, is_reserved_subject, personal_name, principal_id __all__ = ["effective_auth", "is_end_user", "personal_namespace"] -def is_end_user(auth: AuthInfo | None) -> bool: +def is_end_user(auth: AuthorizationClaims | None) -> bool: """True when the request runs as an end user an issuer signed in.""" return auth is not None and bool(principal_id(auth.subject, USER)) diff --git a/server/auth/jwt.py b/server/auth/jwt.py index 3835ed2040..aac4397106 100644 --- a/server/auth/jwt.py +++ b/server/auth/jwt.py @@ -9,10 +9,11 @@ from __future__ import annotations +import hashlib import logging import secrets import time -from dataclasses import dataclass +from dataclasses import dataclass, field from typing import Any import jwt @@ -93,9 +94,45 @@ class KeyringUnavailable(AuthUnavailable): `jwt_revocation_max_stale_seconds`, so a retired kid may not be known.""" +def _key_fingerprint(key: Any) -> str: + value = key.encode() if isinstance(key, str) else key + if not isinstance(value, bytes): + from cryptography.hazmat.primitives import serialization + + value = key.public_bytes( + serialization.Encoding.PEM, serialization.PublicFormat.SubjectPublicKeyInfo + ) + return hashlib.sha256(value).hexdigest() + + +@dataclass(frozen=True, repr=False) +class SigningAuthority: + """The successful verifier's identity, not a bearer or a stored signing key.""" + + issuer: str + kid: str + algorithm: str + fingerprint: str + + def current(self) -> bool: + from server.auth import keyring + + return ( + keyring.loaded() + and not keyring.stale() + and self.issuer in settings.trusted_issuers() + and self.algorithm in keyring.ALGORITHMS + and bool(self.fingerprint) + and any( + secrets.compare_digest(self.fingerprint, _key_fingerprint(key)) + for key in keyring.verification_keys(self.kid, self.algorithm) + ) + ) + + @dataclass(frozen=True) -class AuthInfo: - """The authenticated principal. Passed through to handlers.""" +class AuthorizationClaims: + """Verified claim metadata, without making it an incoming request principal.""" subject: str # effective subject (post-impersonation if any) org: str @@ -181,6 +218,8 @@ class AuthInfo: issued_at: int | None = None # `exp` as unix seconds; None for a token without one. expires_at: int | None = None + # Filled only by the normal verifier after it finds the actual successful key. + signing_authority: SigningAuthority | None = field(default=None, repr=False) def has_role(self, required: Role) -> bool: return any(r.value >= required.value for r in self.roles) @@ -233,6 +272,11 @@ def can_write(self, org: str, namespace: str) -> bool: return namespace in self.write_namespaces or "*" in self.write_namespaces +@dataclass(frozen=True) +class AuthInfo(AuthorizationClaims): + """An authenticated request principal, passed through to handlers.""" + + # The `read_scope` claim's values, widest first. READ_SCOPES: tuple[str, ...] = ("namespace", "own_user", "own_session") @@ -414,7 +458,7 @@ def verify_token(token: str, *, secret: str | None = None) -> AuthInfo: trusted = settings.trusted_issuers() try: - decoded = _decode(token, candidates, algorithm) + decoded, fingerprint = _decode(token, candidates, algorithm) except jwt.ExpiredSignatureError as e: raise TokenError(f"token expired: {e}", reason="expired") from e except jwt.PyJWTError as e: # InvalidKeyError is not an InvalidTokenError @@ -480,6 +524,11 @@ def verify_token(token: str, *, secret: str | None = None) -> AuthInfo: first_party=_bool_claim(decoded, "fps"), issued_at=_iat_of(decoded), expires_at=int(exp) if exp is not None else None, + signing_authority=( + SigningAuthority(str(decoded["iss"]), token_kid, algorithm, fingerprint) + if secret is None + else None + ), ) @@ -569,7 +618,7 @@ def _verify_end_user(token: str, issuer: str, spec: EndUserIssuer, public_key: A from server.core.principals import AGENT, USER, end_user_revocation_subject, principal_id try: - decoded = _decode(token, [public_key], "RS256") + decoded, _ = _decode(token, [public_key], "RS256") except jwt.ExpiredSignatureError as e: raise TokenError(f"token expired: {e}", reason="expired") from e except jwt.PyJWTError as e: @@ -622,14 +671,14 @@ def _verify_end_user(token: str, issuer: str, spec: EndUserIssuer, public_key: A ) -def _decode(token: str, keys: list[Any], algorithm: str) -> dict[str, Any]: +def _decode(token: str, keys: list[Any], algorithm: str) -> tuple[dict[str, Any], str]: """The claims under the first of `keys` the signature matches. `issuer=` is checked by the caller (PyJWT takes one string, not a set); `exp` is not required (indefinite tokens) but enforced when present.""" error = jwt.InvalidSignatureError("no key") for key in keys: try: - return jwt.decode( + decoded = jwt.decode( token, key, algorithms=[algorithm], @@ -637,6 +686,7 @@ def _decode(token: str, keys: list[Any], algorithm: str) -> dict[str, Any]: options={"require": ["iat", "sub", "iss", "aud"]}, leeway=settings.jwt_leeway_seconds, # clock-skew tolerance ) + return decoded, _key_fingerprint(key) except jwt.InvalidSignatureError as e: error = e raise error diff --git a/server/auth/person_erasure.py b/server/auth/person_erasure.py new file mode 100644 index 0000000000..7236a30048 --- /dev/null +++ b/server/auth/person_erasure.py @@ -0,0 +1,374 @@ +"""Verified canonical-person deletion metadata survives contact and sign-in removal. + +This is cleanup input, never a request identity, credential or permission to read as +the person. The required downstream port runs outside SCIM's identity leases. +""" + +from __future__ import annotations + +import json +from dataclasses import asdict, dataclass +from typing import Any, Protocol + +from server.auth import aliases +from server.auth.reader_identity import resolve_reader +from server.core.ch import get_client, own_write_reads, set_query_tag +from server.core.errors import BackendUnavailableError +from server.core.secrets_at_rest import seal, unseal +from server.settings import settings + +PURPOSE = "auth.crawl_person_erasure" + + +def unavailable() -> BackendUnavailableError: + return BackendUnavailableError( + "canonical-person cleanup is unavailable", operation="crawl_erasure" + ) + + +@dataclass(frozen=True, repr=False) +class Person: + org: str + subject: str + emails: tuple[str, ...] + spellings: tuple[str, ...] = () + + def __post_init__(self) -> None: + if ( + any(not isinstance(v, str) or not v or "\0" in v for v in (self.org, self.subject)) + or not self.emails + or self.emails != tuple(sorted(set(self.emails))) + or self.spellings != tuple(sorted(set(self.spellings))) + or self.subject not in self.spellings + or any(not isinstance(v, str) or not v or "\0" in v for v in self.spellings) + or any( + not isinstance(v, str) + or v != v.lower() + or v.count("@") != 1 + or not all(v.split("@")) + or any(ord(c) < 33 or ord(c) > 126 for c in v) + for v in self.emails + ) + ): + raise unavailable() + + +@dataclass(frozen=True, repr=False) +class Workspace: + """A real Directory observation tied to an already verified canonical person.""" + + org: str + subject: str + customer: str + user: str + emails: tuple[str, ...] + + def __post_init__(self) -> None: + Person(self.org, self.subject, self.emails, (self.subject,)) + if any( + not isinstance(value, str) + or not value + or len(value) > 512 + or any(not 33 <= ord(char) <= 126 for char in value) + for value in (self.customer, self.user) + ): + raise unavailable() + + +@dataclass(frozen=True, repr=False) +class Capture: + id: str + people: tuple[Person, ...] + workspaces: tuple[Workspace, ...] = () + prepared: bool = False + + +def _seal(captured: Capture) -> str: + value = {"format": 2, **asdict(captured)} + encoded = seal(PURPOSE, json.dumps(value, separators=(",", ":"), sort_keys=True)) + load(captured.id, encoded) + return encoded + + +async def capture(uid: str, email: str, ties: dict[str, set[str]]) -> str: + """Only verified directory rows can join a contact to a canonical subject and org.""" + set_query_tag("auth.person_erasure.capture") + if not uid or not email or not isinstance(ties, dict): + raise unavailable() + people: dict[tuple[str, str], Person] = {} + for subject, orgs in sorted(ties.items()): + for org in sorted(orgs): + identity = await resolve_reader(org, subject) + if identity is None or email.lower() not in identity.emails: + raise unavailable() + person = Person( + org, + identity.subject, + tuple(sorted(identity.emails)), + tuple(sorted(await aliases.verified_spellings(org, identity.subject))), + ) + key = org, identity.subject + if key in people and people[key] != person: + raise unavailable() + people[key] = person + return _seal(Capture(uid, tuple(people[key] for key in sorted(people)))) + + +def load(uid: str, encoded: str) -> Capture: + try: + value = json.loads(unseal(PURPOSE, encoded, warn=False)) + if ( + not isinstance(value, dict) + or set(value) != {"format", "id", "people", "workspaces", "prepared"} + or type(value["format"]) is not int + or value["format"] != 2 + or not uid + or value["id"] != uid + or not isinstance(value["people"], list) + or not isinstance(value["workspaces"], list) + or type(value["prepared"]) is not bool + or any( + not isinstance(p, dict) + or set(p) != {"org", "subject", "emails", "spellings"} + or not isinstance(p["emails"], list) + or not isinstance(p["spellings"], list) + for p in value["people"] + ) + ): + raise unavailable() + people = tuple( + Person(p["org"], p["subject"], tuple(p["emails"]), tuple(p["spellings"])) + for p in value["people"] + ) + keys = [(p.org, p.subject) for p in people] + if keys != sorted(set(keys)): + raise unavailable() + workspaces = _workspaces(value["workspaces"], people) + return Capture(uid, people, workspaces, value["prepared"]) + except (ValueError, TypeError, KeyError, RecursionError): + raise unavailable() from None + + +def _workspaces(values: Any, people: tuple[Person, ...]) -> tuple[Workspace, ...]: + if not isinstance(values, list) or any( + not isinstance(w, dict) + or set(w) != {"org", "subject", "customer", "user", "emails"} + or not isinstance(w["emails"], list) + for w in values + ): + raise unavailable() + result = tuple( + Workspace(w["org"], w["subject"], w["customer"], w["user"], tuple(w["emails"])) + for w in values + ) + keys = [(w.org, w.subject, w.customer, w.user) for w in result] + if keys != sorted(set(keys)) or any( + not any( + (p.org, p.subject) == (w.org, w.subject) and set(p.emails).intersection(w.emails) + for p in people + ) + for w in result + ): + raise unavailable() + return result + + +@dataclass(frozen=True, repr=False) +class Pending: + """Only this exact durable SCIM deletion can authorize its private cleanup.""" + + captured: Capture + encoded: str + + async def validate(self, org: str | None = None, *, before_delete: bool = False) -> None: + if org is not None and org not in {p.org for p in self.captured.people}: + raise unavailable() + with own_write_reads(): + row = await get_client().query_one( + "SELECT active, deleted, crawl_erasure_enc, " + "crawl_erasure_required, crawl_erasure_done " + "FROM loom.scim_users FINAL WHERE id = {id:String} LIMIT 1", + params={"id": self.captured.id}, + ) + if ( + row is None + or row["crawl_erasure_enc"] != self.encoded + or not row["crawl_erasure_required"] + or row["crawl_erasure_done"] + or before_delete + and row["deleted"] + ): + raise unavailable() + if before_delete: + for person in self.captured.people: + if org is not None and person.org != org: + continue + identity = await resolve_reader(person.org, person.subject) + if ( + identity is None + or identity.subject != person.subject + or identity.emails != frozenset(person.emails) + or tuple(sorted(await aliases.verified_spellings(person.org, person.subject))) + != person.spellings + ): + raise unavailable() + + +class Preparer(Protocol): + async def __call__(self, pending: Pending) -> tuple[Workspace, ...]: ... + + +class Cleanup(Protocol): + async def __call__(self, pending: Pending) -> None: ... + + +_cleanup: Cleanup | None = None +_preparer: Preparer | None = None +_finalizer: Cleanup | None = None + + +def bind(preparer: Preparer, cleanup: Cleanup, finalizer: Cleanup) -> None: + global _cleanup, _preparer, _finalizer + _cleanup, _preparer, _finalizer = cleanup, preparer, finalizer + + +def interrupted(uid: str, encoded: str, observed: str) -> str: + """Retain both verified intents when reactivation changed the canonical person.""" + previous, current = load(uid, encoded), load(uid, observed) + merged: dict[tuple[str, str], set[str]] = {} + spellings: dict[tuple[str, str], set[str]] = {} + for person in (*previous.people, *current.people): + merged.setdefault((person.org, person.subject), set()).update(person.emails) + spellings.setdefault((person.org, person.subject), set()).update(person.spellings) + people = tuple( + Person( + org, + subject, + tuple(sorted(merged[org, subject])), + tuple(sorted(spellings[org, subject])), + ) + for org, subject in sorted(merged) + ) + return _seal(Capture(uid, people, previous.workspaces, False)) + + +async def prepare(uid: str, encoded: str) -> str: + if _preparer is None: + raise unavailable() + captured = load(uid, encoded) + pending = Pending(captured, encoded) + await pending.validate(before_delete=True) + if captured.prepared: + return encoded + workspaces = await _preparer(pending) + await pending.validate(before_delete=True) + return _seal(Capture(uid, captured.people, workspaces, True)) + + +async def finish(uid: str, encoded: str) -> None: + if _cleanup is None: + raise unavailable() + captured = load(uid, encoded) + if not captured.prepared: + raise unavailable() + pending = Pending(captured, encoded) + await pending.validate() + await _cleanup(pending) + await pending.validate() + + +async def finalize(uid: str, encoded: str) -> None: + """SCIM holds its user lease after cleanup, so no newer capture can be overwritten.""" + if _finalizer is None: + raise unavailable() + captured = load(uid, encoded) + if not captured.prepared: + raise unavailable() + pending = Pending(captured, encoded) + await pending.validate() + await _finalizer(pending) + + +async def require_settled(*, ch: Any = None) -> None: + """Missing historical identity proof stops crawls until explicit verified backfill.""" + with own_write_reads(): + row = await (ch or get_client()).query_one( + "SELECT 1 AS pending FROM loom.scim_users FINAL " + "WHERE (deleted = 1 OR crawl_erasure_required = 1) AND NOT crawl_erasure_done LIMIT 1" + ) + if row is not None: + raise unavailable() + + +async def _reseal_user(uid: str, *, dry_run: bool, ch: Any) -> tuple[int, bool]: + from server.auth import scim + from server.core.secrets_at_rest import resealed + + changed = 0 + async with scim._held(scim._user_lease(uid)): + rows = await ch.query_all( + "SELECT DISTINCT crawl_erasure_enc FROM loom.scim_users " + "WHERE id = {id:String} AND crawl_erasure_enc != '' LIMIT {limit:UInt32}", + params={"id": uid, "limit": settings.knowledge_crawl_registry_scan_limit + 1}, + ) + if len(rows) > settings.knowledge_crawl_registry_scan_limit: + raise unavailable() + for row in rows: + encoded = str(row["crawl_erasure_enc"]) + if not unseal(PURPOSE, encoded, warn=False): + return changed, True + captured = load(uid, encoded) + replacement = resealed(PURPOSE, encoded) + if replacement is None: + continue + if load(uid, replacement) != captured: + raise unavailable() + changed += 1 + if not dry_run: + await ch.execute( + "ALTER TABLE loom.scim_users UPDATE crawl_erasure_enc = {new:String} " + "WHERE id = {id:String} AND crawl_erasure_enc = {old:String}", + params={"id": uid, "new": replacement, "old": encoded}, + settings={"mutations_sync": 2}, + ) + residue = await ch.query_one( + "SELECT count() AS n FROM loom.scim_users WHERE id = {id:String} " + "AND crawl_erasure_enc = {old:String}", + params={"id": uid, "old": encoded}, + ) + if residue is None or residue["n"] != 0: + raise unavailable() + return changed, False + + +async def reseal_registry(*, dry_run: bool, ch: Any) -> tuple[int, list[str]]: + """Reseal every historical capsule under the same identity lease as deletion. + + Completion inspects raw capsule history, so resealing only FINAL would leave + that proof unreadable when the preceding root is removed. + """ + if ch is not get_client(): + raise RuntimeError("crawl_person_reseal_target_mismatch") + set_query_tag("auth.person_erasure.reseal") + after, scanned, changed, lost = "", 0, 0, [] + while page := await ch.query_all( + "SELECT DISTINCT id FROM loom.scim_users " + "WHERE id > {after:String} AND crawl_erasure_enc != '' " + "ORDER BY id LIMIT {limit:UInt32}", + params={"after": after, "limit": settings.knowledge_crawl_registry_page_size}, + ): + identifiers = [str(row["id"]) for row in page] + scanned += len(identifiers) + if ( + scanned > settings.knowledge_crawl_registry_scan_limit + or identifiers != sorted(set(identifiers)) + or any(uid <= after for uid in identifiers) + ): + raise unavailable() + for uid in identifiers: + count, unreadable = await _reseal_user(uid, dry_run=dry_run, ch=ch) + changed += count + if unreadable: + lost.append(uid) + after = identifiers[-1] + return changed, lost diff --git a/server/auth/scim.py b/server/auth/scim.py index 6548c64caf..e42d7959c8 100644 --- a/server/auth/scim.py +++ b/server/auth/scim.py @@ -40,10 +40,10 @@ from server.settings import settings if TYPE_CHECKING: - from server.auth.jwt import AuthInfo + from server.auth.jwt import AuthorizationClaims -async def refuses_auth(auth: AuthInfo, *, org: str | None = None) -> bool: +async def refuses_auth(auth: AuthorizationClaims, *, org: str | None = None) -> bool: """Check the effective person and every bearer, creator and parent behind it.""" from server.auth.gateway import is_end_user from server.auth.scim_people import MAX_ITEMS @@ -78,6 +78,9 @@ async def refuses_auth(auth: AuthInfo, *, org: str | None = None) -> bool: "active", "deleted", "subjects", + "crawl_erasure_enc", + "crawl_erasure_done", + "crawl_erasure_required", "created_at", "updated_at", "version", @@ -330,7 +333,18 @@ async def delete(uid: str) -> bool: deleted = bool(row["deleted"]) # A retry finishes a failed try. The addresses beside the email keep no one out. cleanup_row = row - row, _ = await _set_active(row, False, deleted=1, emails="[]") + try: + row = await _capture_person_erasure(row) + finally: + # Downstream capture can fail, but SCIM must still deactivate the identity. + row, _ = await _set_active( + row, + False, + deleted=1, + emails="[]", + crawl_erasure_required=True, + crawl_erasure_done=bool(deleted and row.get("crawl_erasure_done", False)), + ) except BaseException: # A failed or cancelled deactivation still fences its connections. if cleanup_row is not None: await _repair_failed_connections(cleanup_row, erase_owner=True) @@ -365,7 +379,9 @@ def _subject_lease(subject: str) -> str: @asynccontextmanager async def _held(name: str) -> AsyncIterator[None]: try: - async with leases.holding(name, ttl_s=_LEASE_TTL_S, wait_s=_LEASE_WAIT_S) as held: + async with leases.holding( + name, ttl_s=_LEASE_TTL_S, wait_s=_LEASE_WAIT_S, settle_writes=True + ) as held: if not held: raise Busy yield @@ -405,6 +421,82 @@ async def _revoke_tokens(row: dict[str, Any], *, previous_subjects: Sequence[str respect_cutoff=True, ) await service_accounts.revoke_created_by(row["subjects"], revoked_by=_BY) + if row["deleted"] or row.get("crawl_erasure_required", False): + await _finish_person_erasure(row) + + +async def _capture_person_erasure(row: dict[str, Any]) -> dict[str, Any]: + from server.auth import person_erasure + + if row.get("crawl_erasure_done", False) and row["deleted"]: + return row + encoded = str(row.get("crawl_erasure_enc", "")) + if encoded: + captured = person_erasure.load(row["id"], encoded) + if captured.prepared: + if not row["deleted"]: + observed = await person_erasure.capture( + row["id"], row["email"], await _subjects(row["email"]) + ) + if person_erasure.load(row["id"], observed).people != captured.people: + # Preserve both intents instead of finishing an old deletion as + # though it proved a newly assigned email's canonical identity. + row["crawl_erasure_enc"] = person_erasure.interrupted( + row["id"], encoded, observed + ) + await _write(row) + raise person_erasure.unavailable() + return row + if row["deleted"]: + # A reused provider email must not complete a failed historical capture. + raise person_erasure.unavailable() + return await _prepare_person_erasure(row) + if row["deleted"]: + # A historical tombstone cannot be joined to a newly reused email on retry. + raise person_erasure.unavailable() + encoded = await person_erasure.capture(row["id"], row["email"], await _subjects(row["email"])) + # Keep this local full row even when an acknowledged write loses its reply. + # The unconditional deactivation must not overwrite the captured proof with defaults. + row.update(crawl_erasure_enc=encoded, crawl_erasure_required=True, crawl_erasure_done=False) + await _write(row) + return await _prepare_person_erasure(row) + + +async def _prepare_person_erasure(row: dict[str, Any]) -> dict[str, Any]: + from server.auth import person_erasure + + encoded = await person_erasure.prepare(row["id"], row["crawl_erasure_enc"]) + row["crawl_erasure_enc"] = encoded + await _write(row) + return row + + +async def _finish_person_erasure(row: dict[str, Any]) -> None: + from server.auth import person_erasure + + if row.get("crawl_erasure_done", False): + return + encoded = str(row.get("crawl_erasure_enc", "")) + await person_erasure.finish(row["id"], encoded) + async with _held(_user_lease(row["id"])): + current = await _row(row["id"]) + if current is None: + raise person_erasure.unavailable() + if current.get("crawl_erasure_done", False) and not current.get("crawl_erasure_enc"): + return + if current.get("crawl_erasure_enc") != encoded: + raise person_erasure.unavailable() + # Clearing only a new FINAL row would retain the encrypted person's metadata + # in old parts. The required erasure port redacts raw history and marks done + # in the same awaited mutation, so a lost reply is safe to retry. + await person_erasure.finalize(row["id"], encoded) + current = await _row(row["id"]) + if ( + current is None + or not current.get("crawl_erasure_done") + or current.get("crawl_erasure_enc") + ): + raise person_erasure.unavailable() async def _repair_failed_connections(row: dict[str, Any], *, erase_owner: bool) -> None: @@ -446,7 +538,14 @@ async def _set_active( async def _deactivate(row: dict[str, Any], **changes: Any) -> dict[str, Any]: while True: - new = {**row, **changes, "active": 0} + new = { + **row, + **changes, + "active": 0, + "crawl_erasure_required": bool( + row["deleted"] or row.get("crawl_erasure_required", False) or changes.get("deleted") + ), + } found = await _subjects(row["email"]) known = set(row["subjects"]) async with subject_leases([*known, *found]): @@ -483,7 +582,12 @@ async def _converge_active( revokes outright of every subject the email reaches, whatever a failed request left. The cutoff is for the caller to wait out: what this call lifted, and for a reactivation (or its retry) the newest SCIM narrowed to.""" - new = {**row, **changes, "active": 1} + new = { + **row, + **changes, + "active": 1, + "crawl_erasure_required": bool(row["deleted"] or row.get("crawl_erasure_required", False)), + } subjects = sorted(set(row["subjects"]) | set(await _subjects(row["email"]))) rows = await revocation.scim_revocations(subjects) if not _outright(rows): # nothing to lift, as on most pushes: no subject lease @@ -669,6 +773,9 @@ async def _write(row: dict[str, Any]) -> None: ch = get_client() version = await next_version(ch, TABLE, {"id": row["id"]}) entries = [e for e in emails_of(row) if normalized(e["value"])] + row.setdefault("crawl_erasure_enc", "") + row.setdefault("crawl_erasure_done", False) + row.setdefault("crawl_erasure_required", bool(row["deleted"])) row.update( addresses=[normalized(e["value"]) for e in entries], work_addresses=[normalized(e["value"]) for e in entries if normalized(e["type"]) == "work"], diff --git a/server/auth/sharing.py b/server/auth/sharing.py index 544044f5b0..66e15c9847 100644 --- a/server/auth/sharing.py +++ b/server/auth/sharing.py @@ -37,7 +37,7 @@ from server.operations.base import Role, rank_role if TYPE_CHECKING: - from server.auth.jwt import AuthInfo + from server.auth.jwt import AuthorizationClaims log = logging.getLogger(__name__) @@ -160,7 +160,7 @@ async def _load_groups(org: str) -> dict[str, list[str]]: return out -async def principals(auth: AuthInfo, org: str) -> list[str]: +async def principals(auth: AuthorizationClaims, org: str) -> list[str]: """Every grant subject this request holds: its own, its groups', its agent's, and `everyone` (the caller checked it may act in `org`).""" out = [auth.subject, *await groups_of(org, auth.subject), EVERYONE] @@ -169,7 +169,7 @@ async def principals(auth: AuthInfo, org: str) -> list[str]: return out -def owns_personal(auth: AuthInfo, org: str, namespace: str) -> bool: +def owns_personal(auth: AuthorizationClaims, org: str, namespace: str) -> bool: """True when `namespace` is the personal collection of the end user `auth` runs as.""" return bool(principal_id(auth.subject, USER)) and namespace == personal_name(org, auth.subject) @@ -179,7 +179,9 @@ def _best(maps: list[grant_cache.GrantMap], lineage: list[str]) -> Role | None: return rank_role(max((m.rank(ns) or 0 for m in maps for ns in lineage), default=0)) -def _claim_limits(auth: AuthInfo, org: str, namespace: str) -> list[tuple[Role | None, str]]: +def _claim_limits( + auth: AuthorizationClaims, org: str, namespace: str +) -> list[tuple[Role | None, str]]: """Each limit the token's own claim sets on `namespace`, whatever its grants, with what the claim must say to lift it: `viewer` outside its `ns_write` set; for a scoped or delegated token, nothing (None) unless the claim names `namespace` (a `*` does not), and @@ -206,7 +208,7 @@ def _claim_limits(auth: AuthInfo, org: str, namespace: str) -> list[tuple[Role | return limits -def _claim_ceiling(auth: AuthInfo, org: str, namespace: str) -> Role | None: +def _claim_ceiling(auth: AuthorizationClaims, org: str, namespace: str) -> Role | None: """The most the token's own claim lets it hold on `namespace` (`_claim_limits`): None for nothing, `admin` when the claim sets no limit.""" ceilings = [limit for limit, _ in _claim_limits(auth, org, namespace)] @@ -216,7 +218,7 @@ def _claim_ceiling(auth: AuthInfo, org: str, namespace: str) -> Role | None: def _capped( - auth: AuthInfo, org: str, namespace: str, granted: Role | None, creator: Role | None + auth: AuthorizationClaims, org: str, namespace: str, granted: Role | None, creator: Role | None ) -> Role | None: """`granted`, no further than the token's own claim allows (`_claim_ceiling`), and for a delegated token no further than its creator's role there (`creator`).""" @@ -228,12 +230,14 @@ def _capped( return min(ceilings, key=lambda r: r.value) # type: ignore[union-attr] -async def role(auth: AuthInfo, org: str, namespace: str) -> Role | None: +async def role(auth: AuthorizationClaims, org: str, namespace: str) -> Role | None: """The request's role on the collection `namespace`.""" return (await roles(auth, org, [namespace]))[namespace] -async def roles(auth: AuthInfo, org: str, namespaces: list[str]) -> dict[str, Role | None]: +async def roles( + auth: AuthorizationClaims, org: str, namespaces: list[str] +) -> dict[str, Role | None]: """The request's role on each collection of `namespaces`, reading the tree, the principals and their grant maps once. A personal collection needs no read: its name says whose it is. A platform admin holds admin on every one @@ -276,7 +280,7 @@ async def _grant_maps(org: str, subjects: list[str]) -> list[grant_cache.GrantMa return maps -async def readable(auth: AuthInfo | None, org: str) -> list[str]: +async def readable(auth: AuthorizationClaims | None, org: str) -> list[str]: """Every collection of `org` the request may read, sorted; none (but the user's own) when the tree or a grant cannot be read.""" from server.auth.authz import may_act_in_org @@ -309,7 +313,9 @@ async def readable(auth: AuthInfo | None, org: str) -> list[str]: return sorted(found | shared) -def claim_cap(auth: AuthInfo, org: str, namespace: str, required_role: Role) -> str | None: +def claim_cap( + auth: AuthorizationClaims, org: str, namespace: str, required_role: Role +) -> str | None: """What the token's own claim must say when it, rather than a missing grant, keeps the token below `required_role` on `namespace` (`_claim_limits`, the limits the gate applies): every limit that does, so one re-mint lifts them all; None when none does. @@ -323,7 +329,7 @@ def claim_cap(auth: AuthInfo, org: str, namespace: str, required_role: Role) -> return ", and ".join(needs) or None -async def check(auth: AuthInfo, required_role: Role, org: str, namespace: str) -> None: +async def check(auth: AuthorizationClaims, required_role: Role, org: str, namespace: str) -> None: """The gate for a collection: PermissionError unless the request holds `required_role` there. A platform admin (an operator's erasure or export) passes.""" diff --git a/server/auth/source_registration.py b/server/auth/source_registration.py new file mode 100644 index 0000000000..5b1b4c7604 --- /dev/null +++ b/server/auth/source_registration.py @@ -0,0 +1,237 @@ +"""A source retains checked claim metadata, never a bearer or a request identity.""" + +from __future__ import annotations + +import dataclasses +import json +import time +from dataclasses import dataclass + +from pydantic import TypeAdapter, ValidationError + +from server.auth import aliases, authz, revocation, scim, tenancy +from server.auth.context import current_auth +from server.auth.jwt import AuthInfo, AuthorizationClaims, SigningAuthority, _check_not_revoked +from server.auth.reader_identity import resolve_reader +from server.core import cell, grant_cache +from server.core.ch import own_write_reads +from server.core.secrets_at_rest import seal, unseal +from server.operations.base import Role +from server.settings import settings + +PURPOSE = "knowledge_crawl.approval" +_now = time.time +_CLAIMS = TypeAdapter(AuthorizationClaims) +_CLAIM_FIELDS = frozenset(field.name for field in dataclasses.fields(AuthorizationClaims)) +_SIGNING_FIELDS = frozenset(field.name for field in dataclasses.fields(SigningAuthority)) + + +class Refused(PermissionError): + def __init__(self) -> None: + super().__init__("source registration authority unavailable") + + +@dataclass(frozen=True) +class Scope: + org: str + namespace: str + source_id: str + generation: str + resource_hash: str + + def check(self) -> None: + if any(not isinstance(v, str) or not v or "\0" in v for v in dataclasses.astuple(self)): + raise Refused() + if len(self.resource_hash) != 64 or any( + c not in "0123456789abcdef" for c in self.resource_hash + ): + raise Refused() + + +@dataclass(frozen=True, repr=False) +class AdminPermit: + """A current presenter, distinct from the source registrant's retained approval.""" + + _auth: AuthInfo + _approval: Approval + + async def validate(self, org: str) -> None: + if org != self._approval.scope.org or current_auth() is not self._auth: + raise Refused() + await self._approval.validate() + if current_auth() is not self._auth: + raise Refused() + + +async def current_permit(scope: Scope) -> AdminPermit: + scope.check() + auth = current_auth() + if ( + not isinstance(auth, AuthInfo) + or not auth.has_role(Role.ORG_ADMIN) + or not authz.may_act_in_org(auth, scope.org) + ): + raise Refused() + registrant = await aliases.canonical_subject(scope.org, auth.subject) + metadata = _CLAIMS.validate_json(json.dumps(dataclasses.asdict(auth)), strict=True) + permit = AdminPermit(auth, Approval(scope, registrant, metadata)) + await permit.validate(scope.org) + return permit + + +async def capture(auth: AuthInfo, scope: Scope) -> tuple[str, str]: + """Seal the current caller's exact authorization for this one source generation.""" + scope.check() + if not isinstance(auth, AuthInfo) or current_auth() is not auth: + raise Refused() + if auth.signing_authority is None or not auth.signing_authority.current(): + raise Refused() + await authz.check_authorized(auth, Role.ADMIN, scope.org, scope.namespace) + if not auth.has_role(Role.ORG_ADMIN) or current_auth() is not auth: + raise Refused() + registrant = await aliases.canonical_subject(scope.org, auth.subject) + frame = { + "format": 1, + "scope": dataclasses.asdict(scope), + "registrant": registrant, + "claims": dataclasses.asdict(auth), + } + encoded = seal(PURPOSE, json.dumps(frame, sort_keys=True, separators=(",", ":"))) + approval = load(encoded, scope, registrant=registrant) + await approval.validate() + if current_auth() is not auth: + raise Refused() + return registrant, encoded + + +def load(encoded: str, scope: Scope, *, registrant: str) -> Approval: + scope.check() + try: + value = json.loads(unseal(PURPOSE, encoded, warn=False)) + if ( + not isinstance(value, dict) + or set(value) != {"format", "scope", "registrant", "claims"} + or type(value["format"]) is not int + or value["format"] != 1 + or value["scope"] != dataclasses.asdict(scope) + or not registrant + or value["registrant"] != registrant + or not isinstance(value["claims"], dict) + or set(value["claims"]) != _CLAIM_FIELDS + ): + raise Refused() + signing = value["claims"].get("signing_authority") + if signing is not None and ( + not isinstance(signing, dict) or set(signing) != _SIGNING_FIELDS + ): + raise Refused() + claims = _CLAIMS.validate_json(json.dumps(value["claims"]), strict=True) + except (ValueError, TypeError, RecursionError, ValidationError): + raise Refused() from None + if len(claims.parent_subjects) != len(claims.parent_iats): + raise Refused() + return Approval(scope, registrant, claims) + + +def names_person( + encoded: str, *, org: str, namespace: str, source_id: str, spellings: tuple[str, ...] +) -> bool: + """Inspect sealed historical PII, without treating expired metadata as live authority.""" + try: + value = json.loads(unseal(PURPOSE, encoded, warn=False)) + if not isinstance(value, dict) or not isinstance(value.get("scope"), dict): + raise Refused() + scope = Scope(**value["scope"]) + if (scope.org, scope.namespace, scope.source_id) != (org, namespace, source_id): + raise Refused() + registrant = value["registrant"] + if not isinstance(registrant, str) or not registrant: + raise Refused() + approval = load(encoded, scope, registrant=registrant) + subjects = { + approval.registrant, + approval.claims.subject, + approval.claims.creator, + approval.claims.impersonated_by, + *approval.claims.parent_subjects, + } + return bool(subjects.intersection(spellings)) + except (ValueError, TypeError, KeyError, RecursionError): + raise Refused() from None + + +@dataclass(frozen=True, repr=False) +class Approval: + scope: Scope + registrant: str + claims: AuthorizationClaims + + def __repr__(self) -> str: + return "Approval()" + + async def _revocations(self) -> None: + claims = self.claims + _check_not_revoked( + { + "sub": claims.subject, + "iat": claims.issued_at, + "jti": claims.token_id, + "sid": claims.session_id, + "parents": list(claims.parents), + "creator": claims.creator, + "parent_subjects": list(claims.parent_subjects), + "parent_iats": list(claims.parent_iats), + } + ) + # The worker checks current storage, not only the request verifier's bounded snapshot. + ids = [v for v in (claims.token_id, claims.session_id, *claims.parents) if v] + oldest = ( + min([claims.issued_at, *claims.parent_iats]) if claims.issued_at is not None else None + ) + subjects = [ + (claims.subject, claims.issued_at), + *zip(claims.parent_subjects, claims.parent_iats, strict=True), + ] + if claims.creator: + subjects.append((claims.creator, oldest)) + if claims.impersonated_by: + subjects.append((claims.impersonated_by, claims.issued_at)) + cutoffs = await revocation.cutoffs([*ids, *(s for s, _ in subjects)]) + if any(key in cutoffs for key in ids) or any( + subject in cutoffs and (issued is None or revocation.kills(cutoffs[subject], issued)) + for subject, issued in subjects + ): + raise Refused() + + async def validate(self) -> None: + scope, claims = self.scope, self.claims + scope.check() + if ( + not settings.auth_actually_required() + or cell.gate_reason() + or claims.signing_authority is None + or not claims.signing_authority.current() + or claims.issued_at is None + or claims.issued_at > _now() + settings.jwt_leeway_seconds + or not claims.has_role(Role.ORG_ADMIN) + or not authz.may_act_in_org(claims, scope.org) + or claims.expires_at is not None + and _now() > claims.expires_at + settings.jwt_leeway_seconds + or not await tenancy.is_held(scope.org) + ): + raise Refused() + await self._revocations() + identity = await resolve_reader(scope.org, claims.subject) + if identity is None or identity.subject != self.registrant: + raise Refused() + for address in identity.emails: + if await scim.deactivated(address): + raise Refused() + # Same metadata and role law as a real request, with fresh target grants and lineage. + spellings = await aliases.verified_spellings(scope.org, claims.subject) + grant_cache.invalidate(scope.org, [*spellings, claims.creator, *claims.parent_subjects]) + grant_cache.invalidate(scope.org, kind=grant_cache.GROUPS) + grant_cache.invalidate(scope.org, kind=grant_cache.COLLECTIONS) + with own_write_reads(): + await authz.check_acted_for(claims, scope.org) + await authz._check_authorized_claims(claims, Role.ADMIN, scope.org, scope.namespace) diff --git a/server/auth/tenancy.py b/server/auth/tenancy.py index 5042cb22b5..304e08a4ae 100644 --- a/server/auth/tenancy.py +++ b/server/auth/tenancy.py @@ -43,7 +43,7 @@ from server.settings import settings if TYPE_CHECKING: - from server.auth.jwt import AuthInfo + from server.auth.jwt import AuthorizationClaims log = logging.getLogger(__name__) @@ -106,7 +106,7 @@ async def is_held(org: str) -> bool: return held -def scoped_to(auth: AuthInfo | None, org: str) -> bool: +def scoped_to(auth: AuthorizationClaims | None, org: str) -> bool: """Whether a verified principal is scoped to `org`: its token's own org. A platform or `*`-org principal acts in orgs the cell holds and places none.""" from server.operations.base import Role @@ -116,7 +116,7 @@ def scoped_to(auth: AuthInfo | None, org: str) -> bool: return not auth.has_role(Role.PLATFORM_ADMIN) -async def ensure_held(org: str | None, auth: AuthInfo | None) -> None: +async def ensure_held(org: str | None, auth: AuthorizationClaims | None) -> None: """Pass when this cell holds `org`, recording or placing it as the mode allows. Raise `PermissionError` (403) first for a principal that may not act in the org, `WrongCellError` (409) for another cell's org, `ValidationError` (422) for a new org diff --git a/server/cli/_ch.py b/server/cli/_ch.py index bd18707617..ce21859d9a 100644 --- a/server/cli/_ch.py +++ b/server/cli/_ch.py @@ -402,12 +402,17 @@ def run_connected[T]( HTTP session.""" from server import boot from server.auth.account_lifecycle import bind + from server.auth.person_erasure import bind as bind_person_erasure from server.core.background import register_buffer_drainer from server.core.ch import get_client + from server.erasure.app.crawl_person import finalize as finalize_person_erasure from server.insights.app.provider_command import model_command from server.knowledge.app.connected_accounts import stop_subjects + from server.knowledge.app.crawl_capture import prepare as prepare_person_erasure + from server.knowledge.app.crawl_person_erasure import finish as finish_person_erasure bind(stop_subjects) + bind_person_erasure(prepare_person_erasure, finish_person_erasure, finalize_person_erasure) named = _declared(orgs, org_free) register_buffer_drainer("boot-writes", boot.drain_writes) diff --git a/server/cli/restore.py b/server/cli/restore.py index 27b0965b85..4ae1945d41 100644 --- a/server/cli/restore.py +++ b/server/cli/restore.py @@ -110,6 +110,43 @@ async def _refusal(restored: str, point: datetime, *, live: bool, ch: Any) -> st return "" +async def _restore_private_sources(ch: Any, *, apply: bool) -> None: + from server.knowledge.app.connected_restore import restore_all + from server.knowledge.app.crawl_restore import restore_all as restore_crawls + + await restore_all(apply=apply, ch=ch) + await restore_crawls(apply=apply, ch=ch) + + +async def _replay_erasures( + store: Any, + point: datetime, + erasures: list[dict[str, Any]], + still_open: list[dict[str, Any]], +) -> int: + from server.erasure.app import replay as erasure_replay + + failed = 0 + for entry in erasures: + status = await _step(erasure_replay.replay(entry, store, point)) + print(f" erasure {entry['id']} ({entry['scope_type']}): {status}") + failed += status != "completed" + for entry in still_open: + failed += await _step(erasure_replay.requeue(entry, store, point)) == "failed" + return failed + + +async def _replay_redactions(redactions: list[dict[str, Any]], store: Any) -> int: + from server.memory.app import restore_reconcile + + failed = 0 + for entry in redactions: + state = await _step(restore_reconcile.replay_redaction(entry, store)) + print(f" redaction {entry['memory_id']} v{entry['version']}: {state}") + failed += state == "failed" + return failed + + async def reconcile( restored: str, point: datetime, *, apply: bool, live: bool = False, ch: Any ) -> int: @@ -147,9 +184,7 @@ async def reconcile( f"namespace {s['namespace']}, source_uri {s['source_uri']}" ) if not apply: - from server.knowledge.app.connected_restore import restore_all - - await restore_all(apply=False, ch=ch) + await _restore_private_sources(ch, apply=False) return 0 from server.code.app.index import repo_registry @@ -173,12 +208,10 @@ async def reconcile( # live head held here, which `loom rechunk` builds again from the stored content. for line in _rechunk_lines(await knowledge_reconcile.held_sources()): print(line) - from server.knowledge.app.connected_restore import restore_all - # Connection cleanup is fail-fast: no restored document may publish with # retained provider authority, even when an unrelated erasure replay can retry. try: - await restore_all(apply=True, ch=ch) + await _restore_private_sources(ch, apply=True) except Exception: # noqa: BLE001 — local cleanup must finish before any publication print( "loom reconcile: connected-account cleanup failed; publication remains held", @@ -187,20 +220,12 @@ async def reconcile( return 1 # A step that raises is counted and the rest run: what `hold` held is published # whatever an erasure did. - for entry in erasures: - status = await _step(erasure_replay.replay(entry, store, point)) - print(f" erasure {entry['id']} ({entry['scope_type']}): {status}") - failed += status != "completed" - for entry in still_open: - failed += await _step(erasure_replay.requeue(entry, store, point)) == "failed" + failed += await _replay_erasures(store, point, erasures, still_open) if store == "clickhouse": memory = await restore_reconcile.publish_held() knowledge = await knowledge_reconcile.publish_held() print(f" published: {memory} memory revisions, {knowledge} knowledge documents") - for entry in redactions: - state = await _step(restore_reconcile.replay_redaction(entry, store)) - print(f" redaction {entry['memory_id']} v{entry['version']}: {state}") - failed += state == "failed" + failed += await _replay_redactions(redactions, store) if failed: print(f"loom reconcile: {failed} step(s) did not complete; run again", file=sys.stderr) return 1 diff --git a/server/cli/secrets.py b/server/cli/secrets.py index 68234f214f..6e7b20bc96 100644 --- a/server/cli/secrets.py +++ b/server/cli/secrets.py @@ -26,7 +26,10 @@ ("loom.qa_sources", "password_enc", (), "qa_sources.password", ""), ("loom.knowledge_sources", "credential_enc", (), "knowledge_sources.credential", ""), ("loom.knowledge_sources", "secret_enc", (), "knowledge_sources.secret", ""), + ("loom.knowledge_sources", "approval_enc", (), "knowledge_crawl.approval", ""), + ("loom.knowledge_crawl_exclusion_keys", "key_enc", (), "knowledge_crawl.exclusion_key", ""), ("loom.knowledge_source_push", "secret_enc", (), "knowledge_source_push.secret", ""), + ("loom.scim_users", "crawl_erasure_enc", (), "auth.crawl_person_erasure", ""), ("loom.connected_accounts", "credential_enc", (), "connected_accounts.credential", ""), ("loom.connected_accounts", "account_email_enc", (), "connected_accounts.email", ""), ("loom.provider_configuration", "envelope", (), "model-provider-configuration", ""), @@ -58,6 +61,8 @@ "loom.knowledge_sources": ("org", "namespace", "name"), "loom.knowledge_source_push": ("org", "namespace", "source_id"), "loom.connected_accounts": ("org", "subject", "kind"), + "loom.knowledge_crawl_exclusion_keys": ("org",), + "loom.scim_users": ("id",), "loom.provider_configuration": ("config_id",), "loom.oauth_state": ("record_type", "id"), "loom.analysis_secrets": ("id",), @@ -131,6 +136,137 @@ async def _check_previous(ch: Any) -> bool: return bool(recorded) +def _resealed_rows( + table: str, + column: str, + path: tuple[str, ...], + purpose: str, + rows: list[dict[str, Any]], +) -> tuple[list[dict[str, Any]], list[str]]: + out = [] + unopened = [] + for row in rows: + new, opens = _resealed_value(str(row[column]), path, purpose) + if not opens: + unopened.append("/".join(str(row[k]) for k in _KEYS[table])) + if new is not None: + out.append({**row, column: new, "version": int(row["version"]) + 1}) + return out, unopened + + +async def _reseal_source_values( + ch: Any, + column: str, + purpose: str, + *, + dry_run: bool, +) -> tuple[int, list[str]]: + from server.core.ch import own_write_reads + from server.knowledge.app.sources import sources_lease + from server.knowledge.store import sources as source_store + + if column not in {"credential_enc", "secret_enc"}: + raise ValueError("source ciphertext column refused") + namespaces = await ch.query_all( + "SELECT DISTINCT org, namespace FROM loom.knowledge_sources FINAL " + f"WHERE {column} != '' ORDER BY org, namespace" + ) + count, lost = 0, [] + for scope in namespaces: + async with sources_lease(scope["org"], scope["namespace"], ch=ch): + # Canonical-person cleanup holds this same lease. The fresh read + # selects ciphertext; even a late mutation writes no metadata. + query = ( + "SELECT * FROM loom.knowledge_sources FINAL " + "WHERE org = {org:String} AND namespace = {ns:String} " + f"AND {column} != ''" + ) + params = {"org": scope["org"], "ns": scope["namespace"]} + with own_write_reads(): + rows = await ch.query_all(query, params=params) + for row in rows: + new, opens = _resealed_value(str(row[column]), (), purpose) + if not opens: + lost.append("/".join(str(row[k]) for k in _KEYS["loom.knowledge_sources"])) + if new is not None: + if not dry_run: + await source_store.reseal_source_ciphertext( + scope["org"], + scope["namespace"], + row["name"], + str(row["source_id"]), + column, + str(row[column]), + new, + ch=ch, + ) + count += 1 + return count, lost + + +async def _registry_reseal( + ch: Any, + table: str, + column: str, + purpose: str, + *, + dry_run: bool, +) -> tuple[int, list[str]] | None: + if table == "loom.knowledge_sources" and column in {"credential_enc", "secret_enc"}: + return await _reseal_source_values(ch, column, purpose, dry_run=dry_run) + if table == "loom.knowledge_crawl_exclusion_keys" or ( + table == "loom.knowledge_sources" and column == "approval_enc" + ): + from server.knowledge.app import crawl_reseal + + reseal = crawl_reseal.keys if column == "key_enc" else crawl_reseal.approvals + n, missing = await reseal(dry_run=dry_run, ch=ch) + return n, missing + if table == "loom.scim_users" and column == "crawl_erasure_enc": + from server.auth.person_erasure import reseal_registry as reseal_people + + n, missing = await reseal_people(dry_run=dry_run, ch=ch) + return n, missing + if table == "loom.connected_accounts": + from server.knowledge.app.connected_accounts import reseal_registry as reseal_accounts + + n, missing = await reseal_accounts(column, purpose, dry_run=dry_run, ch=ch) + return n, missing + if table == "loom.knowledge_source_push": + # This registry is physically erased. Its writer fence and re-read + # under the source lease prevent a scanned row from reviving it. + from server.knowledge.app.source_push import reseal_registry + + n, missing = await reseal_registry(dry_run=dry_run, ch=ch) + return n, missing + return None + + +async def _ordinary_reseal( + ch: Any, + table: str, + column: str, + path: tuple[str, ...], + purpose: str, + where: str, + *, + dry_run: bool, +) -> tuple[int, list[str]]: + async with contextlib.AsyncExitStack() as held: + if table in _LEASED: + from server.core.ch import own_write_reads + + await held.enter_async_context(_LEASED[table](ch)) + held.enter_context(own_write_reads()) + rows = await ch.query_all( + f"SELECT * FROM {table} FINAL WHERE {column} != ''" + (f" AND {where}" if where else "") + ) + out, missing = _resealed_rows(table, column, path, purpose, rows) + if out and not dry_run: + await ch.insert(table, out, settings={"async_insert": 0}) + return len(out), missing + + async def reseal_all( ch: Any, *, dry_run: bool ) -> tuple[dict[str, int], dict[str, list[str]], bool]: @@ -149,50 +285,15 @@ async def reseal_all( counts: dict[str, int] = {} lost: dict[str, list[str]] = {} for table, column, path, purpose, where in SEALED: - if table == "loom.connected_accounts": - from server.knowledge.app.connected_accounts import reseal_registry as reseal_accounts - - n, missing = await reseal_accounts(column, purpose, dry_run=dry_run, ch=ch) - counts[f"{table}.{column}"] = n - if missing: - lost[f"{table}.{column}"] = missing - continue - if table == "loom.knowledge_source_push": - # This registry is physically erased. Its writer fence and re-read - # under the source lease prevent a scanned row from reviving it. - from server.knowledge.app.source_push import reseal_registry - - n, missing = await reseal_registry(dry_run=dry_run, ch=ch) - counts[f"{table}.{column}"] = n - if missing: - lost[f"{table}.{column}"] = missing - continue - async with contextlib.AsyncExitStack() as held: - if table in _LEASED: - # One writer of the row at a time, reading the newest copy. - from server.core.ch import own_write_reads - - await held.enter_async_context(_LEASED[table](ch)) - held.enter_context(own_write_reads()) - rows = await ch.query_all( - f"SELECT * FROM {table} FINAL WHERE {column} != ''" - + (f" AND {where}" if where else "") + result = await _registry_reseal(ch, table, column, purpose, dry_run=dry_run) + if result is None: + result = await _ordinary_reseal( + ch, table, column, path, purpose, where, dry_run=dry_run ) - out = [] - unopened: list[str] = [] - for row in rows: - new, opens = _resealed_value(str(row[column]), path, purpose) - if not opens: - unopened.append("/".join(str(row[k]) for k in _KEYS[table])) - if new is not None: - # The lowest version above the one read: an edit written - # meanwhile (at the server's clock) stays the newer row. - out.append({**row, column: new, "version": int(row["version"]) + 1}) - counts[f"{table}.{column}"] = len(out) - if unopened: - lost[f"{table}.{column}"] = unopened - if out and not dry_run: - await ch.insert(table, out, settings={"async_insert": 0}) + n, missing = result + counts[f"{table}.{column}"] = n + if missing: + lost[f"{table}.{column}"] = missing return counts, lost, checked diff --git a/server/clients/gdrive.py b/server/clients/gdrive.py index ff861138d0..3e51989397 100644 --- a/server/clients/gdrive.py +++ b/server/clients/gdrive.py @@ -16,6 +16,7 @@ import re import time from dataclasses import dataclass, field +from enum import StrEnum from typing import Any from server.clients._checked_access import Credential @@ -67,6 +68,19 @@ class ServiceAccount: private_key: str = field(repr=False) +class ReadScope(StrEnum): + DRIVE = _SCOPE + DIRECTORY_USERS = "https://www.googleapis.com/auth/admin.directory.user.readonly" + DIRECTORY_MEMBERS = "https://www.googleapis.com/auth/admin.directory.group.member.readonly" + + +class WriteScope(StrEnum): + DRIVE = "https://www.googleapis.com/auth/drive" + DIRECTORY_USERS = "https://www.googleapis.com/auth/admin.directory.user" + DIRECTORY_GROUPS = "https://www.googleapis.com/auth/admin.directory.group" + DIRECTORY_MEMBERS = "https://www.googleapis.com/auth/admin.directory.group.member" + + def parse_service_account(credential: str) -> ServiceAccount: """The service account a JSON key names; CredentialRefused when it is not one.""" try: @@ -82,23 +96,41 @@ def parse_service_account(credential: str) -> ServiceAccount: return ServiceAccount(client_email=email, private_key=key) -async def access_token(account: ServiceAccount) -> str: - """A read-only access token for the service account.""" +def _signed_assertion( + account: ServiceAccount, *, scope: ReadScope | WriteScope, subject: str | None = None +) -> str: import jwt + if not isinstance(scope, (ReadScope, WriteScope)): + raise CredentialRefused("the delegated scope is not supported") + if subject is not None and ( + not isinstance(subject, str) + or len(subject) > 512 + or subject.count("@") != 1 + or not all(subject.split("@")) + or not re.fullmatch(r"[A-Za-z0-9@._+\-]+", subject) + ): + raise CredentialRefused("the delegated subject is not an email") now = int(time.time()) claims = { "iss": account.client_email, - "scope": _SCOPE, + "scope": scope.value, "aud": _TOKEN_URL, "iat": now, "exp": now + 600, } + if subject is not None: + claims["sub"] = subject try: - assertion = jwt.encode(claims, account.private_key, algorithm="RS256") + return jwt.encode(claims, account.private_key, algorithm="RS256") # A public key or a non-RSA key raises AttributeError or InvalidKeyError here. except (ValueError, TypeError, AttributeError, jwt.PyJWTError): raise CredentialRefused("the service account's private_key is not an RSA key") from None + + +async def access_token(account: ServiceAccount) -> str: + """A read-only access token for the service account.""" + assertion = _signed_assertion(account, scope=ReadScope.DRIVE) answer = await exchange_token( get_client(), _TOKEN_URL, @@ -223,6 +255,7 @@ async def list_all( modified_after: str = "", fields: str = LISTED, pace_s: float = PAGE_PACE_S, + limit: int | None = None, ) -> tuple[list[dict[str, Any]], bool]: """Every file of `mime_types` in any of `folder_ids`, oldest change first, or only those changed after `modified_after` (a `valid_stamp`), and whether the store @@ -230,9 +263,12 @@ async def list_all( files: list[dict[str, Any]] = [] whole = True for start in range(0, len(folder_ids), MAX_PARENTS): + if limit is not None and len(files) > limit: + break query = folder_query(folder_ids[start : start + MAX_PARENTS], mime_types) + bound: dict[str, Any] = {"limit": limit - len(files)} if limit is not None else {} found, complete = await list_query( - token, query, modified_after=modified_after, fields=fields, pace_s=pace_s + token, query, modified_after=modified_after, fields=fields, pace_s=pace_s, **bound ) files.extend(found) whole = whole and complete diff --git a/server/clients/google_workspace.py b/server/clients/google_workspace.py new file mode 100644 index 0000000000..6e0bbd886f --- /dev/null +++ b/server/clients/google_workspace.py @@ -0,0 +1,417 @@ +"""Delegated Google reads; provider identity is not Loom source or reader authority. + +The caller supplies finite I/O bounds. Crawl registration, registrant standing, +complete expansion, exclusions and publication are outside this transport boundary. +No request retries or follows redirects; provider errors retain no body or URL. +""" + +from __future__ import annotations + +import asyncio +import json +import math +import re +import time +from collections.abc import Awaitable, Callable +from dataclasses import dataclass +from typing import Any, Literal, SupportsIndex, cast +from urllib.parse import quote + +import httpx + +from server.clients import gdrive +from server.clients._http import get_client +from server.clients._vendor import ( + CredentialRefused, + TooLarge, + VendorError, + raise_for_status, + read_capped, +) + +_now = time.monotonic +_pace = asyncio.sleep +_TOKEN_URL = "https://oauth2.googleapis.com/token" +_DIRECTORY = "https://admin.googleapis.com/admin/directory/v1" +# not a setting: Directory members.list has a fixed documented page maximum. +_PAGE_SIZE = 200 +# not a setting: the selector wire protocol caps the total people at 2000. +MAX_PEOPLE = 2000 +type MemberKind = Literal["USER", "GROUP", "CUSTOMER"] + + +def _positive(value: int) -> None: + if type(value) is not int or value <= 0: + raise ValueError("a positive integer bound is required") + + +def _remaining(deadline: float) -> float: + if type(deadline) not in (int, float): + raise TimeoutError("delegated read deadline") + try: + remaining = deadline - _now() + except OverflowError: + raise TimeoutError("delegated read deadline") from None + if not math.isfinite(remaining) or remaining <= 0: + raise TimeoutError("delegated read deadline") + return remaining + + +def _text(value: Any, *, maximum: int = 8192) -> str: + if ( + not isinstance(value, str) + or not value + or len(value) > maximum + or any(not 33 <= ord(c) <= 126 for c in value) + ): + raise VendorError("invalid directory response") + return value + + +def _key(value: str) -> str: + if ( + not isinstance(value, str) + or value in {".", ".."} + or not re.fullmatch(r"[A-Za-z0-9@._+\-]{1,512}", value) + or "@" in value + and (value.count("@") != 1 or not all(value.split("@"))) + ): + raise ValueError("directory key refused") + return quote(value, safe="") + + +class DelegatedToken: + __slots__ = ("_value", "subject", "scope", "deadline") + + def __init__(self, value: str, subject: str, scope: gdrive.ReadScope, deadline: float) -> None: + self._value, self.subject, self.scope, self.deadline = value, subject, scope, deadline + + def __repr__(self) -> str: + return "DelegatedToken()" + + def __str__(self) -> str: + raise TypeError("a delegated token is not a shared credential") + + def __reduce_ex__(self, protocol: SupportsIndex) -> Any: + raise TypeError("a delegated token cannot be copied or serialized") + + +async def _directory_forbidden(response: httpx.Response, max_bytes: int, deadline: float) -> None: + """Classify bounded reasons without retaining an echoed provider error in the message.""" + raw = await read_capped(response, max_bytes) + _remaining(deadline) + if raw is None: + raise TooLarge("delegated response exceeds its byte bound", status=403) + try: + value = json.loads(raw) + except (ValueError, RecursionError): + raise VendorError("delegated read unavailable", status=403) from None + error = value.get("error") if isinstance(value, dict) else None + if not isinstance(error, dict): + raise VendorError("delegated read unavailable", status=403) + entries = error.get("errors") + reasons = ( + [ + entry["reason"] + for entry in entries + if isinstance(entry, dict) and isinstance(entry.get("reason"), str) and entry["reason"] + ] + if isinstance(entries, list) + else [] + ) + status = error.get("status") + if isinstance(status, str) and status: + reasons.append(status) + if not reasons: + # An unreadable quota response is not evidence of a revoked credential. + raise VendorError("delegated read unavailable", status=403) + classified = httpx.Response(403, json={"error": {"errors": [{"reason": r} for r in reasons]}}) + raise_for_status(classified, "delegated read") + + +async def _request( + method: Literal["GET", "POST"], + url: str, + *, + deadline: float, + max_bytes: int, + headers: dict[str, str] | None = None, + params: dict[str, Any] | None = None, + data: dict[str, str] | None = None, +) -> dict[str, Any]: + _positive(max_bytes) + remaining = _remaining(deadline) + try: + async with asyncio.timeout_at(deadline): + async with get_client().stream( + method, + url, + headers=headers, + params=params, + data=data, + timeout=remaining, + follow_redirects=False, + ) as response: + status = response.status_code + if status != 200: + if status == 403 and url.startswith(_DIRECTORY + "/"): + await _directory_forbidden(response, max_bytes, deadline) + if status == 400 and url == _TOKEN_URL: + raise CredentialRefused("delegated read refused", status=status) + raise_for_status(httpx.Response(status), "delegated read") + raise VendorError("delegated read refused", status=status) + raw = await read_capped(response, max_bytes) + except httpx.RequestError: + raise VendorError("delegated read unavailable") from None + _remaining(deadline) + if raw is None: + raise TooLarge("delegated response exceeds its byte bound") + try: + value = json.loads(raw) + except (ValueError, RecursionError): + raise VendorError("invalid delegated response") from None + if not isinstance(value, dict) or value.get("error") or value.get("errors"): + raise VendorError("invalid delegated response") + _remaining(deadline) + return value + + +async def delegated_token( + account: gdrive.ServiceAccount, + *, + subject: str, + scope: gdrive.ReadScope, + deadline: float, + max_bytes: int, +) -> DelegatedToken: + """An exact read-scope grant; neither scope probing nor source admission.""" + _positive(max_bytes) + _remaining(deadline) + if not isinstance(subject, str): + raise CredentialRefused("the delegated subject is not an email") + assertion = gdrive._signed_assertion(account, scope=scope, subject=subject) + value = await _request( + "POST", + _TOKEN_URL, + deadline=deadline, + max_bytes=max_bytes, + data={"grant_type": "urn:ietf:params:oauth:grant-type:jwt-bearer", "assertion": assertion}, + ) + token = _text(value.get("access_token")) + expires = value.get("expires_in") + token_type = value.get("token_type") + if ( + not isinstance(token_type, str) + or token_type.lower() != "bearer" + or type(expires) is not int + or expires <= 0 + or "scope" in value + and (not isinstance(value["scope"], str) or value["scope"].split() != [scope.value]) + ): + raise CredentialRefused("delegated token response refused") + remaining = _remaining(deadline) + expires_at = _now() + min(expires, remaining) + return DelegatedToken(token, subject, scope, min(deadline, expires_at)) + + +def _authorization(token: DelegatedToken, scope: gdrive.ReadScope) -> str: + if token.scope is not scope: + raise CredentialRefused("directory scope refused") + _remaining(token.deadline) + return "Bearer " + _text(token._value) + + +async def require_write_denied( + account: gdrive.ServiceAccount, + *, + subject: str, + scope: gdrive.WriteScope, + deadline: float, + max_bytes: int, +) -> None: + """Only a bounded, explicit OAuth scope denial proves this fixed grant absent.""" + _positive(max_bytes) + if not isinstance(scope, gdrive.WriteScope) or not isinstance(subject, str): + raise CredentialRefused("write-scope proof refused") + assertion = gdrive._signed_assertion(account, scope=scope, subject=subject) + try: + async with asyncio.timeout_at(deadline): + async with get_client().stream( + "POST", + _TOKEN_URL, + data={ + "grant_type": "urn:ietf:params:oauth:grant-type:jwt-bearer", + "assertion": assertion, + }, + timeout=_remaining(deadline), + follow_redirects=False, + ) as response: + if response.status_code != 400: + raise_for_status(httpx.Response(response.status_code), "write-scope proof") + raise CredentialRefused("write delegation is not refused") + raw = await read_capped(response, max_bytes) + except httpx.RequestError: + raise VendorError("write-scope proof unavailable") from None + _remaining(deadline) + if raw is None: + raise TooLarge("write-scope proof exceeds its byte bound") + try: + value = json.loads(raw) + except (ValueError, RecursionError): + raise VendorError("write-scope proof unavailable") from None + if not isinstance(value, dict) or value.get("error") not in { + "unauthorized_client", + "invalid_scope", + }: + raise CredentialRefused("write-scope proof is not an explicit scope denial") + + +@dataclass(frozen=True, repr=False) +class DirectoryUser: + id: str + email: str + aliases: tuple[str, ...] + customer_id: str + + def __repr__(self) -> str: + return "DirectoryUser()" + + +def _user(value: dict[str, Any]) -> DirectoryUser: + if ( + value.get("suspended") is not False + or value.get("archived") is not False + or "deletionTime" in value + ): + raise CredentialRefused("directory person unavailable") + aliases = value.get("aliases", []) + if not isinstance(aliases, list): + raise VendorError("invalid directory response") + email = _email(value.get("primaryEmail")) + resolved_aliases = tuple(_email(alias) for alias in aliases) + return DirectoryUser( + _text(value.get("id"), maximum=256), + email, + resolved_aliases, + _text(value.get("customerId"), maximum=256), + ) + + +def _email(value: Any) -> str: + email = _text(value, maximum=512) + if email.count("@") != 1 or not all(email.split("@")): + raise VendorError("invalid directory response") + return email + + +async def directory_identity(token: DelegatedToken, *, max_bytes: int) -> DirectoryUser: + """Resolve the delegated directory principal, without inventing a Loom identity.""" + return await _get_user(token, token.subject, max_bytes=max_bytes) + + +async def user( + token: DelegatedToken, key: str, *, customer_id: str, max_bytes: int +) -> DirectoryUser: + """A current active directory user in the explicitly expected workspace customer.""" + expected = _text(customer_id, maximum=256) + found = await _get_user(token, key, max_bytes=max_bytes) + if found.customer_id != expected: + raise CredentialRefused("directory person unavailable") + return found + + +async def _get_user(token: DelegatedToken, key: str, *, max_bytes: int) -> DirectoryUser: + headers = {"Authorization": _authorization(token, gdrive.ReadScope.DIRECTORY_USERS)} + value = await _request( + "GET", + f"{_DIRECTORY}/users/{_key(key)}", + headers=headers, + params={"fields": "id,primaryEmail,aliases,customerId,suspended,archived,deletionTime"}, + deadline=token.deadline, + max_bytes=max_bytes, + ) + found = _user(value) + names = {email.lower() for email in (found.email, *found.aliases)} + if key != found.id and key.lower() not in names: + raise VendorError("directory identity mismatch") + return found + + +@dataclass(frozen=True, repr=False) +class DirectoryMember: + id: str + kind: MemberKind + email: str | None + + def __repr__(self) -> str: + return "DirectoryMember()" + + +def _member(value: Any) -> DirectoryMember: + if not isinstance(value, dict): + raise VendorError("invalid directory response") + kind = value.get("type") + if not isinstance(kind, str) or kind not in {"USER", "GROUP", "CUSTOMER"}: + raise VendorError("invalid directory response") + email = value.get("email") + if email is not None: + email = _email(email) + return DirectoryMember(_text(value.get("id"), maximum=256), cast(MemberKind, kind), email) + + +async def group_members( + token: DelegatedToken, + key: str, + *, + max_bytes: int, + max_pages: int, + before_page: Callable[[], Awaitable[None]] | None = None, +) -> tuple[DirectoryMember, ...]: + """Complete bounded pages of metadata; USER/status is no proof of an active employee. + + GROUP and CUSTOMER are explicit results, never automatically expanded. A caller + must resolve USER identities and customer membership before using them as readers. + """ + _positive(max_pages) + headers = {"Authorization": _authorization(token, gdrive.ReadScope.DIRECTORY_MEMBERS)} + url = f"{_DIRECTORY}/groups/{_key(key)}/members" + cursor = "" + cursors: set[str] = set() + members: dict[tuple[str, str], DirectoryMember] = {} + for _ in range(max_pages): + if cursor: + async with asyncio.timeout_at(token.deadline): + await _pace(gdrive.PAGE_PACE_S) + if before_page is not None: + await before_page() + value = await _request( + "GET", + url, + deadline=token.deadline, + max_bytes=max_bytes, + headers=headers, + params={ + "maxResults": _PAGE_SIZE, + "includeDerivedMembership": "false", + "fields": "members(id,type,email),nextPageToken", + **({"pageToken": cursor} if cursor else {}), + }, + ) + rows = value.get("members", []) + if not isinstance(rows, list) or len(rows) > _PAGE_SIZE: + raise VendorError("invalid directory response") + for row in rows: + member = _member(row) + identity = member.kind, member.id + if identity in members: + raise VendorError("directory membership changed during pagination") + members[identity] = member + if len(members) > MAX_PEOPLE: + raise TooLarge("directory membership exceeds the crawl bound") + if "nextPageToken" not in value: + return tuple(members.values()) + cursor = _text(value["nextPageToken"]) + if cursor in cursors: + raise VendorError("directory pagination did not advance") + cursors.add(cursor) + raise TooLarge("directory membership exceeds its page bound") diff --git a/server/core/ch.py b/server/core/ch.py index 15170e721f..4e72b03640 100644 --- a/server/core/ch.py +++ b/server/core/ch.py @@ -723,9 +723,11 @@ def _stray_columns_sql(table: str, stored: frozenset[str]) -> str: def write_limit_s() -> float: """The longest a leased write runs on the server, and a carried one sent with no longer limit of its own: the configured write timeout, or `_FENCE_MAX_S` when it sets - none.""" + none. A positive scoped statement bound can only shorten this receiver limit.""" limit = float(settings.ch_timeout_write_s) - return _setting_seconds(limit) if limit > 0 else _FENCE_MAX_S + bound = _setting_seconds(limit) if limit > 0 else _FENCE_MAX_S + scoped = statement_timeout_var.get() + return min(bound, _setting_seconds(scoped)) if scoped > 0 else bound def start_guard(deadline_ms_sql: str) -> str: diff --git a/server/core/leases.py b/server/core/leases.py index 30a8e1572f..cc7cacdbe1 100644 --- a/server/core/leases.py +++ b/server/core/leases.py @@ -397,32 +397,53 @@ class ClaimExpired(LeaseUnavailable): @asynccontextmanager async def holding( - name: str, *, ttl_s: float, wait_s: float = 0.0, ch: Any = None + name: str, + *, + ttl_s: float, + wait_s: float = 0.0, + ch: Any = None, + settle_writes: bool = False, ) -> AsyncIterator[bool]: - """Claim `name` for the block, waiting up to `wait_s`: yields True when this - caller has it, False when another holder keeps it. LeaseUnavailable when the - store cannot answer, ClaimExpired when a held block runs past `_WORK_SHARE` of - the TTL.""" + """Claim a name; an unanswered fenced write keeps its key until safe expiry. + + With settlement, every INSERT must end before the claim's server-side TTL. + A client cancellation or lost reply cannot release the key early. The next + holder therefore cannot scrub raw history before an older send has ended. + """ + from server.core import rmt_carry from server.core.ch import statement_timeout lease = await take(name, ttl_s=ttl_s, wait_s=wait_s, ch=ch) if lease is None: yield False return - # From when the winning attempt was sent, not when its reply came: a reply held - # up past the TTL must not open the block beside the claim's next holder. - left = work_left(lease, ttl_s) + end = lease.renewed_at + ttl_s * _WORK_SHARE + if settle_writes: + with statement_timeout(statement_bound(ttl_s)): + end = min(end, lease.renewed_at + ttl_s - rmt_carry.send_bound_s()) + # The winning send predates the store's acquisition stamp. Its local TTL end + # is conservative even when the winning reply is delayed. + pending: set[str] = set() try: + left = end - time.monotonic() if left <= 0: raise TimeoutError async with asyncio.timeout(left): with statement_timeout(statement_bound(ttl_s)): - yield True + if settle_writes: + with rmt_carry.claim_writes(end) as pending: + yield True + if pending: + raise LeaseUnavailable("claim has unanswered writes", operation=name) + else: + yield True except TimeoutError as e: raise ClaimExpired( f"the work under claim {name} did not finish in time", operation=name ) from e finally: + if pending: + lease.let_expire() await release_shielded(lease) diff --git a/server/core/rmt_carry.py b/server/core/rmt_carry.py index c1996b5322..14526163f4 100644 --- a/server/core/rmt_carry.py +++ b/server/core/rmt_carry.py @@ -116,6 +116,10 @@ class _Pin: _lease_end: contextvars.ContextVar[float | None] = contextvars.ContextVar( "rmt_carry_lease_end", default=None ) +# All enclosing claims retain unanswered sends, including sends in nested claims. +_claim_writes: contextvars.ContextVar[tuple[set[str], ...]] = contextvars.ContextVar( + "rmt_claim_writes", default=() +) _pinned: list[_Pin] = [] # How long a send may take to reach the server and start: a request still # queued past this is refused by its guard. @@ -333,6 +337,8 @@ async def writing() -> AsyncIterator[None]: def note_ended(query_id: str) -> None: """The server answered this send (its rows, or an error of its own): the query is over and holds nothing back.""" + for pending in _claim_writes.get(): + pending.discard(query_id) write = _carried.get() if write is not None: write.ended.add(query_id) @@ -341,6 +347,8 @@ def note_ended(query_id: str) -> None: def note_send(query_id: str, deadline: float = 0.0, max_s: float = 0.0) -> None: """The client sent one batch of this carried write as `query_id`, to start by `deadline` (server clock) and run for at most `max_s`.""" + for pending in _claim_writes.get(): + pending.add(query_id) write = _carried.get() if write is not None: write.query_ids.append(query_id) @@ -380,6 +388,18 @@ def under_lease(ends_at: float) -> Iterator[None]: _lease_end.reset(token) +@contextlib.contextmanager +def claim_writes(ends_at: float) -> Iterator[set[str]]: + """Track sends without server completion in every enclosing claim.""" + pending: set[str] = set() + token = _claim_writes.set((*_claim_writes.get(), pending)) + try: + with under_lease(ends_at): + yield pending + finally: + _claim_writes.reset(token) + + def sent_under_lease() -> bool: return _lease_end.get() is not None diff --git a/server/erasure/app/crawl_person.py b/server/erasure/app/crawl_person.py new file mode 100644 index 0000000000..775d22a520 --- /dev/null +++ b/server/erasure/app/crawl_person.py @@ -0,0 +1,117 @@ +"""Canonical-person crawl erasure orders raw stores and verifies completed scope cleanup.""" + +from __future__ import annotations + +from typing import Any + +from server.auth import person_erasure, source_registration +from server.clients._vendor import CredentialRefused +from server.core.ch import set_query_tag +from server.erasure.store import crawl_controls, crawl_person + + +async def erase_source( + pending: person_erasure.Pending, + person: person_erasure.Person, + namespace: str, + source_id: str, + contacts: frozenset[str], + *, + ch: Any, +) -> None: + """The knowledge caller retains its org exclusion fence and exact source lease.""" + set_query_tag("erasure.crawl_person.source") + allowed = frozenset( + { + *person.emails, + *( + email + for w in pending.captured.workspaces + if (w.org, w.subject) == (person.org, person.subject) + for email in w.emails + ), + } + ) + if person not in pending.captured.people or contacts != allowed: + raise person_erasure.unavailable() + await pending.validate(person.org) + for encoded in await crawl_controls.approval_history(person.org, namespace, source_id, ch=ch): + if source_registration.names_person( + encoded, + org=person.org, + namespace=namespace, + source_id=source_id, + spellings=person.spellings, + ): + await crawl_controls.retire_approval(person.org, namespace, source_id, encoded, ch=ch) + copies = await crawl_person.namespaces(person.org, source_id, ch=ch) + for mirror in copies: + # A complete pre-erasure listing cannot admit a post-erasure union. + await crawl_controls.invalidate_listing(person.org, mirror, source_id, ch=ch) + await crawl_person.scrub(person.org, mirror, source_id, contacts, ch=ch) + await crawl_controls.memberships(person.org, mirror, source_id, contacts, ch=ch) + await pending.validate(person.org) + await crawl_controls.controls( + person.org, + namespace, + source_id, + person.subject, + contacts, + spellings=person.spellings, + ch=ch, + ) + await pending.validate(person.org) + # Removing a listing-only namespace may shrink discovery; newly appeared copies + # must not be reported erased without another complete guarded pass. + remaining = await crawl_person.namespaces(person.org, source_id, ch=ch) + if not set(remaining).issubset(copies): + raise CredentialRefused("crawl person copies changed during cleanup") + + +async def finalize(pending: person_erasure.Pending) -> None: + """The SCIM caller holds its user lease after all exact-person cleanup succeeded.""" + set_query_tag("erasure.crawl_person.capture") + await pending.validate() + history = await crawl_controls.capsule_history(pending.captured.id) + if pending.encoded not in history: + raise CredentialRefused("crawl person capture history unavailable") + for encoded in history: + previous = person_erasure.load(pending.captured.id, encoded) + if any( + not any( + (old.org, old.subject) == (new.org, new.subject) + and set(old.emails).issubset(new.emails) + and set(old.spellings).issubset(new.spellings) + for new in pending.captured.people + ) + for old in previous.people + ) or any( + not any( + (old.org, old.subject, old.customer, old.user) + == (new.org, new.subject, new.customer, new.user) + and set(old.emails).issubset(new.emails) + for new in pending.captured.workspaces + ) + for old in previous.workspaces + ): + raise CredentialRefused("historical crawl person capture requires verified repair") + await pending.validate() + await crawl_controls.capsules(pending.captured.id) + + +async def require_refusals( + org: str, + digests: list[str], + *, + excluded: bool, + ch: Any, +) -> None: + from server.erasure.store.census import crawl_refusal_residue + + if await crawl_refusal_residue(org, digests, excluded=excluded, ch=ch): + raise person_erasure.unavailable() + + +async def invalidate_restored_approvals(org: str, *, ch: Any) -> None: + """Called under the org fence after every historical source has retirement work.""" + await crawl_controls.invalidate_restored_approvals(org, ch=ch) diff --git a/server/erasure/store/census.py b/server/erasure/store/census.py index ce1551e109..ecb0e48dbe 100644 --- a/server/erasure/store/census.py +++ b/server/erasure/store/census.py @@ -62,6 +62,16 @@ } +# Canonical-person cleanup has no memory user_id or namespace axis. Capsules are +# physically scrubbed; keyed refusal intent deliberately survives that scrub so +# a rescan, sign-in recreation or restored source cannot readmit the person. +CANONICAL_PERSON_TABLES: dict[str, str] = { + "loom.scim_users": "crawl_person.finalize", + "loom.knowledge_crawl_exclusion_keys": "crawl_person.exclude", + "loom.knowledge_crawl_exclusions": "crawl_person.exclude", +} + + # One census row per substrate: the live predicate the census counts ("1" = # every row), the user-scope predicate (None → namespace-only), and the saga # step that owns it in each scope. Zero live rows in scope is the only pass. @@ -691,10 +701,6 @@ def owner(self, scope_type: str) -> str | None: "loom.sso_links": ( "auth plane: verified account links and signed issuer/domain evidence; no memory content." ), - "loom.scim_users": ( - "auth plane: the people the identity provider provisions (email, the emails " - "list it sent, userName, active) and the subjects a deactivation revoked; no memory." - ), "loom.oauth_signin_state": ( "auth plane: Keeper sign-in state (codes, devices, sessions), including optional " "verified sign-in email metadata; no memory content." @@ -781,7 +787,7 @@ def owner(self, scope_type: str) -> str | None: def classified_tables() -> dict[str, str]: """Every table the registry knows → which list holds it - (`census` | `step_only` | `no_memory_content` | `canonical_owner`).""" + (`census` | `step_only` | `no_memory_content` | `canonical_owner` | `canonical_person`).""" out: dict[str, str] = {} for sub in _CENSUS: out[sub.table] = "census" @@ -792,6 +798,8 @@ def classified_tables() -> dict[str, str]: out.setdefault(t, "no_memory_content") for t in CANONICAL_OWNER_TABLES: out.setdefault(t, "canonical_owner") + for t in CANONICAL_PERSON_TABLES: + out.setdefault(t, "canonical_person") return out @@ -1066,3 +1074,30 @@ async def connected_restore_residue(*, ch: _CHClient | None = None) -> int: raise RuntimeError("connected_restore_census_unavailable") total += int(row["n"]) return total + + +async def crawl_refusal_residue( + org: str, + digests: list[str], + *, + excluded: bool, + ch: _CHClient | None = None, +) -> int: + """Count missing or contrary retained refusal intent before releasing its fence.""" + if not digests or any( + len(value) != 64 or any(char not in "0123456789abcdef" for char in value) + for value in digests + ): + raise ValueError("crawl refusal census scope refused") + requested = set(digests) + rows = await (ch or _default_client()).query_all( + "SELECT digest, excluded FROM loom.knowledge_crawl_exclusions FINAL " + "WHERE org = {org:String} AND digest IN {digests:Array(String)} LIMIT {limit:UInt32}", + params={"org": org, "digests": sorted(requested), "limit": len(requested)}, + ) + if len(rows) != len({row["digest"] for row in rows}) or any( + row["digest"] not in requested or row["excluded"] not in (0, 1) for row in rows + ): + raise RuntimeError("crawl_refusal_census_unverified") + verified = {row["digest"] for row in rows if bool(row["excluded"]) is excluded} + return len(requested - verified) diff --git a/server/erasure/store/crawl_controls.py b/server/erasure/store/crawl_controls.py new file mode 100644 index 0000000000..bfd170ef18 --- /dev/null +++ b/server/erasure/store/crawl_controls.py @@ -0,0 +1,309 @@ +"""Exact-person selector and membership redaction keeps other administrators' keys.""" + +from __future__ import annotations + +from typing import Any + +from server.core.ch import own_write_reads, set_query_tag +from server.core.repo_base import _CHClient, _default_client +from server.erasure.store.mutations import count_then_mutate +from server.knowledge.domain.source_crawl import email, without_selectors +from server.settings import settings + +_SOURCES = "loom.knowledge_sources" +_GROUPS = "loom.knowledge_group_members" +_SCOPE = "org = {org:String} AND namespace = {ns:String} AND source_id = {id:String}" +_CRAWL_SCOPE = _SCOPE + " AND credential_mode = 'crawl'" + + +def _params(org: str, namespace: str, source_id: str) -> dict[str, str]: + if any(not isinstance(v, str) or not v or "\0" in v for v in (org, namespace, source_id)): + raise ValueError("crawl cleanup scope refused") + return {"org": org, "ns": namespace, "id": source_id} + + +async def _history(client: _CHClient, params: dict[str, Any]) -> list[dict[str, str]]: + maximum = settings.knowledge_crawl_registry_scan_limit + with own_write_reads(): + rows = await client.query_all( + f"SELECT DISTINCT options_json, registrant_subject FROM {_SOURCES} " + f"WHERE {_CRAWL_SCOPE} ORDER BY options_json, registrant_subject " + "LIMIT {limit:UInt32}", + params={**params, "limit": maximum + 1}, + ) + if len(rows) > maximum or any( + not isinstance(r.get(key), str) + for r in rows + for key in ("options_json", "registrant_subject") + ): + raise RuntimeError("crawl_control_history_unverified") + return rows + + +async def _retire(client: _CHClient, where: str, params: dict[str, Any]) -> None: + # A newer, independently approved configuration of the same source id is not + # retired merely because an older version named the deleted person. + await count_then_mutate( + _SOURCES, + count_where=where + " AND (active OR approval_enc != '')", + mutation="UPDATE active = 0, approval_enc = '', crawl_generation = ''", + mutation_where=where, + params=params, + final=False, + ch=client, + ) + + +async def controls( + org: str, + namespace: str, + source_id: str, + subject: str, + addresses: frozenset[str], + *, + spellings: tuple[str, ...] = (), + ch: _CHClient | None = None, +) -> bool: + """Caller holds the refusal fence and namespace lease; no new approval is minted.""" + set_query_tag("erasure.crawl_person.controls") + if not isinstance(subject, str) or not subject or "\0" in subject: + raise ValueError("canonical crawl subject required") + addresses = frozenset(email(v) for v in addresses) + if not addresses: + raise ValueError("verified crawl contacts required") + client = ch or _default_client() + params: dict[str, Any] = { + **_params(org, namespace, source_id), + "spellings": sorted({subject, *spellings}), + } + rows = await _history(client, params) + rewrites = {r["options_json"]: without_selectors(r["options_json"], addresses) for r in rows} + affected = any(before != after for before, after in rewrites.items()) or any( + row["registrant_subject"] == subject for row in rows + ) + for before, after in rewrites.items(): + if before != after: + where = _CRAWL_SCOPE + " AND options_json = {before:String}" + # The resource hash no longer matches. Keep the shared service-principal + # key; only a checked administrator can approve this changed registration. + await _retire(client, where, {**params, "before": before}) + await count_then_mutate( + _SOURCES, + count_where=where, + mutation="UPDATE options_json = {after:String}", + mutation_where=where, + params={**params, "before": before, "after": after}, + final=False, + ch=client, + ) + for field in ("registrant_subject", "created_by"): + where = _CRAWL_SCOPE + ( + " AND registrant_subject = {subject:String}" + if field == "registrant_subject" + else " AND created_by IN {spellings:Array(String)}" + ) + if field == "registrant_subject": + await _retire(client, where, {**params, "subject": subject}) + await count_then_mutate( + _SOURCES, + count_where=where, + mutation=f"UPDATE {field} = ''", + mutation_where=where, + params={**params, "subject": subject}, + final=False, + ch=client, + ) + if await control_residue( + org, namespace, source_id, subject, addresses, spellings=spellings, ch=client + ): + raise RuntimeError("crawl_control_history_remains") + return affected + + +async def control_residue( + org: str, + namespace: str, + source_id: str, + subject: str, + addresses: frozenset[str], + *, + spellings: tuple[str, ...] = (), + ch: _CHClient | None = None, +) -> int: + client, params = ch or _default_client(), _params(org, namespace, source_id) + rows = await _history(client, params) + selectors = sum( + without_selectors(r["options_json"], addresses) != r["options_json"] for r in rows + ) + with own_write_reads(): + row = await client.query_one( + f"SELECT count() AS n FROM {_SOURCES} WHERE {_CRAWL_SCOPE} " + "AND (registrant_subject = {subject:String} " + "OR created_by IN {spellings:Array(String)})", + params={**params, "subject": subject, "spellings": sorted({subject, *spellings})}, + ) + if row is None or type(row.get("n")) is not int or row["n"] < 0: + raise RuntimeError("crawl_control_history_unverified") + return selectors + int(row["n"]) + + +async def approval_history( + org: str, + namespace: str, + source_id: str, + *, + ch: _CHClient | None = None, +) -> list[str]: + maximum = settings.knowledge_crawl_registry_scan_limit + with own_write_reads(): + rows = await (ch or _default_client()).query_all( + f"SELECT DISTINCT approval_enc FROM {_SOURCES} WHERE {_CRAWL_SCOPE} " + "AND approval_enc != '' ORDER BY approval_enc LIMIT {limit:UInt32}", + params={**_params(org, namespace, source_id), "limit": maximum + 1}, + ) + if len(rows) > maximum or any(not isinstance(r["approval_enc"], str) for r in rows): + raise RuntimeError("crawl_approval_history_unverified") + return [r["approval_enc"] for r in rows] + + +async def retire_approval( + org: str, + namespace: str, + source_id: str, + encoded: str, + *, + ch: _CHClient | None = None, +) -> None: + client = ch or _default_client() + where = _CRAWL_SCOPE + " AND approval_enc = {encoded:String}" + await _retire(client, where, {**_params(org, namespace, source_id), "encoded": encoded}) + with own_write_reads(): + row = await client.query_one( + f"SELECT count() AS n FROM {_SOURCES} WHERE {where}", + params={**_params(org, namespace, source_id), "encoded": encoded}, + ) + if row is None or type(row.get("n")) is not int or row["n"] != 0: + raise RuntimeError("crawl_approval_history_remains") + + +async def memberships( + org: str, + mirror: str, + source_id: str, + addresses: frozenset[str], + *, + ch: _CHClient | None = None, +) -> None: + """All membership versions lose only the requested person's verified contacts.""" + set_query_tag("erasure.crawl_person.memberships") + client = ch or _default_client() + params = {**_params(org, mirror, source_id), "emails": sorted(email(v) for v in addresses)} + if not params["emails"]: + raise ValueError("verified crawl contacts required") + where = _SCOPE + " AND hasAny(members, {emails:Array(String)})" + await count_then_mutate( + _GROUPS, + count_where=where, + mutation=( + "UPDATE members = arrayFilter(value -> NOT has({emails:Array(String)}, value), members)" + ), + mutation_where=where, + params=params, + final=False, + ch=client, + ) + with own_write_reads(): + row = await client.query_one( + f"SELECT count() AS n FROM {_GROUPS} WHERE {where}", params=params + ) + if row is None or type(row.get("n")) is not int or row["n"] != 0: + raise RuntimeError("crawl_membership_history_remains") + + +async def invalidate_listing( + org: str, + mirror: str, + source_id: str, + *, + ch: _CHClient | None = None, +) -> None: + client, params = ch or _default_client(), _params(org, mirror, source_id) + table = "loom.knowledge_source_listings" + await count_then_mutate( + table, + count_where=_SCOPE, + mutation="DELETE", + mutation_where=_SCOPE, + params=params, + final=False, + ch=client, + ) + with own_write_reads(): + row = await client.query_one( + f"SELECT count() AS n FROM {table} WHERE {_SCOPE}", params=params + ) + if row is None or type(row.get("n")) is not int or row["n"] != 0: + raise RuntimeError("crawl_listing_history_remains") + + +async def capsules(uid: str, *, ch: _CHClient | None = None) -> None: + """Under the exact SCIM user lease, remove every completed capture's encrypted PII.""" + if not isinstance(uid, str) or not uid or "\0" in uid: + raise ValueError("SCIM capture scope refused") + client, params = ch or _default_client(), {"id": uid} + where = "id = {id:String} AND (crawl_erasure_enc != '' OR NOT crawl_erasure_done)" + await count_then_mutate( + "loom.scim_users", + count_where=where, + mutation_where=where, + final=False, + mutation=( + "UPDATE crawl_erasure_enc = '', crawl_erasure_done = true, " + "crawl_erasure_required = true" + ), + params=params, + ch=client, + ) + with own_write_reads(): + row = await client.query_one( + f"SELECT count() AS n FROM loom.scim_users WHERE {where}", params=params + ) + if row is None or type(row.get("n")) is not int or row["n"] != 0: + raise RuntimeError("crawl_capture_history_remains") + + +async def capsule_history(uid: str, *, ch: _CHClient | None = None) -> list[str]: + if not isinstance(uid, str) or not uid or "\0" in uid: + raise ValueError("SCIM capture scope refused") + maximum = settings.knowledge_crawl_registry_scan_limit + with own_write_reads(): + rows = await (ch or _default_client()).query_all( + "SELECT DISTINCT crawl_erasure_enc FROM loom.scim_users " + "WHERE id = {id:String} AND crawl_erasure_enc != '' " + "ORDER BY crawl_erasure_enc LIMIT {limit:UInt32}", + params={"id": uid, "limit": maximum + 1}, + ) + if len(rows) > maximum or any(not isinstance(r["crawl_erasure_enc"], str) for r in rows): + raise RuntimeError("crawl_capture_history_unverified") + return [r["crawl_erasure_enc"] for r in rows] + + +async def invalidate_restored_approvals(org: str, *, ch: _CHClient) -> None: + """A restore never reactivates historical delegated source authority.""" + where = "org = {org:String} AND credential_mode = 'crawl'" + await count_then_mutate( + _SOURCES, + count_where=where + " AND (active OR approval_enc != '' OR crawl_generation != '')", + mutation="UPDATE active = 0, approval_enc = '', crawl_generation = ''", + mutation_where=where, + params={"org": org}, + final=False, + ch=ch, + ) + residue = await ch.query_one( + f"SELECT count() AS n FROM {_SOURCES} WHERE {where} " + "AND (active OR approval_enc != '' OR crawl_generation != '')", + params={"org": org}, + ) + if residue is None or residue["n"]: + raise RuntimeError("crawl_restore_approval_residue") diff --git a/server/erasure/store/crawl_person.py b/server/erasure/store/crawl_person.py new file mode 100644 index 0000000000..0cd6913333 --- /dev/null +++ b/server/erasure/store/crawl_person.py @@ -0,0 +1,209 @@ +"""Historical crawl ACL redaction preserves document text and other readers' grants.""" + +from __future__ import annotations + +from typing import Any + +from server.core import pg +from server.core.ch import own_write_reads, set_query_tag +from server.core.repo_base import _CHClient, _default_client +from server.erasure.store.mutations import count_then_mutate +from server.knowledge.domain.source_crawl import email, without_people +from server.knowledge.domain.sources import SYNC_WORKER_ACTOR +from server.settings import settings + +_CH_SCOPE = ( + "org = {org:String} AND namespace = {ns:String} " + "AND actor = {actor:String} AND has(tags, {tag:String})" +) +_PG_SCOPE = "org = $1 AND namespace = $2 AND row->>'actor' = $3 AND row->'tags' ? $4::text" + + +def _params(org: str, namespace: str, source_id: str) -> dict[str, str]: + if any(not isinstance(v, str) or not v or "\0" in v for v in (org, namespace, source_id)): + raise ValueError("crawl cleanup scope refused") + return { + "org": org, + "ns": namespace, + "actor": SYNC_WORKER_ACTOR, + "tag": "source-id:" + source_id, + } + + +async def _pg_values(conn: Any, args: list[str]) -> list[str]: + maximum = settings.knowledge_crawl_registry_scan_limit + rows = await conn.fetch( + "SELECT DISTINCT row->>'access_json' AS access_json FROM (" + f"SELECT row FROM loom.knowledge_heads WHERE {_PG_SCOPE} " + f"UNION ALL SELECT row FROM loom.knowledge_deliveries WHERE {_PG_SCOPE}" + ") AS copies ORDER BY access_json LIMIT $5", + *args, + maximum + 1, + ) + if len(rows) > maximum or any( + r["access_json"] is not None and not isinstance(r["access_json"], str) for r in rows + ): + raise RuntimeError("crawl_permission_history_bound") + return [r["access_json"] or "" for r in rows] + + +async def _pg_scrub(args: list[str], addresses: frozenset[str]) -> None: + if not pg.enabled(): + return + async with pg.transaction() as conn: + await conn.execute(f"SELECT pg_advisory_xact_lock({pg.KNOWLEDGE_LOCK_KEY})", *args[:2]) + await conn.execute( + f"SELECT 1 FROM loom.knowledge_heads WHERE {_PG_SCOPE} " + "ORDER BY org, namespace, source_uri FOR UPDATE", + *args, + ) + held = await conn.fetch( + f"SELECT {pg.LEASE_LEFT_S} AS left_s FROM loom.knowledge_deliveries d " + f"WHERE {_PG_SCOPE} FOR UPDATE", + *args, + ) + # Inspect every retained delivery, including already-redacted retries. Its stored + # fence keeps an earlier attempt's old publisher covered after a crash. + left = [float(row["left_s"]) for row in held] + for before in await _pg_values(conn, args): + after = without_people(before, addresses) + if after == before: + continue + await conn.execute( + "UPDATE loom.knowledge_heads SET row = jsonb_set(row, '{access_json}', " + f"to_jsonb($6::text), true) WHERE {_PG_SCOPE} " + "AND coalesce(row->>'access_json', '') = $5", + *args, + before, + after, + ) + await conn.execute( + "UPDATE loom.knowledge_deliveries SET row = jsonb_set(row, '{access_json}', " + "to_jsonb($6::text), true), lease_token = lease_token + 1, " + "fenced_until = GREATEST(fenced_until, CASE WHEN isfinite(lease_until) " + "THEN lease_until ELSE fenced_until END) " + f"WHERE {_PG_SCOPE} AND coalesce(row->>'access_json', '') = $5", + *args, + before, + after, + ) + # Existing in-memory publishers may still send the old row until their lease ends. + await pg.wait_out(left) + + +async def _ch_values(client: _CHClient, params: dict[str, str]) -> list[str]: + maximum = settings.knowledge_crawl_registry_scan_limit + with own_write_reads(): + rows = await client.query_all( + f"SELECT DISTINCT access_json FROM loom.documents WHERE {_CH_SCOPE} " + "ORDER BY access_json LIMIT {limit:UInt32}", + params={**params, "limit": maximum + 1}, + ) + if len(rows) > maximum or any(not isinstance(row["access_json"], str) for row in rows): + raise RuntimeError("crawl_permission_history_bound") + return [row["access_json"] for row in rows] + + +async def scrub( + org: str, + namespace: str, + source_id: str, + addresses: frozenset[str], + *, + ch: _CHClient | None = None, +) -> None: + """Caller holds its checked person-erasure fence and exact org→source leases.""" + set_query_tag("erasure.crawl_person.acl") + params = _params(org, namespace, source_id) + addresses = frozenset(email(value) for value in addresses) + if not addresses: + raise ValueError("crawl cleanup contacts required") + await _pg_scrub(list(params.values()), addresses) + client = ch or _default_client() + for before in await _ch_values(client, params): + after = without_people(before, addresses) + if after == before: + continue + where = _CH_SCOPE + " AND access_json = {before:String}" + await count_then_mutate( + "loom.documents", + count_where=where, + final=False, + mutation="UPDATE access_json = {after:String}", + mutation_where=where, + params={**params, "before": before, "after": after}, + ch=client, + ) + if await residue(org, namespace, source_id, addresses, ch=client): + raise RuntimeError("crawl_permission_history_remains") + + +async def residue( + org: str, + namespace: str, + source_id: str, + addresses: frozenset[str], + *, + ch: _CHClient | None = None, +) -> int: + """Raw history in both authorities, not FINAL-only positive serving rows.""" + params = _params(org, namespace, source_id) + total = 0 + for value in await _ch_values(ch or _default_client(), params): + total += without_people(value, addresses) != value + if pg.enabled(): + async with pg.transaction() as conn: + for value in await _pg_values(conn, list(params.values())): + total += without_people(value, addresses) != value + return total + + +async def namespaces(org: str, source_id: str, *, ch: _CHClient | None = None) -> list[str]: + """Discover raw retained copies, including older mirrors and pending PG deliveries.""" + if any(not isinstance(v, str) or not v or "\0" in v for v in (org, source_id)): + raise ValueError("crawl copy scope refused") + client = ch or _default_client() + maximum = settings.knowledge_crawl_registry_scan_limit + params = { + "org": org, + "id": source_id, + "actor": SYNC_WORKER_ACTOR, + "tag": "source-id:" + source_id, + "limit": maximum + 1, + } + with own_write_reads(): + rows = await client.query_all( + "SELECT DISTINCT namespace FROM (" + "SELECT namespace FROM loom.documents WHERE org = {org:String} " + "AND actor = {actor:String} AND has(tags, {tag:String}) UNION ALL " + "SELECT namespace FROM loom.knowledge_group_members WHERE org = {org:String} " + "AND source_id = {id:String} UNION ALL " + "SELECT namespace FROM loom.knowledge_source_listings WHERE org = {org:String} " + "AND source_id = {id:String}) ORDER BY namespace LIMIT {limit:UInt32}", + params=params, + ) + if pg.enabled(): + async with pg.transaction() as conn: + rows.extend( + await conn.fetch( + "SELECT DISTINCT namespace FROM (" + "SELECT namespace FROM loom.knowledge_heads WHERE org = $1 " + "AND row->>'actor' = $2 AND row->'tags' ? $3::text UNION ALL " + "SELECT namespace FROM loom.knowledge_deliveries WHERE org = $1 " + "AND row->>'actor' = $2 AND row->'tags' ? $3::text) AS copies " + "ORDER BY namespace LIMIT $4", + org, + SYNC_WORKER_ACTOR, + params["tag"], + maximum + 1, + ) + ) + if len(rows) > 2 * maximum or any( + not isinstance(r["namespace"], str) or not r["namespace"] or "\0" in r["namespace"] + for r in rows + ): + raise RuntimeError("crawl_copy_scope_unverified") + result = sorted({r["namespace"] for r in rows}) + if len(result) > maximum: + raise RuntimeError("crawl_copy_scope_unverified") + return result diff --git a/server/knowledge/api/models_sources.py b/server/knowledge/api/models_sources.py index 2a8c6c7e06..7e140f15fb 100644 --- a/server/knowledge/api/models_sources.py +++ b/server/knowledge/api/models_sources.py @@ -5,8 +5,9 @@ from typing import Annotated, Any, Literal -from pydantic import BaseModel, Field +from pydantic import BaseModel, ConfigDict, Field +from server.knowledge.domain.source_crawl import MAX_PEOPLE from server.knowledge.domain.sources import ( MAX_DEPTH, MAX_ITEMS, @@ -110,6 +111,8 @@ class KnowledgeSourceEntry(BaseModel): credential_mode: str = "shared" # Namespace-wide or checked mirrored document access. access: str = "namespace" + # Crawl selectors, only for a current org and namespace administrator. + crawl_options: dict[str, Any] = Field(default_factory=dict) # Canonical connected owner, only in the owner's permitted detail projection. owner_subject: str = "" # The things the source reads (folders, files, shared drives, filters, pages, @@ -167,6 +170,40 @@ class KnowledgeSourceListOutput(BaseModel): ) +class KnowledgeSourceCrawlSubject(BaseModel): + """An explicitly selected Workspace person or group, resolved through Directory.""" + + model_config = ConfigDict(extra="forbid") + + type: Literal["person", "group"] + email: str = Field(..., max_length=512, description="The selected person's or group's address.") + + +class KnowledgeSourceCrawlOptions(BaseModel): + """Delegation identity and the complete bounded union of selected people.""" + + model_config = ConfigDict(extra="forbid") + + directory_subject: str = Field( + ..., max_length=512, description="The delegated Directory reader's address." + ) + subjects: list[KnowledgeSourceCrawlSubject] = Field( + ..., + min_length=1, + max_length=MAX_PEOPLE, + description="The people and groups to expand, at most 2000 people in total.", + ) + + +class KnowledgeSourceCrawlProof(BaseModel): + """The read-only delegation proof, without person addresses or credentials.""" + + subject_hashes: list[str] + read_scopes: list[str] + write_denials: list[str] + members: int + + class KnowledgeSourceAddInput(BaseModel): """Register a document source on a namespace: a kind, what it reads (items and links, or a config) and a read-only credential. A namespace admin may. The @@ -212,9 +249,15 @@ class KnowledgeSourceAddInput(BaseModel): description="gdrive with items or links: the file kinds read, of docs, slides, pdf, " "pptx, docx (default all but docx).", ) - credential_mode: Literal["shared", "connected"] = Field( + credential_mode: Literal["shared", "connected", "crawl"] = Field( "shared", - description="Use the supplied shared credential, or your own mirrored connected account.", + description=( + "Use shared credentials, your connected account, or approved Workspace delegation." + ), + ) + crawl_options: KnowledgeSourceCrawlOptions | None = Field( + None, + description="Crawl only: delegation identity and explicit people or groups; default none.", ) credential: str = Field( "", @@ -229,6 +272,8 @@ class KnowledgeSourceAddInput(BaseModel): class KnowledgeSourceAddOutput(BaseModel): + # Crawl only: checked scope and keyed-subject proof; None for another credential mode. + crawl_proof: KnowledgeSourceCrawlProof | None = None # The registered source's name, normalised. name: str # The source kind, normalised. @@ -252,7 +297,9 @@ class KnowledgeSourceUpdateInput(BaseModel): source's file kinds. A namespace admin may. The added items are probed with the stored credential (every one, or the 20 given last) before anything is stored, and a changed `include` probes the file items the source keeps; a probe that fails - stores nothing. Removing every item is refused: remove the source instead.""" + stores nothing. Removing every item is refused: remove the source instead. A + crawl update requires org and namespace administration and reapproves its + exact selectors, items and credential for a fresh generation.""" org: str = Field(..., description="The org.") namespace: str = Field(..., description="The namespace the source's documents live in.") @@ -291,6 +338,18 @@ class KnowledgeSourceUpdateInput(BaseModel): "the same id the source holds." ), ) + crawl_readmit: list[str] = Field( + default_factory=list, + max_length=MAX_PEOPLE, + description=( + "Crawl only: readmit these Directory-verified people after SCIM reactivation; " + "default none." + ), + ) + crawl_options: KnowledgeSourceCrawlOptions | None = Field( + None, + description="Crawl only: replace the explicit delegation selectors; default unchanged.", + ) include: list[str] | None = Field( None, description="gdrive only: the new file kinds read, of docs, slides, pdf, pptx, docx; " @@ -299,6 +358,8 @@ class KnowledgeSourceUpdateInput(BaseModel): class KnowledgeSourceUpdateOutput(BaseModel): + # Crawl only: checked scope and keyed-subject proof; None for another credential mode. + crawl_proof: KnowledgeSourceCrawlProof | None = None # The source updated. name: str # The source's items as now stored. @@ -375,6 +436,8 @@ class KnowledgeSourceInspectOutput(BaseModel): credential_mode: str = "shared" # Namespace-wide or checked mirrored document access. access: str = "namespace" + # Crawl selectors, only for a current org and namespace administrator. + crawl_options: dict[str, Any] = Field(default_factory=dict) # Canonical connected owner, only in the owner's permitted detail projection. owner_subject: str = "" # The things the source reads; [] for a recording source. @@ -398,7 +461,9 @@ class KnowledgeSourceCheckInput(BaseModel): """Probe a registered document source with its stored credential now (a check of every item, or of the 20 added last, and a listing), nothing written. A namespace admin may; a connected owner may also check their own source. An administrator - checking another person's connection receives source-level standing only.""" + checking another person's connection receives source-level standing only. A + crawl check requires org and namespace administration and proves delegated + scope standing without returning item titles or counts.""" org: str = Field(..., description="The org.") namespace: str = Field(..., description="The namespace the source's documents live in.") @@ -432,7 +497,9 @@ class KnowledgeSourceCheckOutput(BaseModel): class KnowledgeSourceRotateInput(BaseModel): """Replace a registered document source's credential. A namespace admin may; - the new credential is probed first, and a probe that fails stores nothing.""" + the new credential is probed first, and a probe that fails stores nothing. A + crawl credential requires current org and namespace administration and a + fresh exact-source delegation approval.""" org: str = Field(..., description="The org.") namespace: str = Field(..., description="The namespace the source's documents live in.") @@ -445,6 +512,8 @@ class KnowledgeSourceRotateInput(BaseModel): class KnowledgeSourceRotateOutput(BaseModel): + # Crawl only: checked scope and keyed-subject proof; None for another credential mode. + crawl_proof: KnowledgeSourceCrawlProof | None = None # The source whose credential was replaced. name: str # What the probe with the new credential found. diff --git a/server/knowledge/app/crawl_authority.py b/server/knowledge/app/crawl_authority.py new file mode 100644 index 0000000000..24ea1284dc --- /dev/null +++ b/server/knowledge/app/crawl_authority.py @@ -0,0 +1,389 @@ +"""Crawl checkouts retain exact registration, provider identity and refusal generations.""" + +from __future__ import annotations + +import asyncio +import hashlib +import json +import math +import secrets +import time +from collections.abc import AsyncIterator, Awaitable, Callable +from contextlib import asynccontextmanager +from contextvars import ContextVar +from dataclasses import dataclass +from typing import TypeVar + +from server.auth import person_erasure, scim, source_registration +from server.clients import gdrive +from server.clients import google_workspace as google +from server.clients._checked_access import CheckedAccess +from server.clients._vendor import CredentialRefused +from server.core.ch import get_client, own_write_reads +from server.core.secrets_at_rest import unseal +from server.knowledge.app import crawl_delegation, crawl_exclusions, crawl_expansion +from server.knowledge.app import sources as source_ops +from server.knowledge.domain.source_crawl import Expansion, Options, Person +from server.knowledge.domain.sources import MIRRORED_ACCESS, KnowledgeSource +from server.knowledge.store import sources as store +from server.settings import settings + +_now = time.monotonic +_T = TypeVar("_T") +_CURRENT: ContextVar[Checkout | None] = ContextVar("crawl_checkout", default=None) + + +@dataclass(frozen=True) +class Bounds: + directory: crawl_expansion.Limits + lease_ttl_s: float + lease_wait_s: float + + def __post_init__(self) -> None: + if ( + not math.isfinite(self.lease_ttl_s) + or self.lease_ttl_s <= 0 + or not math.isfinite(self.lease_wait_s) + or self.lease_wait_s < 0 + ): + raise ValueError("crawl lease bounds refused") + + +def configured_bounds() -> Bounds: + return Bounds( + crawl_expansion.Limits( + settings.knowledge_crawl_directory_max_response_bytes, + settings.knowledge_crawl_directory_max_pages, + settings.knowledge_crawl_max_groups, + ), + settings.knowledge_crawl_lease_ttl_s, + settings.knowledge_crawl_lease_wait_s, + ) + + +@dataclass(frozen=True) +class Checkout: + authority: Authority + people: tuple[Person, ...] + + async def validate(self) -> None: + await self.authority.validate() + for person in self.people: + await self.authority._person(person) + + @asynccontextmanager + async def publication_guard(self) -> AsyncIterator[None]: + # Provider observations happen before local leases; no network runs under them. + await self.validate() + async with self.authority.publication_guard(): + yield + + +def current(source: KnowledgeSource) -> Checkout: + checkout = _CURRENT.get() + if ( + checkout is None + or checkout.authority.source.name != source.name + or scope(checkout.authority.source) != scope(source) + ): + raise CredentialRefused("crawl checkout unavailable") + return checkout + + +def scope(source: KnowledgeSource) -> source_registration.Scope: + payload = json.dumps( + { + "kind": source.kind, + "access": source.access, + "items": source.items, + "config": source.config, + "options": source.crawl_options, + "customer_id": source.crawl_customer_id, + "credential_hash": source.crawl_credential_hash, + }, + sort_keys=True, + separators=(",", ":"), + ) + return source_registration.Scope( + source.org, + source.namespace, + source.source_id, + source.crawl_generation, + hashlib.sha256(payload.encode()).hexdigest(), + ) + + +def _shape(source: KnowledgeSource) -> None: + if ( + source.credential_mode != "crawl" + or source.kind != "gdrive" + or source.access != MIRRORED_ACCESS + or not source.active + or source.pending_delete + or source.owner_subject + or source.connection_id + or source.credential_enc + or not source.secret_enc + or not source.registrant_subject + or not source.approval_enc + or not source.crawl_customer_id + or not source.crawl_generation + or len(source.crawl_credential_hash) != 64 + or any(c not in "0123456789abcdef" for c in source.crawl_credential_hash) + or not settings.knowledge_mirrored_sources_enabled + or not settings.auth_actually_required() + ): + raise CredentialRefused("crawl source unavailable") + Options.parse(source.crawl_options) + + +async def resolve( + source: KnowledgeSource, + *, + deadline: float, + bounds: Bounds, + presenter: source_registration.AdminPermit | None = None, +) -> Authority: + """A worker-only checkout; ordinary live credential selection never calls this path.""" + if type(deadline) not in (float, int) or not math.isfinite(deadline) or deadline <= _now(): + raise TimeoutError("crawl deadline elapsed") + async with asyncio.timeout_at(deadline): + _shape(source) + approval = source_registration.load( + source.approval_enc, scope(source), registrant=source.registrant_subject + ) + await approval.validate() + index = await crawl_exclusions.Index.open(source.org, ch=get_client()) + authority = Authority(source, approval, index, deadline, bounds, presenter=presenter) + await authority.validate() + # Credentials are touched only after the source, registrant and refusal proof. + plain = unseal(source_ops._SECRET_PURPOSE, source.secret_enc, warn=False) + if not plain or not secrets.compare_digest( + hashlib.sha256(plain.encode()).hexdigest(), source.crawl_credential_hash + ): + raise CredentialRefused("crawl source unavailable") + authority._account = gdrive.parse_service_account(plain) + await authority.validate() + return authority + + +async def validate_indexed(source: KnowledgeSource) -> str: + """Indexed reads require live local approval and the completed exclusion epoch. + + No provider request or source credential is needed to check an indexed copy. + """ + _shape(source) + await person_erasure.require_settled(ch=get_client()) + approval = source_registration.load( + source.approval_enc, scope(source), registrant=source.registrant_subject + ) + await approval.validate() + await source_ops._require_source_namespace(source.org, source.namespace) + await source_ops._require_no_erasure(source.org, source.namespace) + index = await crawl_exclusions.Index.open(source.org, ch=get_client()) + return await index.version() + + +class Authority: + __slots__ = ( + "source", + "approval", + "exclusions", + "deadline", + "bounds", + "_account", + "_expansion", + "_preparing", + "_directory_token", + "_drive_tokens", + "_presenter", + ) + + def __init__( + self, + source: KnowledgeSource, + approval: source_registration.Approval, + exclusions: crawl_exclusions.Index, + deadline: float, + bounds: Bounds, + *, + presenter: source_registration.AdminPermit | None = None, + ) -> None: + self.source, self.approval, self.exclusions = source, approval, exclusions + self._presenter = presenter + self.deadline, self.bounds = deadline, bounds + self._account: gdrive.ServiceAccount | None = None + self._expansion: Expansion | None = None + self._preparing = asyncio.Lock() + self._directory_token: google.DelegatedToken | None = None + self._drive_tokens: dict[str, google.DelegatedToken] = {} + + def __repr__(self) -> str: + return "Authority()" + + @asynccontextmanager + async def using(self, people: tuple[Person, ...]) -> AsyncIterator[None]: + if self._expansion is None or any(p not in self._expansion.people for p in people): + raise CredentialRefused("crawl person unavailable") + checkout = Checkout(self, people) + await checkout.validate() + marker = _CURRENT.set(checkout) + try: + yield + await checkout.validate() + finally: + _CURRENT.reset(marker) + + async def validate(self) -> None: + if _now() >= self.deadline: + raise TimeoutError("crawl deadline elapsed") + async with asyncio.timeout_at(self.deadline): + _shape(self.source) + await person_erasure.require_settled(ch=get_client()) + await self.approval.validate() + if self._presenter is not None: + await self._presenter.validate(self.source.org) + await self.exclusions.version() + with own_write_reads(): + current = await store.latest_source( + self.source.org, self.source.namespace, self.source.name + ) + if ( + current is None + or not current.active + or any( + getattr(current, field) != getattr(self.source, field) + for field in ( + "org", + "namespace", + "name", + "source_id", + "kind", + "credential_mode", + "access", + "items", + "config", + "crawl_options", + "registrant_subject", + "approval_enc", + "crawl_generation", + "crawl_customer_id", + "crawl_credential_hash", + "secret_enc", + "owner_subject", + "connection_id", + "credential_enc", + ) + ) + ): + raise CredentialRefused("crawl registration changed") + await source_ops._require_source_namespace(self.source.org, self.source.namespace) + await source_ops._require_no_erasure(self.source.org, self.source.namespace) + await self.exclusions.version() + + def _service_account(self) -> gdrive.ServiceAccount: + if self._account is None: + raise CredentialRefused("crawl source unavailable") + return self._account + + async def prepare(self) -> Expansion: + async with asyncio.timeout_at(self.deadline): + async with self._preparing: + await self.validate() + if self._expansion is None: + result = await crawl_expansion.expand( + Options.parse(self.source.crawl_options), + self._service_account(), + standing=self, + exclusions=self.exclusions, + limits=self.bounds.directory, + deadline=self.deadline, + ) + if result.customer != self.source.crawl_customer_id: + raise CredentialRefused("crawl workspace changed") + await crawl_delegation.require_read_only( + self._service_account(), + subject=Options.parse(self.source.crawl_options).directory_subject, + standing=self, + deadline=self.deadline, + max_bytes=self.bounds.directory.body_bytes, + ) + await self.validate() + self._expansion = result + return self._expansion + + async def _person(self, person: Person) -> None: + await self.validate() + if self._expansion is None or person not in self._expansion.people: + raise CredentialRefused("crawl person unavailable") + if await self.exclusions.contains(person): + raise CredentialRefused("crawl person unavailable") + for address in person.addresses: + if await scim.deactivated(address): + raise CredentialRefused("crawl person unavailable") + if self._directory_token is None or self._directory_token.deadline <= _now(): + self._directory_token = await google.delegated_token( + self._service_account(), + subject=Options.parse(self.source.crawl_options).directory_subject, + scope=gdrive.ReadScope.DIRECTORY_USERS, + deadline=self.deadline, + max_bytes=self.bounds.directory.body_bytes, + ) + await self.validate() + current = await google.user( + self._directory_token, + person.id, + customer_id=person.customer, + max_bytes=self.bounds.directory.body_bytes, + ) + await self.validate() + if ( + current.id != person.id + or current.email != person.email + or frozenset(current.aliases) != frozenset(person.aliases) + ): + raise CredentialRefused("crawl person changed") + + async def drive(self, person: Person) -> CheckedAccess: + """The proven provider person selects sub; no caller, audit or Loom subject does.""" + async with asyncio.timeout_at(self.deadline): + await self._person(person) + token = self._drive_tokens.get(person.id) + if token is None or token.deadline <= _now(): + token = await google.delegated_token( + self._service_account(), + subject=person.email, + scope=gdrive.ReadScope.DRIVE, + deadline=self.deadline, + max_bytes=self.bounds.directory.body_bytes, + ) + await self._person(person) + self._drive_tokens[person.id] = token + + async def validate() -> None: + await self._person(person) + + return CheckedAccess( + token._value, kind="gdrive", deadline=token.deadline, validate=validate + ) + + async def run(self, person: Person, callback: Callable[[CheckedAccess], Awaitable[_T]]) -> _T: + async with asyncio.timeout_at(self.deadline): + access = await self.drive(person) + result = await callback(access) + await access.check() + return result + + @asynccontextmanager + async def publication_guard(self) -> AsyncIterator[None]: + """No content, ACL, membership or freshness write outlives this exact authority.""" + await self.validate() + async with crawl_exclusions.org_lease( + self.source.org, + ttl_s=self.bounds.lease_ttl_s, + wait_s=self.bounds.lease_wait_s, + ): + async with source_ops.sources_lease(self.source.org, self.source.namespace): + await self.validate() + yield + await self.validate() diff --git a/server/knowledge/app/crawl_capture.py b/server/knowledge/app/crawl_capture.py new file mode 100644 index 0000000000..f5f78faf6c --- /dev/null +++ b/server/knowledge/app/crawl_capture.py @@ -0,0 +1,158 @@ +"""Before contact deletion, prove Google identity through the existing exact source authority. + +This path can only capture Directory identity. It cannot read Drive content, serve +results, substitute a request identity or admit a crawl while deletion is pending. +""" + +from __future__ import annotations + +import asyncio +import hashlib +import secrets +import time +from dataclasses import dataclass + +from server.auth import person_erasure, source_registration +from server.clients import gdrive +from server.clients import google_workspace as google +from server.clients._vendor import CredentialRefused +from server.core.ch import get_client, set_query_tag +from server.core.secrets_at_rest import unseal +from server.knowledge.app import crawl_authority +from server.knowledge.app import sources as source_ops +from server.knowledge.domain.source_crawl import Options +from server.knowledge.domain.sources import KnowledgeSource +from server.knowledge.store import sources as store +from server.settings import settings + +_now = time.monotonic + + +@dataclass(frozen=True, repr=False) +class _Standing: + pending: person_erasure.Pending + source: KnowledgeSource + approval: source_registration.Approval + deadline: float + + async def validate(self) -> None: + if _now() >= self.deadline: + raise TimeoutError("crawl identity capture deadline") + await self.pending.validate(self.source.org, before_delete=True) + crawl_authority._shape(self.source) + await self.approval.validate() + await source_ops._require_source_namespace(self.source.org, self.source.namespace) + await source_ops._require_no_erasure(self.source.org, self.source.namespace) + current = await store.latest_source( + self.source.org, self.source.namespace, self.source.name, ch=get_client() + ) + if ( + current is None + or not current.active + or crawl_authority.scope(current) != crawl_authority.scope(self.source) + or current.approval_enc != self.source.approval_enc + or current.secret_enc != self.source.secret_enc + or current.registrant_subject != self.source.registrant_subject + or current.owner_subject != self.source.owner_subject + or current.connection_id != self.source.connection_id + or current.credential_enc != self.source.credential_enc + ): + raise CredentialRefused("crawl identity authority changed") + crawl_authority._shape(current) + await self.pending.validate(self.source.org, before_delete=True) + + +async def _token( + pending: person_erasure.Pending, source: KnowledgeSource, deadline: float +) -> tuple[_Standing, google.DelegatedToken]: + approval = source_registration.load( + source.approval_enc, crawl_authority.scope(source), registrant=source.registrant_subject + ) + standing = _Standing(pending, source, approval, deadline) + await standing.validate() + plain = unseal(source_ops._SECRET_PURPOSE, source.secret_enc, warn=False) + if not plain or not secrets.compare_digest( + hashlib.sha256(plain.encode()).hexdigest(), source.crawl_credential_hash + ): + raise CredentialRefused("crawl identity credential unavailable") + account = gdrive.parse_service_account(plain) + await standing.validate() + options = Options.parse(source.crawl_options) + token = await google.delegated_token( + account, + subject=options.directory_subject, + scope=gdrive.ReadScope.DIRECTORY_USERS, + deadline=deadline, + max_bytes=settings.knowledge_crawl_directory_max_response_bytes, + ) + await standing.validate() + principal = await google.directory_identity( + token, max_bytes=settings.knowledge_crawl_directory_max_response_bytes + ) + await standing.validate() + if principal.customer_id != source.crawl_customer_id: + raise CredentialRefused("crawl identity workspace changed") + return standing, token + + +async def _for_workspace( + pending: person_erasure.Pending, + person: person_erasure.Person, + source: KnowledgeSource, + deadline: float, +) -> tuple[person_erasure.Workspace, ...]: + standing, token = await _token(pending, source, deadline) + found: dict[str, person_erasure.Workspace] = {} + for address in person.emails: + await standing.validate() + user = await google.user( + token, + address, + customer_id=source.crawl_customer_id, + max_bytes=settings.knowledge_crawl_directory_max_response_bytes, + ) + await standing.validate() + observed = person_erasure.Workspace( + person.org, + person.subject, + user.customer_id, + user.id, + tuple(sorted({user.email.lower(), *(a.lower() for a in user.aliases)})), + ) + if address not in observed.emails or user.id in found and found[user.id] != observed: + raise CredentialRefused("crawl identity changed during capture") + found[user.id] = observed + return tuple(found[key] for key in sorted(found)) + + +async def _current_sources(org: str) -> list[KnowledgeSource]: + history = await store.crawl_history( + org, maximum=settings.knowledge_crawl_registry_scan_limit, ch=get_client() + ) + customers = {source.crawl_customer_id for source in history} + if "" in customers: + raise CredentialRefused("crawl workspace history unavailable") + current: dict[str, KnowledgeSource] = {} + for namespace, name in sorted({(s.namespace, s.name) for s in history}): + source = await store.latest_source(org, namespace, name, ch=get_client()) + if source is not None and source.active and source.credential_mode == "crawl": + current.setdefault(source.crawl_customer_id, source) + if set(current) != customers: + # A lost/expired authority cannot be replaced by guessing a current user's id. + raise CredentialRefused("historical crawl identity requires verified repair") + return [current[key] for key in sorted(current)] + + +async def prepare(pending: person_erasure.Pending) -> tuple[person_erasure.Workspace, ...]: + """Capture only existing crawl workspaces; absence never invents provider configuration.""" + set_query_tag("knowledge.crawl.person_capture") + deadline = _now() + settings.knowledge_crawl_identity_capture_timeout_s + result: list[person_erasure.Workspace] = [] + async with asyncio.timeout_at(deadline): + for person in pending.captured.people: + await pending.validate(person.org, before_delete=True) + for source in await _current_sources(person.org): + await pending.validate(person.org, before_delete=True) + result.extend(await _for_workspace(pending, person, source, deadline)) + await pending.validate(before_delete=True) + return tuple(sorted(result, key=lambda w: (w.org, w.subject, w.customer, w.user))) diff --git a/server/knowledge/app/crawl_delegation.py b/server/knowledge/app/crawl_delegation.py new file mode 100644 index 0000000000..e83fafe984 --- /dev/null +++ b/server/knowledge/app/crawl_delegation.py @@ -0,0 +1,34 @@ +"""An admitted crawl proves all fixed read grants and each distinct write denial.""" + +from __future__ import annotations + +import asyncio + +from server.clients import gdrive +from server.clients import google_workspace as google +from server.core.ch import set_query_tag +from server.knowledge.app.crawl_expansion import Standing + + +async def require_read_only( + account: gdrive.ServiceAccount, + *, + subject: str, + standing: Standing, + deadline: float, + max_bytes: int, +) -> None: + set_query_tag("knowledge.crawl.delegation") + async with asyncio.timeout_at(deadline): + for read in gdrive.ReadScope: + await standing.validate() + await google.delegated_token( + account, subject=subject, scope=read, deadline=deadline, max_bytes=max_bytes + ) + await standing.validate() + for write in gdrive.WriteScope: + await standing.validate() + await google.require_write_denied( + account, subject=subject, scope=write, deadline=deadline, max_bytes=max_bytes + ) + await standing.validate() diff --git a/server/knowledge/app/crawl_exclusions.py b/server/knowledge/app/crawl_exclusions.py new file mode 100644 index 0000000000..18ea789fea --- /dev/null +++ b/server/knowledge/app/crawl_exclusions.py @@ -0,0 +1,303 @@ +"""Persistent refusal fences outlive rescans and root-secret rotation.""" + +from __future__ import annotations + +import hashlib +import hmac +import json +import math +import secrets +import uuid +from collections.abc import AsyncIterator, Awaitable, Callable +from contextlib import asynccontextmanager +from typing import Any, Protocol + +from server.auth import claims +from server.clients._vendor import VendorError +from server.core import leases +from server.core.ch import get_client, set_query_tag +from server.core.secrets_at_rest import seal, unseal +from server.knowledge.domain.source_crawl import Person, email +from server.knowledge.store import crawl_exclusions as store + + +class OrgAuthority(Protocol): + async def validate(self, org: str) -> None: ... + + +KEY_PURPOSE = "knowledge_crawl.exclusion_key" +# KeeperMap expiry is UInt32 seconds; this is its fixed representable upper bound. +_EXPIRY = 2**32 - 1 + + +def _fence(org: str) -> str: + if not isinstance(org, str) or not org or "\0" in org: + raise ValueError("crawl org refused") + return "crawl-exclusions:" + hashlib.sha256(org.encode()).hexdigest() + + +@asynccontextmanager +async def org_lease( + org: str, *, ttl_s: float, wait_s: float, ch: Any = None +) -> AsyncIterator[None]: + if not all(type(v) in (int, float) and math.isfinite(v) for v in (ttl_s, wait_s)): + raise ValueError("finite crawl lease bounds required") + if ttl_s <= 0 or wait_s < 0: + raise ValueError("crawl lease bounds refused") + async with leases.cluster_lease(_fence(org), ttl_s=ttl_s, wait_s=wait_s, ch=ch or get_client()): + yield + + +async def _move(ch: Any, org: str, old: str | None, new: str) -> None: + if await claims.swap(ch, _fence(org), old, new, _EXPIRY) != new: + raise VendorError("crawl exclusions changed") + + +async def initialize( + org: str, *, authority: OrgAuthority, ttl_s: float, wait_s: float, ch: Any = None +) -> Index: + """Registration initializes once under its checked org authority; reads never initialize.""" + set_query_tag("knowledge.crawl.exclusions.initialize") + ch = ch or get_client() + await authority.validate(org) + async with org_lease(org, ttl_s=ttl_s, wait_s=wait_s, ch=ch): + await authority.validate(org) + row = await store.key(org, ch=ch) + current = await claims.get(ch, _fence(org)) + if row is None: + if current is not None or await store.any_refusal(org, ch=ch): + raise VendorError("crawl exclusion key unavailable") + key = secrets.token_hex(32) + key_id = hashlib.sha256(bytes.fromhex(key)).hexdigest() + blocked = f"initializing:{key_id}:{uuid.uuid4().hex}" + await _move(ch, org, current, blocked) + key_enc = seal(KEY_PURPOSE, key) + await store.key_put(org, key_id, key_enc, generation=blocked, ch=ch) + else: + key_id = str(row["key_id"]) + if ( + isinstance(current, str) + and current.startswith("initializing:" + key_id + ":") + and not row["pending_id"] + and not await store.any_refusal(org, ch=ch) + ): + _open_key(row) + active = f"active:{key_id}:{uuid.uuid4().hex}" + await _move(ch, org, current, active) + await store.key_put(org, key_id, str(row["key_enc"]), generation=active, ch=ch) + return await Index.open(org, ch=ch) + if ( + isinstance(current, str) + and current.startswith("active:" + key_id + ":") + and str(row["generation"]).startswith("initializing:" + key_id + ":") + and not row["pending_id"] + and not await store.any_refusal(org, ch=ch) + ): + _open_key(row) + await store.key_put(org, key_id, str(row["key_enc"]), generation=current, ch=ch) + return await Index.open(org, ch=ch) + # A blocked existing registry requires its actual failed cleanup to retry. + if current is None or not current.startswith("active:" + key_id + ":"): + raise VendorError("crawl exclusion repair required") + return await Index.open(org, ch=ch) + active = f"active:{key_id}:{uuid.uuid4().hex}" + await _move(ch, org, blocked, active) + await store.key_put(org, key_id, key_enc, generation=active, ch=ch) + return await Index.open(org, ch=ch) + + +def _open_key(row: dict[str, Any]) -> bytes: + plain = unseal(KEY_PURPOSE, str(row["key_enc"]), warn=False) + if len(plain) != 64 or any(c not in "0123456789abcdef" for c in plain): + raise VendorError("crawl exclusion key unavailable") + key = bytes.fromhex(plain) + if not secrets.compare_digest(hashlib.sha256(key).hexdigest(), str(row["key_id"])): + raise VendorError("crawl exclusion key unavailable") + return key + + +async def _finish( + org: str, + row: dict[str, Any], + *, + authority: OrgAuthority, + cleanup: Callable[[], Awaitable[None]], + ch: Any, +) -> None: + key_id, pending = str(row["key_id"]), str(row["pending_id"]) + digests = row["pending_digests"] + if ( + len(pending) != 32 + or any(c not in "0123456789abcdef" for c in pending) + or not isinstance(digests, list) + or not digests + or any( + not isinstance(d, str) or len(d) != 64 or any(c not in "0123456789abcdef" for c in d) + for d in digests + ) + ): + raise VendorError("crawl exclusion repair unavailable") + current = await claims.get(ch, _fence(org)) + blocked, active = f"blocked:{key_id}:{pending}", f"active:{key_id}:{pending}" + # A crash before the initial CAS left the previous active fence in place. + if current not in {row["generation"], blocked, active}: + raise VendorError("crawl exclusion repair unavailable") + if type(row["pending_exclude"]) not in (bool, int) or row["pending_exclude"] not in (0, 1): + raise VendorError("crawl exclusion repair unavailable") + if current != blocked: + await _move(ch, org, current, blocked) + await store.put(org, digests, exclude=bool(row["pending_exclude"]), ch=ch) + await cleanup() + await authority.validate(org) + from server.erasure.app.crawl_person import require_refusals + + await require_refusals(org, digests, excluded=bool(row["pending_exclude"]), ch=ch) + await _move(ch, org, blocked, active) + # Until this full row lands, new readers still refuse the unfinished manifest. + await store.key_put(org, key_id, str(row["key_enc"]), generation=active, ch=ch) + + +async def repair( + org: str, + *, + ttl_s: float, + wait_s: float, + authority: OrgAuthority, + cleanup: Callable[[], Awaitable[None]], + expected_digests: tuple[str, ...] | None = None, + ch: Any = None, +) -> Index: + """Finish the persisted mutation after a crash; never guess a readmission.""" + set_query_tag("knowledge.crawl.exclusions.repair") + ch = ch or get_client() + await authority.validate(org) + async with org_lease(org, ttl_s=ttl_s, wait_s=wait_s, ch=ch): + await authority.validate(org) + row = await store.key(org, ch=ch) + if row is None: + raise VendorError("crawl exclusion repair unavailable") + _open_key(row) + if expected_digests is not None and row["pending_digests"] != list(expected_digests): + raise VendorError("another crawl exclusion cleanup is pending") + await _finish(org, row, authority=authority, cleanup=cleanup, ch=ch) + return await Index.open(org, ch=ch) + + +class Index: + """One private immutable refusal observation, never a source authority or credential.""" + + __slots__ = ("org", "_key", "_key_id", "_version", "_ch") + + def __init__(self, org: str, key: bytes, key_id: str, version: str, ch: Any) -> None: + self.org, self._key, self._key_id, self._version, self._ch = org, key, key_id, version, ch + + def __repr__(self) -> str: + return "Index()" + + @classmethod + async def open(cls, org: str, *, ch: Any = None) -> Index: + ch = ch or get_client() + row = await store.key(org, ch=ch) + observed = await claims.get(ch, _fence(org)) + if row is None or not isinstance(observed, str): + raise VendorError("crawl exclusions unavailable") + key_id = str(row["key_id"]) + if ( + not key_id + or row["pending_id"] + or row["generation"] != observed + or not observed.startswith("active:" + key_id + ":") + ): + raise VendorError("crawl exclusion cleanup incomplete") + return cls(org, _open_key(row), key_id, observed, ch) + + async def version(self) -> str: + row = await store.key(self.org, ch=self._ch) + if ( + await claims.get(self._ch, _fence(self.org)) != self._version + or row is None + or row["pending_id"] + or row["key_id"] != self._key_id + or row["generation"] != self._version + ): + raise VendorError("crawl exclusions changed") + return self._version + + def _digest(self, purpose: str, values: list[str]) -> str: + payload = json.dumps([purpose, self.org, *values], separators=(",", ":")).encode() + return hmac.digest(self._key, payload, "sha256").hex() + + def person_digests(self, person: Person) -> list[str]: + if not person.customer or not person.id: + raise ValueError("provider identity required") + return [ + self._digest("google-user", [person.customer, person.id]), + *( + self._digest("verified-email", [email(address)]) + for address in sorted(person.addresses) + ), + ] + + def contact_digests(self, contacts: tuple[str, ...]) -> list[str]: + return sorted({self._digest("verified-email", [email(v)]) for v in contacts}) + + async def contains(self, person: Person) -> bool: + await self.version() + found = await store.excluded(self.org, self.person_digests(person), ch=self._ch) + await self.version() + return bool(found) + + async def excluded_addresses(self, addresses: tuple[str, ...]) -> frozenset[str]: + await self.version() + digests = { + email(address): self._digest("verified-email", [email(address)]) + for address in addresses + } + excluded = await store.excluded(self.org, list(digests.values()), ch=self._ch) + await self.version() + return frozenset(address for address, digest in digests.items() if digest in excluded) + + async def change( + self, + people: tuple[Person, ...], + *, + exclude: bool, + ttl_s: float, + wait_s: float, + authority: OrgAuthority, + cleanup: Callable[[], Awaitable[None]], + contacts: tuple[str, ...] = (), + ) -> Index: + """A checked mutation stays fenced until every local rewrite and census succeeds.""" + set_query_tag("knowledge.crawl.exclusions.change") + if type(exclude) is not bool or not people and not contacts: + raise ValueError("explicit provider-person exclusion or readmission required") + await authority.validate(self.org) + async with org_lease(self.org, ttl_s=ttl_s, wait_s=wait_s, ch=self._ch): + await authority.validate(self.org) + await self.version() + digests = sorted( + { + *(digest for person in people for digest in self.person_digests(person)), + *self.contact_digests(contacts), + } + ) + row = await store.key(self.org, ch=self._ch) + if row is None: + raise VendorError("crawl exclusion key unavailable") + pending = uuid.uuid4().hex + await store.key_put( + self.org, + self._key_id, + str(row["key_enc"]), + generation=self._version, + pending_id=pending, + pending_digests=tuple(digests), + pending_exclude=exclude, + ch=self._ch, + ) + row = await store.key(self.org, ch=self._ch) + if row is None or row["pending_id"] != pending: + raise VendorError("crawl exclusion repair unavailable") + await _finish(self.org, row, authority=authority, cleanup=cleanup, ch=self._ch) + return await Index.open(self.org, ch=self._ch) diff --git a/server/knowledge/app/crawl_expansion.py b/server/knowledge/app/crawl_expansion.py new file mode 100644 index 0000000000..37736aa8c2 --- /dev/null +++ b/server/knowledge/app/crawl_expansion.py @@ -0,0 +1,195 @@ +"""Complete bounded Workspace membership; no partial expansion becomes reader evidence.""" + +from __future__ import annotations + +import asyncio +from dataclasses import dataclass +from typing import Protocol + +from server.auth import scim +from server.clients import gdrive +from server.clients import google_workspace as google +from server.clients._vendor import CredentialRefused, TooLarge, VendorError +from server.core.ch import set_query_tag +from server.knowledge.domain.source_crawl import MAX_PEOPLE, Expansion, Options, Person + + +class Standing(Protocol): + async def validate(self) -> None: ... + + +class Exclusions(Protocol): + async def version(self) -> str: ... + + async def contains(self, person: Person) -> bool: ... + + +@dataclass(frozen=True) +class Limits: + body_bytes: int + member_pages: int + groups: int + + def __post_init__(self) -> None: + if any( + type(v) is not int or v <= 0 for v in (self.body_bytes, self.member_pages, self.groups) + ): + raise ValueError("positive crawl bounds required") + + +async def expand( + options: Options, + account: gdrive.ServiceAccount, + *, + standing: Standing, + exclusions: Exclusions, + limits: Limits, + deadline: float, +) -> Expansion: + """Resolve all selected people/groups under one source and exclusion observation.""" + set_query_tag("knowledge.crawl.expand") + async with asyncio.timeout_at(deadline): + observation = _Expansion(options, account, standing, exclusions, limits, deadline) + return await observation.run() + + +class _Expansion: + def __init__( + self, + options: Options, + account: gdrive.ServiceAccount, + standing: Standing, + exclusions: Exclusions, + limits: Limits, + deadline: float, + ) -> None: + self.options, self.account = options, account + self.standing, self.exclusions, self.limits = standing, exclusions, limits + self.deadline = deadline + self.version = "" + self.customer = "" + self.tokens: dict[gdrive.ReadScope, google.DelegatedToken] = {} + self.people: dict[str, Person] = {} + self.identities: dict[str, Person | None] = {} + self.groups: dict[str, tuple[str, ...]] = {} + self.group_names: dict[str, str] = {} + self.visiting: set[str] = set() + self.group_count = 0 + self.page_count = 0 + + async def guard(self) -> None: + await self.standing.validate() + if await self.exclusions.version() != self.version: + raise VendorError("crawl exclusions changed") + + async def token(self, scope: gdrive.ReadScope) -> google.DelegatedToken: + if scope not in self.tokens: + await self.guard() + token = await google.delegated_token( + self.account, + subject=self.options.directory_subject, + scope=scope, + deadline=self.deadline, + max_bytes=self.limits.body_bytes, + ) + await self.guard() + self.tokens[scope] = token + return self.tokens[scope] + + async def run(self) -> Expansion: + await self.standing.validate() + self.version = await self.exclusions.version() + if not self.version: + raise VendorError("crawl exclusions unavailable") + token = await self.token(gdrive.ReadScope.DIRECTORY_USERS) + await self.guard() + principal = await google.directory_identity(token, max_bytes=self.limits.body_bytes) + await self.guard() + self.customer = principal.customer_id + for subject in self.options.subjects: + if subject.kind == "person": + await self.person(subject.email) + else: + await self.group(subject.email, subject.email) + await self.guard() + return Expansion( + self.customer, + tuple(self.people[k] for k in sorted(self.people)), + tuple(sorted(self.groups.items())), + self.version, + ) + + async def person(self, key: str) -> Person | None: + if key in self.identities: + return self.identities[key] + token = await self.token(gdrive.ReadScope.DIRECTORY_USERS) + await self.guard() + found = await google.user( + token, key, customer_id=self.customer, max_bytes=self.limits.body_bytes + ) + await self.guard() + person = Person( + found.customer_id, found.id, found.email.lower(), tuple(sorted(set(found.aliases))) + ) + old = self.people.get(person.id) + if old is not None and old != person: + raise VendorError("crawl person changed") + denied = await self.exclusions.contains(person) + for address in person.addresses: + denied = await scim.deactivated(address) or denied + await self.guard() + admitted = None if denied else person + for spelling in (person.id, *person.addresses): + if spelling in self.identities and self.identities[spelling] != admitted: + raise VendorError("crawl identity is ambiguous") + self.identities[spelling] = admitted + if admitted is not None: + self.people[person.id] = person + if len(self.people) > MAX_PEOPLE: + raise TooLarge("crawl exceeds its people bound") + return admitted + + async def before_page(self) -> None: + await self.guard() + if self.page_count >= self.limits.member_pages: + raise TooLarge("crawl exceeds its directory page bound") + self.page_count += 1 + + async def group(self, key: str, name: str) -> tuple[str, ...]: + canonical = self.group_names.get(name, key) + if canonical in self.visiting: + raise VendorError("crawl group cycle") + if name in self.groups: + return self.groups[name] + self.group_count += 1 + if self.group_count > self.limits.groups: + raise TooLarge("crawl exceeds its group bound") + self.visiting.add(canonical) + self.group_names[name] = canonical + token = await self.token(gdrive.ReadScope.DIRECTORY_MEMBERS) + await self.guard() + members = await google.group_members( + token, + key, + max_bytes=self.limits.body_bytes, + max_pages=self.limits.member_pages, + before_page=self.before_page, + ) + await self.guard() + found: set[str] = set() + for member in members: + if member.kind == "CUSTOMER": + raise CredentialRefused("customer-wide crawl membership refused") + if member.kind == "GROUP": + if member.email is None: + raise VendorError("crawl group address unavailable") + nested = await self.group(member.id, member.email) + found.update(nested) + else: + person = await self.person(member.id) + if person is not None: + found.add(person.email) + await self.guard() + self.visiting.remove(canonical) + self.groups[name] = tuple(sorted(found)) + return self.groups[name] diff --git a/server/knowledge/app/crawl_person_erasure.py b/server/knowledge/app/crawl_person_erasure.py new file mode 100644 index 0000000000..dff067c269 --- /dev/null +++ b/server/knowledge/app/crawl_person_erasure.py @@ -0,0 +1,132 @@ +"""A durable canonical-person deletion fences every affected crawl until raw cleanup ends.""" + +from __future__ import annotations + +import asyncio +import functools +from typing import Any + +from server.auth import person_erasure +from server.clients._vendor import CredentialRefused +from server.core.ch import get_client, set_query_tag +from server.erasure.app import crawl_person +from server.knowledge.app import crawl_exclusions as exclusions +from server.knowledge.app import sources as source_ops +from server.knowledge.domain.source_crawl import Person +from server.knowledge.store import crawl_exclusions as exclusion_store +from server.knowledge.store import sources as source_store +from server.settings import settings + + +async def _inventory(org: str, allowed_customers: set[str], ch: Any) -> dict[str, str]: + history = await source_store.crawl_history( + org, maximum=settings.knowledge_crawl_registry_scan_limit, ch=ch + ) + found: dict[str, str] = {} + for source in history: + if ( + source.org != org + or not source.namespace + or not source.source_id + or source.crawl_customer_id not in allowed_customers + or source.source_id in found + and found[source.source_id] != source.namespace + ): + raise CredentialRefused("crawl person history requires verified repair") + found[source.source_id] = source.namespace + return found + + +async def _clean_source( + pending: person_erasure.Pending, + person: person_erasure.Person, + namespace: str, + source_id: str, + contacts: frozenset[str], + ch: Any, +) -> None: + await pending.validate(person.org) + async with source_ops.sources_lease(person.org, namespace): + await pending.validate(person.org) + await crawl_person.erase_source(pending, person, namespace, source_id, contacts, ch=ch) + await pending.validate(person.org) + + +async def _cleanup( + pending: person_erasure.Pending, + person: person_erasure.Person, + contacts: frozenset[str], + customers: set[str], + ch: Any, +) -> None: + before = await _inventory(person.org, customers, ch) + for source_id, namespace in sorted(before.items()): + await _clean_source(pending, person, namespace, source_id, contacts, ch) + await pending.validate(person.org) + if await _inventory(person.org, customers, ch) != before: + raise CredentialRefused("crawl person registrations changed during cleanup") + + +def _people( + captured: person_erasure.Capture, + person: person_erasure.Person, +) -> tuple[Person, ...]: + return tuple( + Person(w.customer, w.user, w.emails[0], w.emails[1:]) + for w in captured.workspaces + if (w.org, w.subject) == (person.org, person.subject) + ) + + +async def _exclude(pending: person_erasure.Pending, person: person_erasure.Person, ch: Any) -> None: + people = _people(pending.captured, person) + contacts = frozenset({*person.emails, *(e for p in people for e in p.addresses)}) + cleanup = functools.partial( + _cleanup, pending, person, contacts, {p.customer for p in people}, ch + ) + kwargs = { + "ttl_s": settings.knowledge_crawl_lease_ttl_s, + "wait_s": settings.knowledge_crawl_lease_wait_s, + "authority": pending, + "ch": ch, + } + await pending.validate(person.org) + row = await exclusion_store.key(person.org, ch=ch) + if row is not None and row["pending_id"]: + # This object calculates HMACs only; open/version still refuse a blocked row. + proof = exclusions.Index( + person.org, exclusions._open_key(row), row["key_id"], row["generation"], ch + ) + expected = tuple( + sorted( + { + *(d for p in people for d in proof.person_digests(p)), + *proof.contact_digests(tuple(contacts)), + } + ) + ) + if row["pending_exclude"] != 1: + raise CredentialRefused("another crawl exclusion cleanup is pending") + await exclusions.repair(person.org, cleanup=cleanup, expected_digests=expected, **kwargs) + else: + index = await exclusions.initialize(person.org, **kwargs) + await index.change( + people, + contacts=tuple(sorted(contacts)), + exclude=True, + ttl_s=settings.knowledge_crawl_lease_ttl_s, + wait_s=settings.knowledge_crawl_lease_wait_s, + authority=pending, + cleanup=cleanup, + ) + + +async def finish(pending: person_erasure.Pending) -> None: + """Only the persisted, prepared deletion capsule can drive these exact-person writes.""" + set_query_tag("knowledge.crawl.person_erasure") + if not pending.captured.prepared: + raise person_erasure.unavailable() + async with asyncio.timeout(settings.knowledge_crawl_person_cleanup_timeout_s): + for person in pending.captured.people: + await _exclude(pending, person, get_client()) + await pending.validate() diff --git a/server/knowledge/app/crawl_reseal.py b/server/knowledge/app/crawl_reseal.py new file mode 100644 index 0000000000..99dddb07cd --- /dev/null +++ b/server/knowledge/app/crawl_reseal.py @@ -0,0 +1,125 @@ +"""Root rotation carries refusal keys and source approvals without changing authority.""" + +from __future__ import annotations + +from typing import Any + +from server.auth.source_registration import PURPOSE, names_person +from server.clients._vendor import VendorError +from server.core.ch import get_client, set_query_tag +from server.core.secrets_at_rest import resealed, unseal +from server.knowledge.app import crawl_exclusions as exclusions +from server.knowledge.app import sources as source_ops +from server.knowledge.store import crawl_exclusions as store +from server.knowledge.store import sources as source_store +from server.settings import settings + + +async def keys(*, dry_run: bool, ch: Any) -> tuple[int, list[str]]: + if ch is not get_client(): + raise RuntimeError("crawl_reseal_target_mismatch") + set_query_tag("knowledge.crawl.reseal.keys") + count, scanned, after, lost = 0, 0, "", [] + while page := await store.page(after, settings.knowledge_crawl_registry_page_size, ch=ch): + scanned += len(page) + if scanned > settings.knowledge_crawl_registry_scan_limit: + raise RuntimeError("crawl_reseal_scan_bound") + for org in page: + async with exclusions.org_lease( + org, + ttl_s=settings.knowledge_crawl_lease_ttl_s, + wait_s=settings.knowledge_crawl_lease_wait_s, + ch=ch, + ): + row = await store.key(org, ch=ch) + if row is not None: + changed = await _key(org, row, dry_run=dry_run, ch=ch) + if changed is None: + lost.append(org) + else: + count += changed + after = page[-1] + return count, lost + + +async def _key(org: str, row: dict[str, Any], *, dry_run: bool, ch: Any) -> int | None: + # A lost key never gets regenerated: doing so would discard every old refusal. + try: + exclusions._open_key(row) + except (ValueError, TypeError, VendorError): + return None + new = resealed(exclusions.KEY_PURPOSE, str(row["key_enc"])) + if new is None: + return 0 + if not dry_run: + await store.key_put( + org, + str(row["key_id"]), + new, + generation=str(row["generation"]), + pending_id=str(row["pending_id"]), + pending_digests=tuple(row["pending_digests"]), + pending_exclude=bool(row["pending_exclude"]), + ch=ch, + ) + return 1 + + +async def approvals(*, dry_run: bool, ch: Any) -> tuple[int, list[str]]: + if ch is not get_client(): + raise RuntimeError("crawl_reseal_target_mismatch") + set_query_tag("knowledge.crawl.reseal.approvals") + count, scanned, after, lost = 0, 0, ("", "", ""), [] + while page := await source_store.crawl_approval_page( + after, maximum=settings.knowledge_crawl_registry_page_size, ch=ch + ): + scanned += len(page) + if scanned > settings.knowledge_crawl_registry_scan_limit: + raise RuntimeError("crawl_reseal_scan_bound") + for org, ns, name in page: + n, unreadable = await _approvals(org, ns, name, dry_run=dry_run, ch=ch) + count += n + if unreadable: + lost.append("/".join((org, ns, name))) + after = page[-1] + return count, lost + + +async def _approvals( + org: str, + namespace: str, + name: str, + *, + dry_run: bool, + ch: Any, +) -> tuple[int, bool]: + count = 0 + async with source_ops.sources_lease(org, namespace): + rows = await source_store.crawl_approval_values( + org, + namespace, + name, + maximum=settings.knowledge_crawl_registry_scan_limit, + ch=ch, + ) + for row in rows: + encoded = str(row["approval_enc"]) + if not unseal(PURPOSE, encoded, warn=False): + return count, True + # Validate source-bound historical metadata without requiring a live grant. + names_person( + encoded, org=org, namespace=namespace, source_id=row["source_id"], spellings=() + ) + new = resealed(PURPOSE, encoded) + if new is not None: + count += 1 + if not dry_run: + await source_store.reseal_crawl_approval( + org, + namespace, + row["source_id"], + encoded, + new, + ch=ch, + ) + return count, False diff --git a/server/knowledge/app/crawl_restore.py b/server/knowledge/app/crawl_restore.py new file mode 100644 index 0000000000..5d548e96b0 --- /dev/null +++ b/server/knowledge/app/crawl_restore.py @@ -0,0 +1,76 @@ +"""Offline restore invalidates approvals while preserving durable refusal identities.""" + +from __future__ import annotations + +import asyncio +import dataclasses +from typing import Any + +from server.core.ch import get_client, set_query_tag +from server.erasure.app import crawl_person +from server.knowledge.app import crawl_exclusions +from server.knowledge.app import sources as source_ops +from server.knowledge.app.connected_restore import offline_refusal +from server.knowledge.domain.sources import KnowledgeSource +from server.knowledge.store import crawl_exclusions as exclusions_store +from server.knowledge.store import sources as source_store +from server.settings import settings + + +async def _retire(source: KnowledgeSource, *, ch: Any) -> None: + async with source_ops.sources_lease(source.org, source.namespace): + pending = await source_ops._queue_retirement(source) + current = await source_store.latest_source(source.org, source.namespace, source.name, ch=ch) + if ( + current is not None + and current.credential_mode == "crawl" + and current.source_id == source.source_id + ): + await source_store.insert_source( + dataclasses.replace( + current, + active=False, + approval_enc="", + crawl_generation="", + pending_delete=pending, + ), + ch=ch, + ) + + +async def restore_all(*, apply: bool, ch: Any) -> None: + """No restored crawl approval survives; unverified historical refusal state blocks serving. + + This cannot recover missing post-backup person identity proofs. A mismatched + refusal registry remains closed for verified repair, never reset to empty. + """ + if ch is not get_client(): + raise RuntimeError("crawl_restore_target_mismatch") + set_query_tag("knowledge.crawl.restore") + if why := await offline_refusal(ch): + raise RuntimeError(why) + maximum = settings.knowledge_crawl_registry_scan_limit + orgs = await exclusions_store.restore_orgs(maximum=maximum, ch=ch) + async with asyncio.timeout(settings.knowledge_crawl_person_cleanup_timeout_s): + for org in orgs: + async with crawl_exclusions.org_lease( + org, + ttl_s=settings.knowledge_crawl_lease_ttl_s, + wait_s=settings.knowledge_crawl_lease_wait_s, + ch=ch, + ): + # Reads never initialize or repair a stale refusal key or pending manifest. + index = await crawl_exclusions.Index.open(org, ch=ch) + history = await source_store.crawl_history(org, maximum=maximum, ch=ch) + if not apply: + continue + if why := await offline_refusal(ch): + raise RuntimeError(why) + seen = set() + for source in history: + identity = source.namespace, source.source_id + if identity not in seen: + await _retire(source, ch=ch) + seen.add(identity) + await crawl_person.invalidate_restored_approvals(org, ch=ch) + await index.version() diff --git a/server/knowledge/app/crawl_sources.py b/server/knowledge/app/crawl_sources.py new file mode 100644 index 0000000000..9224e7aeba --- /dev/null +++ b/server/knowledge/app/crawl_sources.py @@ -0,0 +1,406 @@ +"""Crawl registration binds a current administrator to an exact delegated source.""" + +from __future__ import annotations + +import asyncio +import dataclasses +import hashlib +import hmac +import json +import time +import uuid +from collections.abc import AsyncIterator +from contextlib import asynccontextmanager +from typing import Any, cast + +from server.auth import machines, person_erasure, scim, source_registration +from server.auth.context import current_auth +from server.clients import gdrive +from server.clients import google_workspace as google +from server.clients._vendor import VendorError +from server.core import request_work +from server.core.ch import get_client, own_write_reads, set_query_tag +from server.core.diagnostics import private_work +from server.core.errors import OverloadedError, ValidationError +from server.core.secrets_at_rest import unseal +from server.knowledge.api.models_sources import ( + KnowledgeSourceAddInput, + KnowledgeSourceAddOutput, + KnowledgeSourceCheckOutput, + KnowledgeSourceCrawlProof, + KnowledgeSourceProbe, + KnowledgeSourceRotateOutput, + KnowledgeSourceUpdateInput, + KnowledgeSourceUpdateOutput, +) +from server.knowledge.app import ( + crawl_authority, + crawl_delegation, + crawl_exclusions, + crawl_expansion, +) +from server.knowledge.app import sources as ops +from server.knowledge.domain import sources as domain +from server.knowledge.domain.source_crawl import Options, Person, email +from server.knowledge.domain.sources import KnowledgeSource +from server.knowledge.store import sources as store +from server.settings import settings + +_CRAWL_MIGRATION = 114 + + +def unavailable() -> ValidationError: + return ValidationError("crawl source authority unavailable", code="CREDENTIAL_REFUSED") + + +@asynccontextmanager +async def private() -> AsyncIterator[None]: + if not request_work.mark_mirrored(): + raise unavailable() + with private_work(): + try: + yield + except (VendorError, source_registration.Refused, ValueError, TimeoutError): + raise unavailable() from None + + +async def require_operator(source: KnowledgeSource) -> source_registration.AdminPermit: + if source.credential_mode != "crawl" or not request_work.mark_mirrored(): + raise unavailable() + return await source_registration.current_permit(crawl_authority.scope(source)) + + +@dataclasses.dataclass +class _Draft: + source: KnowledgeSource + permit: source_registration.AdminPermit + expected: KnowledgeSource | None + deadline: float + exclusion_version: str = "" + members: int = 0 + + async def validate(self) -> None: + source = self.source + if time.monotonic() >= self.deadline: + raise TimeoutError("crawl registration deadline") + await self.permit.validate(source.org) + if not settings.knowledge_mirrored_sources_enabled or not settings.auth_actually_required(): + raise unavailable() + await person_erasure.require_settled(ch=get_client()) + await ops._require_source_namespace(source.org, source.namespace) + await ops._require_no_erasure(source.org, source.namespace) + if self.expected is None: + await ops._require_free(source.org, source.namespace, source.name) + else: + with own_write_reads(): + current = await store.latest_source(source.org, source.namespace, source.name) + if current != self.expected: + raise OverloadedError("the crawl source changed; retry", retry_after_s=1) + if self.exclusion_version: + index = await crawl_exclusions.Index.open(source.org, ch=get_client()) + if await index.version() != self.exclusion_version: + raise unavailable() + await self.permit.validate(source.org) + + +async def _draft(source: KnowledgeSource, expected: KnowledgeSource | None) -> _Draft: + permit = await source_registration.current_permit(crawl_authority.scope(source)) + draft = _Draft( + source, permit, expected, time.monotonic() + settings.knowledge_source_probe_timeout_s + ) + await draft.validate() + if not await machines.fleet_at(get_client(), _CRAWL_MIGRATION): + raise OverloadedError("not every process enforces crawl authority yet", retry_after_s=30) + await draft.validate() + return draft + + +async def _approve( + source: KnowledgeSource, + credential: str, + *, + expected: KnowledgeSource | None, + dry_run: bool = False, + readmit: list[str] | None = None, +) -> tuple[KnowledgeSource, _Draft]: + draft = await _draft(source, expected) + async with asyncio.timeout_at(draft.deadline): + ops._require_sealable(credential) + account = gdrive.parse_service_account(credential) + bounds = crawl_authority.configured_bounds() + if dry_run: + # A dry run never creates a refusal registry or modifies a pending fence. + index = await crawl_exclusions.Index.open(source.org, ch=get_client()) + else: + index = await crawl_exclusions.initialize( + source.org, + authority=draft.permit, + ttl_s=bounds.lease_ttl_s, + wait_s=bounds.lease_wait_s, + ) + expansion = await crawl_expansion.expand( + Options.parse(source.crawl_options), + account, + standing=draft, + exclusions=index, + limits=bounds.directory, + deadline=draft.deadline, + ) + await crawl_delegation.require_read_only( + account, + subject=Options.parse(source.crawl_options).directory_subject, + standing=draft, + deadline=draft.deadline, + max_bytes=bounds.directory.body_bytes, + ) + if readmit: + index = await _readmit(index, draft, account, readmit, customer=expansion.customer) + expansion = await crawl_expansion.expand( + Options.parse(source.crawl_options), + account, + standing=draft, + exclusions=index, + limits=bounds.directory, + deadline=draft.deadline, + ) + draft.members = len(expansion.people) + draft.exclusion_version = expansion.exclusion_version + await draft.validate() + source = dataclasses.replace(source, crawl_customer_id=expansion.customer) + auth = current_auth() + if auth is None: + raise unavailable() + subject, approval = await source_registration.capture(auth, crawl_authority.scope(source)) + source = dataclasses.replace( + source, + registrant_subject=subject, + approval_enc=approval, + secret_enc=ops._sealed(ops._SECRET_PURPOSE, credential), + credential_enc="", + ) + await draft.validate() + if await index.version() != expansion.exclusion_version: + raise unavailable() + return source, draft + + +async def _commit(source: KnowledgeSource, draft: _Draft) -> None: + bounds = crawl_authority.configured_bounds() + async with crawl_exclusions.org_lease( + source.org, + ttl_s=bounds.lease_ttl_s, + wait_s=bounds.lease_wait_s, + ): + async with ops.sources_lease(source.org, source.namespace): + await draft.validate() + # Recheck the approval and exclusion fence under the publication leases. + await crawl_authority.validate_indexed(source) + await store.write_state( + source.org, source.namespace, source.name, state_value="", detail={} + ) + await store.insert_source(source) + await draft.permit.validate(source.org) + await crawl_authority.validate_indexed(source) + + +def _options(value: Any) -> dict[str, Any]: + return cast(dict[str, Any], json.loads(Options.parse(value).stored())) + + +def _renew(source: KnowledgeSource, credential: str) -> KnowledgeSource: + return dataclasses.replace( + source, + crawl_generation=uuid.uuid4().hex, + crawl_credential_hash=hashlib.sha256(credential.encode()).hexdigest(), + approval_enc="", + registrant_subject="", + crawl_customer_id="", + ) + + +async def add( + input: KnowledgeSourceAddInput, + *, + name: str, + kind: str, + config: dict[str, Any], + items: list[dict[str, Any]], +) -> KnowledgeSourceAddOutput: + set_query_tag("knowledge.crawl.add") + async with private(): + if kind != "gdrive" or input.crawl_options is None or not input.credential: + raise unavailable() + source = KnowledgeSource( + org=input.org, + namespace=input.namespace, + name=name, + kind=kind, + source_id=str(uuid.uuid4()), + config=config, + items=items, + access=domain.MIRRORED_ACCESS, + credential_mode="crawl", + crawl_options=_options(input.crawl_options.model_dump()), + ) + source = _renew(source, input.credential) + source, draft = await _approve( + source, input.credential, expected=None, dry_run=input.dry_run + ) + if not input.dry_run: + await _commit(source, draft) + return KnowledgeSourceAddOutput( + name=name, + kind=kind, + namespace=source.namespace, + items=source.items, + include=ops._answered_include(source), + probe=KnowledgeSourceProbe(), + dry_run=input.dry_run, + crawl_proof=_proof(source, draft), + ) + + +async def check(source: KnowledgeSource) -> KnowledgeSourceCheckOutput: + set_query_tag("knowledge.crawl.check") + async with private(): + permit = await require_operator(source) + deadline = time.monotonic() + settings.knowledge_source_probe_timeout_s + async with asyncio.timeout_at(deadline): + authority = await crawl_authority.resolve( + source, + deadline=deadline, + bounds=crawl_authority.configured_bounds(), + presenter=permit, + ) + await authority.prepare() + await permit.validate(source.org) + return KnowledgeSourceCheckOutput( + name=source.name, + summary="the delegated read-only authority was accepted", + checked_at=ops._now_iso(), + ) + + +async def rotate(source: KnowledgeSource, credential: str) -> KnowledgeSourceRotateOutput: + set_query_tag("knowledge.crawl.rotate") + async with private(): + await require_operator(source) + changed, draft = await _approve(_renew(source, credential), credential, expected=source) + await _commit(changed, draft) + return KnowledgeSourceRotateOutput( + name=source.name, rotated_at=ops._now_iso(), crawl_proof=_proof(changed, draft) + ) + + +async def update( + source: KnowledgeSource, input: KnowledgeSourceUpdateInput +) -> KnowledgeSourceUpdateOutput: + set_query_tag("knowledge.crawl.update") + async with private(): + await require_operator(source) + added = ops._items_of("gdrive", input.add_items, input.add_links, "add_", held=source) + removed = ops._as_held( + source, ops._items_of("gdrive", input.remove_items, input.remove_links, "remove_") + ) + change = ops._updated(source, added, removed, ops._include(input.include, "gdrive")) + candidate = dataclasses.replace( + change.source, + crawl_options=( + _options(input.crawl_options.model_dump()) + if input.crawl_options is not None + else source.crawl_options + ), + ) + # Credential access follows current presenter and exact source standing. + original = await _draft(source, source) + await original.validate() + await crawl_authority.validate_indexed(source) + credential = unseal(ops._SECRET_PURPOSE, source.secret_enc, warn=False) + if ( + not credential + or hashlib.sha256(credential.encode()).hexdigest() != source.crawl_credential_hash + ): + raise unavailable() + changed, draft = await _approve( + _renew(candidate, credential), credential, expected=source, readmit=input.crawl_readmit + ) + await _commit(changed, draft) + return KnowledgeSourceUpdateOutput( + name=source.name, + items=changed.items, + include=ops._answered_include(changed), + added=change.added, + removed=change.removed, + crawl_proof=_proof(changed, draft), + ) + + +def _proof(source: KnowledgeSource, draft: _Draft) -> KnowledgeSourceCrawlProof: + key = hmac.digest(settings.jwt_secret.encode(), b"crawl-audit-subjects", "sha256") + options = Options.parse(source.crawl_options) + return KnowledgeSourceCrawlProof( + subject_hashes=[ + hmac.digest(key, (s.kind + ":" + s.email).encode(), "sha256").hex() + for s in options.subjects + ], + read_scopes=[scope.value for scope in gdrive.ReadScope], + write_denials=[scope.value for scope in gdrive.WriteScope], + members=draft.members, + ) + + +async def _readmit( + index: crawl_exclusions.Index, + draft: _Draft, + account: gdrive.ServiceAccount, + requested: list[str], + *, + customer: str, +) -> crawl_exclusions.Index: + addresses = [email(value) for value in requested] + if len(addresses) != len(set(addresses)): + raise unavailable() + await draft.validate() + maximum = settings.knowledge_crawl_directory_max_response_bytes + token = await google.delegated_token( + account, + subject=Options.parse(draft.source.crawl_options).directory_subject, + scope=gdrive.ReadScope.DIRECTORY_USERS, + deadline=draft.deadline, + max_bytes=maximum, + ) + await draft.validate() + principal = await google.directory_identity(token, max_bytes=maximum) + await draft.validate() + if principal.customer_id != customer: + raise unavailable() + people = [] + for address in addresses: + found = await google.user( + token, address, customer_id=principal.customer_id, max_bytes=maximum + ) + await draft.validate() + person = Person(found.customer_id, found.id, found.email, tuple(sorted(found.aliases))) + for contact in person.addresses: + if await scim.deactivated(contact): + raise unavailable() + await draft.validate() + people.append(person) + + async def cleanup() -> None: + await draft.validate() + for person in people: + for contact in person.addresses: + if await scim.deactivated(contact): + raise unavailable() + await draft.validate() + + bounds = crawl_authority.configured_bounds() + return await index.change( + tuple(people), + exclude=False, + ttl_s=bounds.lease_ttl_s, + wait_s=bounds.lease_wait_s, + authority=draft.permit, + cleanup=cleanup, + ) diff --git a/server/knowledge/app/source_access.py b/server/knowledge/app/source_access.py index 254345e946..b1f2bcb900 100644 --- a/server/knowledge/app/source_access.py +++ b/server/knowledge/app/source_access.py @@ -85,6 +85,8 @@ class ReadPlan: default_factory=dict, repr=False ) + _crawl_pins: dict[tuple[str, str], KnowledgeSource] = field(default_factory=dict, repr=False) + @property def discovery_sources(self) -> dict[tuple[str, str], str]: """The request's fixed discovery scope, unaffected by conditional checks.""" @@ -146,6 +148,7 @@ async def _refresh_scope(self) -> None: snapshot._auth = self._auth snapshot._scope_frozen = True snapshot._connected_pins = self._connected_pins + snapshot._crawl_pins = self._crawl_pins await _prepare(snapshot, sources, self._reader_subject) self.sources = { key: original @@ -303,17 +306,53 @@ async def prepare(org: str, namespaces: list[str]) -> ReadPlan: return plan -async def _prepare(plan: ReadPlan, sources: list[dict[str, Any]], subject: str) -> None: +async def _eligible_sources( + plan: ReadPlan, + sources: list[dict[str, Any]], +) -> tuple[list[dict[str, Any]], dict[tuple[str, str], tuple[str, str]]]: + from server.knowledge.app import crawl_authority from server.knowledge.app.connected_accounts import validate_indexed from server.knowledge.store import sources as source_store eligible = [] + crawl_epochs: dict[tuple[str, str], tuple[str, str]] = {} for row in sources: key = row["namespace"], row["source_id"] mode = row.get("credential_mode", "shared") if mode == "shared": - if key not in plan._connected_pins: + if key not in plan._connected_pins and key not in plan._crawl_pins: + eligible.append(row) + continue + if mode == "crawl": + if plan._scope_frozen and key not in plan._crawl_pins: + continue + try: + source = await source_store.latest_source(plan.org, row["namespace"], row["name"]) + if ( + source is None + or source.source_id != row["source_id"] + or plan._auth is None + or current_auth() is not plan._auth + ): + continue + pinned = plan._crawl_pins.get(key) + if pinned is not None and ( + crawl_authority.scope(source) != crawl_authority.scope(pinned) + or source.approval_enc != pinned.approval_enc + or source.registrant_subject != pinned.registrant_subject + or source.secret_enc != pinned.secret_enc + ): + continue + await check_authorized(plan._auth, Role.VIEWER, plan.org, row["namespace"]) + crawl_epochs[key] = ( + await crawl_authority.validate_indexed(source), + source.crawl_generation, + ) eligible.append(row) + if not plan._scope_frozen: + plan._crawl_pins[key] = copy.deepcopy(source) + except Exception: # noqa: BLE001 — unverifiable crawl authority admits no mirror leg + continue continue if mode != "connected" or (plan._scope_frozen and key not in plan._connected_pins): continue @@ -336,7 +375,11 @@ async def _prepare(plan: ReadPlan, sources: list[dict[str, Any]], subject: str) plan._connected_pins[key] = copy.deepcopy(source) except Exception: # noqa: BLE001 — one unverifiable connection admits no private leg continue - sources = eligible + return eligible, crawl_epochs + + +async def _prepare(plan: ReadPlan, sources: list[dict[str, Any]], subject: str) -> None: + sources, crawl_epochs = await _eligible_sources(plan, sources) names = sorted({s["namespace"] for s in sources}) mirrors = await mirrors_repo.mirrors_of(plan.org, names, live_only=True) by_original: dict[str, list[str]] = {} @@ -355,5 +398,10 @@ async def _prepare(plan: ReadPlan, sources: list[dict[str, Any]], subject: str) for r in rows if (r["namespace"], r["source_id"]) in pairs and fresh(r["started_at"], now(), settings.knowledge_source_acl_stale_seconds) + and ( + (pairs[(r["namespace"], r["source_id"])], r["source_id"]) not in crawl_epochs + or (r.get("crawl_exclusion_generation"), r.get("crawl_generation")) + == crawl_epochs[(pairs[(r["namespace"], r["source_id"])], r["source_id"])] + ) } plan.sources = {key: pairs[key] for key in plan.listing_starts} diff --git a/server/knowledge/app/source_gdrive_items.py b/server/knowledge/app/source_gdrive_items.py index 5d43cea2fb..ce734d98b2 100644 --- a/server/knowledge/app/source_gdrive_items.py +++ b/server/knowledge/app/source_gdrive_items.py @@ -134,6 +134,7 @@ async def list_items( fields: str = gdrive.LISTED, pace_s: float = gdrive.PAGE_PACE_S, out_of_time: Callable[[], bool] = lambda: False, + limit: int | None = None, ) -> Listing: """Every file the items list, each once, or only those changed after `modified_after`. A rate limit, a refused credential or a failed query raises; @@ -145,12 +146,16 @@ async def list_items( the turn keeps its stamp and lists again. A `file` item is one metadata read on every listing, an incremental one too. `fields`: what is read of each file (`gdrive.LISTED_WITH_PERMISSIONS` adds its permissions).""" + if limit is not None and (type(limit) is not int or limit <= 0): + raise ValueError("listing file bound refused") out = Listing(mime_types=include_mime_types(include)) files: dict[str, dict[str, Any]] = {} def add(found: list[dict[str, Any]]) -> None: for f in found: files.setdefault(str(f.get("id") or ""), f) + if limit is not None and len(files) > limit: + raise gdrive.TooLarge("listing exceeds its file bound") roots = [i["id"] for i in items if i["type"] == "folder" and i.get("recursive")] if roots: @@ -159,6 +164,7 @@ def add(found: list[dict[str, Any]]) -> None: if out.ran_out or out_of_time(): out.ran_out = True return out + bound: dict[str, Any] = {"limit": limit} if limit is not None else {} found, whole = await gdrive.list_all( token, sorted(out.folders), @@ -166,6 +172,7 @@ def add(found: list[dict[str, Any]]) -> None: modified_after=modified_after, fields=fields, pace_s=pace_s, + **bound, ) add(found) out.whole = out.whole and whole @@ -184,6 +191,7 @@ def add(found: list[dict[str, Any]]) -> None: modified_after=modified_after, fields=fields, pace_s=pace_s, + **bound, ) except gdrive.NotFound: # A drive the account left lists nothing, which says nothing is absent. @@ -202,6 +210,7 @@ def add(found: list[dict[str, Any]]) -> None: modified_after=modified_after, fields=fields, pace_s=pace_s, + **bound, ) add(found) out.whole = out.whole and whole diff --git a/server/knowledge/app/source_sync.py b/server/knowledge/app/source_sync.py index c506f80126..13ad49ea6d 100644 --- a/server/knowledge/app/source_sync.py +++ b/server/knowledge/app/source_sync.py @@ -534,6 +534,10 @@ def _turn_of(src: KnowledgeSource) -> Callable[..., Awaitable[TurnResult]]: """The turn function of a source; this module's `_turn` reads a folder, and a mirrored folder source writes into its namespace's mirror.""" # Imported here: a kind's module builds on this one's helpers. + if src.credential_mode == "crawl" and src.kind == "gdrive" and mirror_synced(src): + from server.knowledge.app import source_sync_crawl + + return source_sync_crawl.turn if ( src.credential_mode == "connected" and src.kind in {"notion", "linear"} @@ -1273,6 +1277,10 @@ async def require_current(src: KnowledgeSource) -> None: if account is None: raise connected_accounts.unavailable() await connected_accounts._source_standing(src, account) + elif src.credential_mode == "crawl": + from server.knowledge.app import crawl_authority + + await crawl_authority.current(src).validate() @contextlib.asynccontextmanager @@ -1282,6 +1290,11 @@ async def publication_guard(src: KnowledgeSource) -> AsyncIterator[None]: async with connected_accounts.publication_guard(src): yield + elif src.credential_mode == "crawl": + from server.knowledge.app import crawl_authority + + async with crawl_authority.current(src).publication_guard(): + yield else: yield @@ -1356,7 +1369,7 @@ async def write_document( permissions=permissions, commit_guard=( functools.partial(publication_guard, src) - if src.credential_mode == "connected" + if src.credential_mode in {"connected", "crawl"} else None ), ) diff --git a/server/knowledge/app/source_sync_crawl.py b/server/knowledge/app/source_sync_crawl.py new file mode 100644 index 0000000000..dd94685522 --- /dev/null +++ b/server/knowledge/app/source_sync_crawl.py @@ -0,0 +1,250 @@ +"""A complete Workspace union precedes every content, ACL and membership publication.""" + +from __future__ import annotations + +import dataclasses +import functools +from datetime import UTC, datetime + +from server.clients import gdrive +from server.clients._vendor import VendorError +from server.core.ch import sequential_reads, set_query_tag +from server.knowledge.app import crawl_authority, mirrors, private_sync +from server.knowledge.app import source_gdrive_items as items +from server.knowledge.app import source_sync as sync +from server.knowledge.app import source_sync_mirror as mirror_sync +from server.knowledge.domain import access_list +from server.knowledge.domain.source_crawl import Expansion, File, Union +from server.knowledge.domain.sources import ( + KnowledgeSource, + home, + json_object, + mirror_uri, + mirror_uri_suffix, +) +from server.knowledge.store import crawl_publication, documents +from server.knowledge.store import mirrors as mirror_store +from server.knowledge.store import sources as store +from server.settings import settings + + +async def observe( + authority: crawl_authority.Authority, + run: sync.Pass, + *, + maximum_files: int, + observations: dict[str, items.Listing] | None = None, + result: sync.TurnResult | None = None, +) -> tuple[Expansion, tuple[File, ...]]: + """No partial person's listing, membership or ACL reaches persistent storage.""" + source = authority.source + expansion = await authority.prepare() + union = Union(expansion, maximum_files) + for person in expansion.people: + if run.budget.exhausted(): + raise TimeoutError("crawl union exceeds the pass budget") + token = await authority.drive(person) + listing = await items.list_items( + token, + source.items, + source.config.get("include", []), + fields=gdrive.LISTED_WITH_PERMISSIONS + ",owners(me,emailAddress)", + out_of_time=run.budget.exhausted, + limit=maximum_files, + ) + await token.check() + visibility = result if result is not None else sync.TurnResult() + if ( + not await sync._items_visible(token, source.items, visibility, run.budget.exhausted) + or not visibility.complete + ): + raise VendorError("crawl selected items unavailable") + if observations is not None: + observations[person.id] = listing + for file in listing.files: + if "permissions" not in file: + file["permissions"] = await gdrive.file_permissions(token, str(file["id"])) + await token.check() + union.add( + person, + listing.files, + whole=( + listing.whole + and not listing.ran_out + and not listing.lost_item + and not listing.walk_cut + ), + ) + await authority.validate() + files = union.complete() + addresses = {sync.source_uri_of(file.metadata) for file in files} + if len(addresses) != len(files) or any(not uri.startswith("https://") for uri in addresses): + raise ValueError("crawl file address is unavailable or ambiguous") + return expansion, files + + +async def turn( + source: KnowledgeSource, + state: dict[str, str], + run: sync.Pass, + *, + bounds: crawl_authority.Bounds | None = None, + membership_bound: int | None = None, + maximum_files: int | None = None, +) -> sync.TurnResult: + """Publish only a complete observation under the exact retained registration.""" + set_query_tag("knowledge.crawl.sync") + bounds = bounds or crawl_authority.configured_bounds() + maximum_files = settings.knowledge_crawl_max_files if maximum_files is None else maximum_files + if membership_bound is None: + # A group can be reached by configured alias, primary address and provider id. + membership_bound = 3 * bounds.directory.groups + len(source.crawl_options["subjects"]) + authority = await crawl_authority.resolve(source, deadline=run.budget.stop_after, bounds=bounds) + asked = json_object(state.get("detail", "")) + known = await mirrors.live_mirror(source.org, source.namespace) + detail = await mirror_sync._own_state(source, known, asked) + result = sync.TurnResult(full=bool(detail.get("full")) or sync.full_due(detail)) + sync.start_waits(result, detail) + observations: dict[str, items.Listing] = {} + started = datetime.now(UTC) + try: + expansion, files = await observe( + authority, run, maximum_files=maximum_files, observations=observations, result=result + ) + except (VendorError, ValueError, TimeoutError): + result.fail("failed", "crawl observation incomplete") + return result + async with authority.using(expansion.people): + mirror = await mirrors.ensure_mirror( + source.org, + source.namespace, + still_current=functools.partial(sync.require_current, source), + ) + if mirror != known: + detail = await mirror_sync._own_state(source, "", asked) + result.full = bool(detail.get("full")) or sync.full_due(detail) + sync.start_waits(result, detail) + source = dataclasses.replace(source, mirror=mirror) + result.mirror = mirror + async with private_sync.mirror_costs(source.org, mirror, sync.WORKER): + await _publish( + source, + authority, + expansion, + files, + observations, + started, + run, + result, + membership_bound, + ) + return result + + +async def _permissions(authority: crawl_authority.Authority, file: File) -> access_list.AccessList: + denied = await authority.exclusions.excluded_addresses(file.access.emails) + return dataclasses.replace( + file.access, emails=tuple(e for e in file.access.emails if e not in denied) + ) + + +async def _publish( + source: KnowledgeSource, + authority: crawl_authority.Authority, + expansion: Expansion, + files: tuple[File, ...], + observations: dict[str, items.Listing], + started: datetime, + run: sync.Pass, + result: sync.TurnResult, + membership_bound: int, +) -> None: + result.documents_seen = len(files) + with sequential_reads(): + copies = await store.every_copy(source.org, *home(source)) + stored = sync.index_with_room(result, [c for c in copies if c["searchable"]]) + people = {person.id: person for person in expansion.people} + listed = { + mirror_uri(sync.source_uri_of(file.metadata), source.source_id): file for file in files + } + # Narrow every old ACL before fresh content or a complete-listing proof can land. + lists = {copy["source_uri"]: access_list.UNREAD for copy in copies} + for uri, file in listed.items(): + permissions = await _permissions(authority, file) + previous = next((copy for copy in copies if copy["source_uri"] == uri), None) + if previous is not None and previous["source_revision"] != str( + file.metadata.get("modifiedTime") or "" + ): + permissions = access_list.common( + access_list.from_json(previous["access_json"]), permissions + ) + lists[uri] = permissions + if not await mirror_sync._write_lists(source, lists): + result.fail("failed", "crawl permissions changed ownership") + return + for uri, file in listed.items(): + selected = tuple(people[id] for id in file.presenters) + async with authority.using(selected): + token = await authority.drive(selected[0]) + read_access = functools.partial(_permissions, authority, file) + if not await sync.sync_file( + source, + token, + file.metadata, + uri, + stored.get(file.id), + run, + result, + read_access=read_access, + ): + break + if not result.complete or result.status != "ok" or run.budget.exhausted(): + return + if await documents.publications_owed( + source.org, source.mirror, mirror_uri_suffix(source.source_id) + ): + return + for copy in copies: + if copy["source_uri"] in listed: + continue + if run.spent(): + result.mark(sync.TurnMark.PASS_BOUND) + return + run.left -= 1 + gone = await _gone(authority, expansion, observations, copy["file_id"], result) + if gone is None: + return + if gone: + async with sync.publication_guard(source): + if await sync.retire(source, copy["source_uri"]): + result.removed += 1 + async with sync.publication_guard(source): + await crawl_publication.groups(source, expansion, started, maximum=membership_bound) + await mirror_store.record_listing( + source.org, + source.mirror, + source.source_id, + source.name, + started, + crawl_exclusion_generation=await authority.exclusions.version(), + crawl_generation=source.crawl_generation, + ) + + +async def _gone( + authority: crawl_authority.Authority, + expansion: Expansion, + observations: dict[str, items.Listing], + file_id: str, + result: sync.TurnResult, +) -> bool | None: + # Page movement can omit a reached file. Every eligible presenter must confirm absence. + if not expansion.people: + return False + for person in expansion.people: + token = await authority.drive(person) + gone = await sync.file_gone(token, file_id, observations[person.id], result) + await token.check() + if not gone: + return gone + return True diff --git a/server/knowledge/app/sources.py b/server/knowledge/app/sources.py index fcbc9f0d73..581ca7ed21 100644 --- a/server/knowledge/app/sources.py +++ b/server/knowledge/app/sources.py @@ -404,6 +404,15 @@ def _sync(src: KnowledgeSource, state: dict[str, str] | None) -> KnowledgeSource async def _details_allowed(src: KnowledgeSource) -> bool: if src.credential_mode == "shared": return True + if src.credential_mode == "crawl": + from server.knowledge.app.crawl_sources import require_operator + + with private_work(): + try: + await require_operator(src) + return True + except Exception: # noqa: BLE001 — unverifiable crawl selectors remain private + return False if src.credential_mode != "connected" or not request_work.mark_mirrored(): return False from server.knowledge.app import connected_accounts as accounts @@ -439,6 +448,7 @@ async def handle_knowledge_source_list( credential_mode=s.credential_mode, access=s.access, owner_subject=s.owner_subject if details else "", + crawl_options=s.crawl_options if details and s.credential_mode == "crawl" else {}, items=s.items if details else [], include=_answered_include(s) if details else [], config=(domain.id_list_config(s) or {}) if details else {}, @@ -468,6 +478,7 @@ async def handle_knowledge_source_inspect( credential_mode=src.credential_mode, access=src.access, owner_subject=src.owner_subject if details else "", + crawl_options=src.crawl_options if details and src.credential_mode == "crawl" else {}, items=src.items if details else [], include=_answered_include(src) if details else [], config=(domain.id_list_config(src) or {}) if details else {}, @@ -515,6 +526,12 @@ async def handle_knowledge_source_add(input: KnowledgeSourceAddInput) -> Knowled _require_kind(kind) sent = [f for f in ("items", "links") if getattr(input, f) is not None] config, items = _registration(input, kind, sent) + if input.credential_mode == "crawl": + from server.knowledge.app.crawl_sources import add as add_crawl + + return await add_crawl(input, name=name, kind=kind, config=config, items=items) + if input.crawl_options is not None: + raise BadConfigError("crawl_options requires crawl credential mode") if input.credential_mode == "connected": from server.knowledge.app.connected_sources import add @@ -591,7 +608,11 @@ async def sources_lease( # The TTL outlives the few reads and inserts of one write with room to spare (a # holder that dies is taken over after it). async with leases.holding( - _sources_lease_name(org, namespace), ttl_s=60.0, wait_s=wait_s, ch=ch + _sources_lease_name(org, namespace), + ttl_s=60.0, + wait_s=wait_s, + ch=ch, + settle_writes=True, ) as held: if not held: raise OverloadedError( @@ -670,6 +691,10 @@ async def handle_knowledge_source_check( input: KnowledgeSourceCheckInput, ) -> KnowledgeSourceCheckOutput: src = await _source_or_404(input.org, input.namespace, input.name) + if src.credential_mode == "crawl": + from server.knowledge.app.crawl_sources import check + + return await check(src) if src.credential_mode == "connected": from server.knowledge.app.connected_sources import check @@ -720,6 +745,10 @@ async def handle_knowledge_source_rotate( input: KnowledgeSourceRotateInput, ) -> KnowledgeSourceRotateOutput: src = await _source_or_404(input.org, input.namespace, input.name) + if src.credential_mode == "crawl": + from server.knowledge.app.crawl_sources import rotate + + return await rotate(src, input.credential) _require_served(src) _require_supported_recording_provider(src) if not input.credential.strip(): @@ -759,6 +788,12 @@ async def handle_knowledge_source_update( input: KnowledgeSourceUpdateInput, ) -> KnowledgeSourceUpdateOutput: src = await _source_or_404(input.org, input.namespace, input.name) + if src.credential_mode == "crawl": + from server.knowledge.app.crawl_sources import update + + return await update(src, input) + if input.crawl_options is not None or input.crawl_readmit: + raise BadConfigError("crawl_options and crawl_readmit require crawl credential mode") if src.credential_mode == "connected": from server.knowledge.app.connected_sources import update @@ -1025,6 +1060,10 @@ async def handle_knowledge_source_remove( from server.knowledge.app.connected_sources import require_operator await require_operator(src) + elif src.credential_mode == "crawl": + from server.knowledge.app.crawl_sources import require_operator as crawl_operator + + await crawl_operator(src) elif not await authz.row_in_scope(input.org, input.namespace, Role.ADMIN): raise AuthInsufficientError( "a namespace administrator is required for a shared source" @@ -1063,6 +1102,10 @@ async def handle_knowledge_source_sync_now( from server.knowledge.app.connected_sources import require_operator await require_operator(src) + elif src.credential_mode == "crawl": + from server.knowledge.app.crawl_sources import require_operator as crawl_operator + + await crawl_operator(src) else: if not await authz.row_in_scope(input.org, input.namespace, Role.ADMIN): raise AuthInsufficientError("a namespace administrator is required for a shared source") @@ -1081,6 +1124,8 @@ async def handle_knowledge_source_sync_now( ) if fresh.credential_mode == "connected": await require_operator(fresh) + elif fresh.credential_mode == "crawl": + await crawl_operator(fresh) state = await _read_state(fresh) # A full request waiting (an update's) already asks for more: a plain sync now # leaves it as it is, neither narrowed nor started over. @@ -1215,13 +1260,23 @@ async def _wait_until_free(lease_name: str, deadline: float, busy: str) -> None: def source_add_audit_subject(input: KnowledgeSourceAddInput, response: Any) -> dict[str, Any]: """What an add's audit row records: the source as stored, never the credential. `loom.audit_log` outlives the source and must hold no secret.""" + if input.credential_mode == "crawl": + return { + "org": input.org, + "namespace": input.namespace, + "name": response["name"], + "kind": response["kind"], + "credential_mode": "crawl", + "dry_run": input.dry_run, + "delegation": response["crawl_proof"], + } if input.credential_mode == "connected": return { "org": input.org, "namespace": input.namespace, "name": response["name"], "kind": response["kind"], - "credential_mode": "connected", + "credential_mode": input.credential_mode, "dry_run": input.dry_run, } return { @@ -1238,6 +1293,13 @@ def source_add_audit_subject(input: KnowledgeSourceAddInput, response: Any) -> d def source_update_audit_subject(input: KnowledgeSourceUpdateInput, response: Any) -> dict[str, Any]: """Which source changed and the items added and removed, never the credential.""" + if response.get("crawl_proof") is not None: + return { + "org": input.org, + "namespace": input.namespace, + "name": response["name"], + "delegation": response["crawl_proof"], + } if request_work.mirror_marked(): return {"org": input.org, "namespace": input.namespace, "name": response["name"]} return { @@ -1252,7 +1314,10 @@ def source_update_audit_subject(input: KnowledgeSourceUpdateInput, response: Any def source_rotate_audit_subject(input: KnowledgeSourceRotateInput, response: Any) -> dict[str, Any]: """Which source's credential was replaced, never the credential.""" - return {"org": input.org, "namespace": input.namespace, "name": response["name"]} + subject = {"org": input.org, "namespace": input.namespace, "name": response["name"]} + if response.get("crawl_proof") is not None: + subject["delegation"] = response["crawl_proof"] + return subject __all__ = [ diff --git a/server/knowledge/domain/source_crawl.py b/server/knowledge/domain/source_crawl.py new file mode 100644 index 0000000000..72947b9e59 --- /dev/null +++ b/server/knowledge/domain/source_crawl.py @@ -0,0 +1,272 @@ +"""Workspace selectors and complete crawl observations; identities remain provider-local.""" + +from __future__ import annotations + +import json +import re +from dataclasses import dataclass, field, replace +from typing import Any, Literal + +from server.knowledge.domain.access_list import ( + UNREAD, + AccessList, + from_drive_permissions, + from_json, + to_json, +) + +# not a setting: the selector wire protocol caps the total people at 2000. +MAX_PEOPLE = 2000 +_EMAIL = re.compile(r"[A-Za-z0-9._+\-]+@[A-Za-z0-9.\-]+") +type SubjectKind = Literal["person", "group"] + + +def email(value: Any) -> str: + if not isinstance(value, str) or len(value) > 512 or not _EMAIL.fullmatch(value): + raise ValueError("crawl address refused") + return value.lower() + + +@dataclass(frozen=True, repr=False) +class Subject: + kind: SubjectKind + email: str + + def __repr__(self) -> str: + return "Subject()" + + +@dataclass(frozen=True, repr=False) +class Options: + directory_subject: str + subjects: tuple[Subject, ...] + + def __repr__(self) -> str: + return "Options()" + + @classmethod + def parse(cls, value: Any) -> Options: + if not isinstance(value, dict) or set(value) != {"directory_subject", "subjects"}: + raise ValueError("crawl options refused") + entries = value["subjects"] + if not isinstance(entries, list) or not 1 <= len(entries) <= MAX_PEOPLE: + raise ValueError("crawl subjects refused") + selected: list[Subject] = [] + for entry in entries: + if ( + not isinstance(entry, dict) + or set(entry) != {"type", "email"} + or entry["type"] not in ("person", "group") + ): + raise ValueError("crawl subject refused") + subject = Subject(entry["type"], email(entry["email"])) + if subject in selected: + raise ValueError("duplicate crawl subject") + selected.append(subject) + return cls(email(value["directory_subject"]), tuple(selected)) + + def stored(self) -> str: + return json.dumps( + { + "directory_subject": self.directory_subject, + "subjects": [{"type": s.kind, "email": s.email} for s in self.subjects], + }, + sort_keys=True, + separators=(",", ":"), + ) + + +@dataclass(frozen=True, repr=False) +class Person: + customer: str + id: str + email: str + aliases: tuple[str, ...] + + def __repr__(self) -> str: + return "Person()" + + @property + def addresses(self) -> frozenset[str]: + return frozenset((self.email, *self.aliases)) + + +@dataclass(frozen=True, repr=False) +class Expansion: + customer: str + people: tuple[Person, ...] + # Each selected or nested group has the entire eligible transitive membership. + groups: tuple[tuple[str, tuple[str, ...]], ...] + exclusion_version: str + + def __repr__(self) -> str: + return "Expansion()" + + +def crawl_access(person: Person, file: dict[str, Any]) -> AccessList: + """Listing as a person grants nothing without ownership or an explicit permission.""" + raw = file.get("permissions") + if raw is not None and (not isinstance(raw, list) or any(not isinstance(p, dict) for p in raw)): + return UNREAD + parsed = from_drive_permissions(raw) + owners = file.get("owners") + owned = isinstance(owners, list) and any( + isinstance(owner, dict) + and owner.get("me") is True + and isinstance(owner.get("emailAddress"), str) + and owner["emailAddress"].lower() in person.addresses + for owner in owners + ) + explicit = not parsed.unread and bool(person.addresses.intersection(parsed.emails)) + if parsed.unread: + return AccessList(emails=(person.email,)) if owned else UNREAD + # Keep actual provider ACLs, but do not invent an entry for a link-only reader. + addresses = set(parsed.emails) + if owned or explicit: + addresses.add(person.email) + return AccessList( + emails=tuple(addresses), groups=parsed.groups, domains=parsed.domains, anyone=parsed.anyone + ) + + +def union_access(left: AccessList, right: AccessList) -> AccessList: + """Only positive observations widen a complete cycle's permission union.""" + if left.unread: + return right + if right.unread: + return left + if left.owner or right.owner: + raise ValueError("a crawl has no connected owner") + return AccessList( + emails=(*left.emails, *right.emails), + groups=(*left.groups, *right.groups), + domains=(*left.domains, *right.domains), + anyone=left.anyone or right.anyone, + ) + + +def without_people(raw: str, addresses: frozenset[str]) -> str: + """Erase exact contact entries without changing another person's valid ACL or text.""" + if not isinstance(raw, str): + raise ValueError("crawl permission history is unreadable") + try: + parsed = from_json(raw) + except RecursionError: + return "" + if parsed.unread or parsed.owner: + # A crawl never has a connected owner; malformed historical JSON proves no grant. + return "" + kept = tuple(value for value in parsed.emails if value not in addresses) + return raw if kept == parsed.emails else to_json(replace(parsed, emails=kept)) + + +def without_selectors(raw: str, addresses: frozenset[str]) -> str: + """A retired registration may have no directory subject or remaining selector.""" + try: + value = json.loads(raw) + if not isinstance(value, dict) or set(value) != {"directory_subject", "subjects"}: + raise ValueError("crawl selector history refused") + directory = value["directory_subject"] + if directory: + directory = email(directory) + elif directory != "": + raise ValueError("crawl selector history refused") + subjects = value["subjects"] + if not isinstance(subjects, list) or len(subjects) > MAX_PEOPLE: + raise ValueError("crawl selector history refused") + checked = [] + for subject in subjects: + if ( + not isinstance(subject, dict) + or set(subject) != {"type", "email"} + or subject["type"] not in ("person", "group") + ): + raise ValueError("crawl selector history refused") + checked.append({"type": subject["type"], "email": email(subject["email"])}) + kept = [s for s in checked if s["type"] != "person" or s["email"] not in addresses] + remaining = "" if directory in addresses else directory + if len(kept) == len(subjects) and remaining == directory: + return raw + return json.dumps( + {"directory_subject": remaining, "subjects": kept}, + sort_keys=True, + separators=(",", ":"), + ) + except (TypeError, KeyError, RecursionError): + raise ValueError("crawl selector history refused") from None + + +@dataclass(frozen=True, repr=False) +class File: + id: str + metadata: dict[str, Any] + access: AccessList + presenters: tuple[str, ...] + + def __repr__(self) -> str: + return "File()" + + +@dataclass(repr=False) +class Union: + """Absence is meaningful only after every admitted person's full listing finishes.""" + + expansion: Expansion + maximum_files: int + _done: set[str] = field(default_factory=set, init=False, repr=False) + _files: dict[str, File] = field(default_factory=dict, init=False, repr=False) + + def __repr__(self) -> str: + return "Union()" + + def __post_init__(self) -> None: + if type(self.maximum_files) is not int or self.maximum_files <= 0: + raise ValueError("crawl file bound refused") + + def add(self, person: Person, files: list[dict[str, Any]], *, whole: bool) -> None: + if person not in self.expansion.people or person.id in self._done or not whole: + raise ValueError("crawl listing is incomplete or changed") + observed: set[str] = set() + next_files = dict(self._files) + for file in files: + identifier = file.get("id") + if not isinstance(identifier, str) or not identifier or identifier in observed: + raise ValueError("crawl file identity is ambiguous") + if not any( + isinstance(file.get(k), str) and file[k] for k in ("version", "modifiedTime") + ): + raise ValueError("crawl file revision is unavailable") + observed.add(identifier) + access = crawl_access(person, file) + previous = next_files.get(identifier) + if previous is not None: + # A single publication must not pair one revision's text with another's ACL. + if any( + previous.metadata.get(k) != file.get(k) + for k in ( + "version", + "modifiedTime", + "md5Checksum", + "mimeType", + "webViewLink", + "name", + ) + ): + raise ValueError("crawl file changed during its union") + next_files[identifier] = File( + identifier, + previous.metadata, + union_access(previous.access, access), + (*previous.presenters, person.id), + ) + else: + next_files[identifier] = File(identifier, dict(file), access, (person.id,)) + if len(next_files) > self.maximum_files: + raise ValueError("crawl union exceeds its file bound") + self._files = next_files + self._done.add(person.id) + + def complete(self) -> tuple[File, ...]: + if self._done != {person.id for person in self.expansion.people}: + raise ValueError("crawl union is incomplete") + return tuple(self._files[key] for key in sorted(self._files)) diff --git a/server/knowledge/domain/sources.py b/server/knowledge/domain/sources.py index 7686cb3b06..899242dfbd 100644 --- a/server/knowledge/domain/sources.py +++ b/server/knowledge/domain/sources.py @@ -130,6 +130,13 @@ class KnowledgeSource: # Connected authority is explicit; created_by remains audit attribution only. owner_subject: str = "" connection_id: str = "" + # Crawl authority is independent of connected ownership and audit attribution. + crawl_options: dict[str, Any] = field(default_factory=dict) + registrant_subject: str = "" + approval_enc: str = "" + crawl_generation: str = "" + crawl_customer_id: str = "" + crawl_credential_hash: str = "" created_by: str = "" active: bool = True # On an inactive row: an admin asked for the source's documents to be retired. @@ -572,7 +579,7 @@ def mirror_synced(src: KnowledgeSource) -> bool: """Whether the sync worker writes `src` into its namespace's mirror: a mirrored folder-store source or a person's connected page/project source.""" return src.access == MIRRORED_ACCESS and ( - src.kind == "gdrive" + (src.kind == "gdrive" and src.credential_mode in {"shared", "connected", "crawl"}) or (src.credential_mode == "connected" and src.kind in {"notion", "linear"}) ) diff --git a/server/knowledge/store/crawl_exclusions.py b/server/knowledge/store/crawl_exclusions.py new file mode 100644 index 0000000000..c955aa5ab5 --- /dev/null +++ b/server/knowledge/store/crawl_exclusions.py @@ -0,0 +1,130 @@ +"""Org-scoped refusal identities; no raw provider id or address is stored.""" + +from __future__ import annotations + +from typing import Any + +from server.core.ch import own_write_reads +from server.core.repo_base import _CHClient, _default_client +from server.core.versions import next_version + +KEYS = "loom.knowledge_crawl_exclusion_keys" +REFUSALS = "loom.knowledge_crawl_exclusions" + + +async def key(org: str, *, ch: _CHClient | None = None) -> dict[str, Any] | None: + with own_write_reads(): + return await (ch or _default_client()).query_one( + "SELECT key_id, key_enc, generation, pending_id, pending_digests, " + "pending_exclude, version " + f"FROM {KEYS} FINAL " + "WHERE org = {org:String} LIMIT 1", + params={"org": org}, + ) + + +async def key_put( + org: str, + key_id: str, + key_enc: str, + *, + generation: str, + pending_id: str = "", + pending_digests: tuple[str, ...] = (), + pending_exclude: bool = False, + ch: _CHClient | None = None, +) -> None: + client = ch or _default_client() + await client.insert( + KEYS, + [ + { + "org": org, + "key_id": key_id, + "key_enc": key_enc, + "generation": generation, + "pending_id": pending_id, + "pending_digests": list(pending_digests), + "pending_exclude": pending_exclude, + "version": await next_version(client, KEYS, {"org": org}), + } + ], + ) + + +async def any_refusal(org: str, *, ch: _CHClient | None = None) -> bool: + with own_write_reads(): + return ( + await (ch or _default_client()).query_one( + f"SELECT digest FROM {REFUSALS} FINAL WHERE org = {{org:String}} LIMIT 1", + params={"org": org}, + ) + is not None + ) + + +async def excluded(org: str, digests: list[str], *, ch: _CHClient | None = None) -> set[str]: + if not digests: + return set() + with own_write_reads(): + rows = await (ch or _default_client()).query_all( + f"SELECT digest FROM {REFUSALS} FINAL WHERE org = {{org:String}} " + "AND digest IN {digests:Array(String)} AND excluded = 1 LIMIT {limit:UInt32}", + params={"org": org, "digests": digests, "limit": len(digests)}, + ) + found = {str(row["digest"]) for row in rows} + if len(rows) != len(found) or not found.issubset(digests): + raise RuntimeError("crawl_exclusions_unverified") + return found + + +async def put(org: str, digests: list[str], *, exclude: bool, ch: _CHClient | None = None) -> None: + if digests: + client = ch or _default_client() + await client.insert( + REFUSALS, + [ + { + "org": org, + "digest": digest, + "excluded": exclude, + "version": await next_version(client, REFUSALS, {"org": org, "digest": digest}), + } + for digest in sorted(set(digests)) + ], + ) + + +async def page(after: str, maximum: int, *, ch: _CHClient | None = None) -> list[str]: + if type(maximum) is not int or maximum <= 0: + raise ValueError("crawl registry page bound refused") + with own_write_reads(): + rows = await (ch or _default_client()).query_all( + f"SELECT org FROM {KEYS} FINAL WHERE org > {{after:String}} " + "ORDER BY org LIMIT {limit:UInt32}", + params={"after": after, "limit": maximum}, + ) + orgs = [str(row["org"]) for row in rows] + if len(orgs) > maximum or orgs != sorted(set(orgs)) or any(org <= after for org in orgs): + raise RuntimeError("crawl_registry_page_unverified") + return orgs + + +async def restore_orgs(*, maximum: int, ch: _CHClient) -> list[str]: + with own_write_reads(): + rows = await ch.query_all( + "SELECT DISTINCT org FROM (SELECT org FROM loom.knowledge_sources " + "WHERE credential_mode = 'crawl' UNION ALL " + "SELECT org FROM loom.knowledge_crawl_exclusion_keys) " + "ORDER BY org LIMIT {limit:UInt32}", + params={"limit": maximum + 1}, + ) + orgs = [row["org"] for row in rows] + if ( + maximum <= 0 + or len(orgs) > maximum + or orgs != sorted(set(orgs)) + or any(not isinstance(org, str) or not org or "\0" in org for org in orgs) + ): + raise RuntimeError("crawl_restore_inventory_unverified") + return orgs diff --git a/server/knowledge/store/crawl_publication.py b/server/knowledge/store/crawl_publication.py new file mode 100644 index 0000000000..4d0b767ecb --- /dev/null +++ b/server/knowledge/store/crawl_publication.py @@ -0,0 +1,59 @@ +"""Full source-scoped membership replacement, written only under crawl publication guards.""" + +from __future__ import annotations + +from datetime import datetime + +from server.core.ch import own_write_reads, set_query_tag +from server.core.repo_base import _CHClient, _default_client +from server.core.versions import next_version +from server.knowledge.domain.source_crawl import Expansion +from server.knowledge.domain.sources import KnowledgeSource + +GROUPS = "loom.knowledge_group_members" + + +async def groups( + source: KnowledgeSource, + expansion: Expansion, + started: datetime, + *, + maximum: int, + ch: _CHClient | None = None, +) -> None: + set_query_tag("knowledge.crawl.groups") + if type(maximum) is not int or maximum <= 0 or not source.mirror: + raise ValueError("crawl membership bounds refused") + client = ch or _default_client() + params = {"org": source.org, "ns": source.mirror, "id": source.source_id} + with own_write_reads(): + prior = await client.query_all( + f"SELECT `group` FROM {GROUPS} FINAL WHERE org = {{org:String}} " + "AND namespace = {ns:String} AND source_id = {id:String} " + "LIMIT {limit:UInt32}", + params={**params, "limit": maximum + 1}, + ) + found = dict(expansion.groups) + names = set(found) | {str(row["group"]) for row in prior} + if len(prior) > maximum or len(names) > maximum: + raise ValueError("crawl membership bound exceeded") + rows = [] + for group in sorted(names): + key = { + "org": source.org, + "namespace": source.mirror, + "source_id": source.source_id, + "group": group, + } + rows.append( + { + **key, + "name": source.name, + "members": list(found.get(group, ())), + "read_at": started, + "deleted": group not in found, + "version": await next_version(client, GROUPS, key), + } + ) + if rows: + await client.insert(GROUPS, rows) diff --git a/server/knowledge/store/mirrors.py b/server/knowledge/store/mirrors.py index 1d9211b02f..e1fcf12ab6 100644 --- a/server/knowledge/store/mirrors.py +++ b/server/knowledge/store/mirrors.py @@ -53,6 +53,8 @@ async def record_listing( name: str, started_at: datetime, *, + crawl_exclusion_generation: str = "", + crawl_generation: str = "", ch: _CHClient | None = None, ) -> None: """The start of a mirrored source's last complete listing, written once every list @@ -63,6 +65,8 @@ async def record_listing( "source_id": source_id, "name": name, "started_at": started_at, + "crawl_exclusion_generation": crawl_exclusion_generation, + "crawl_generation": crawl_generation, } await (ch or _default_client()).insert(LISTINGS, [row]) diff --git a/server/knowledge/store/source_access.py b/server/knowledge/store/source_access.py index 00adc82a30..778d4d7ee7 100644 --- a/server/knowledge/store/source_access.py +++ b/server/knowledge/store/source_access.py @@ -61,7 +61,8 @@ async def listings( if not pairs: return [] return await (ch or _default_client()).query_all( - "SELECT namespace, source_id, started_at FROM loom.knowledge_source_listings FINAL " + "SELECT namespace, source_id, started_at, crawl_exclusion_generation, crawl_generation " + "FROM loom.knowledge_source_listings FINAL " "WHERE org = {org:String} AND (namespace, source_id) " "IN {pairs:Array(Tuple(String, String))} " "LIMIT {limit:UInt32}", diff --git a/server/knowledge/store/sources.py b/server/knowledge/store/sources.py index 3a39f327ca..310d57fedd 100644 --- a/server/knowledge/store/sources.py +++ b/server/knowledge/store/sources.py @@ -2,14 +2,15 @@ each source's sync state (a `loom.code_index_state` row under `knowledge_source:`). -Both tables are ReplacingMergeTree(version): every write is a full version row -and every read uses FINAL. A registry row changes only on add, update, rotate, remove +Ordinary writes use ReplacingMergeTree(version) rows; latest-version reads use +FINAL. Canonical-person cleanup redacts raw history under the namespace lease. +A registry row otherwise changes on add, update, rotate, remove and the retirement when its namespace goes (`code.repo.remove`, `namespace.delete`, `collection.delete`, the erasure step `knowledge_sources.retire`), all under the -namespace's lease, and on `loom secrets reseal`, the one writer outside it: it -writes the row it read at version + 1, so any later write wins while the ClickHouse -servers' clocks differ by less than the time between the two writes. The sync state is -a separate row, so a sync never rewrites the row that holds the credential. +namespace's lease. `loom secrets reseal` takes the same lease and reads the newest +ciphertext before conditionally updating only that column across its history. +Even a late statement cannot revive metadata removed by canonical-person cleanup. +The sync state is a separate row, so a sync never rewrites the credential row. A source a build without items can read (`id_list_config` is not None) is written as that build writes it, `kind`, `config_json` and the credential in `credential_enc`, @@ -26,9 +27,11 @@ from datetime import datetime from typing import Any, Literal, NamedTuple +from server.core.ch import own_write_reads from server.core.datetimes import naive_utc_now_seconds from server.core.repo_base import _CHClient, _default_client from server.knowledge.domain.live_chunks import has_live_chunks_sql +from server.knowledge.domain.source_crawl import Options, without_selectors from server.knowledge.domain.sources import ( NAMESPACE_ACCESS, SYNC_WORKER_ACTOR, @@ -53,11 +56,133 @@ _COLUMNS = ( "org, namespace, name, source_id, kind, source_kind, config_json, items_json, " "credential_enc, secret_enc, access, credential_mode, owner_subject, connection_id, " + "options_json, registrant_subject, approval_enc, crawl_generation, " + "crawl_customer_id, crawl_credential_hash, " "created_by, active, pending_delete, created_at, " "updated_at" ) +async def crawl_approval_page( + after: tuple[str, str, str], *, maximum: int, ch: _CHClient | None = None +) -> list[tuple[str, str, str]]: + if type(maximum) is not int or maximum <= 0: + raise ValueError("crawl approval page bound refused") + with own_write_reads(): + rows = await (ch or _default_client()).query_all( + f"SELECT DISTINCT org, namespace, name FROM {TABLE} " + "WHERE (org, namespace, name) > {after:Tuple(String, String, String)} " + "AND approval_enc != '' ORDER BY org, namespace, name LIMIT {limit:UInt32}", + params={"after": after, "limit": maximum}, + ) + keys = [(str(row["org"]), str(row["namespace"]), str(row["name"])) for row in rows] + if len(keys) > maximum or keys != sorted(set(keys)) or any(key <= after for key in keys): + raise RuntimeError("crawl_approval_page_unverified") + return keys + + +async def crawl_approval_values( + org: str, + namespace: str, + name: str, + *, + maximum: int, + ch: _CHClient, +) -> list[dict[str, Any]]: + rows = await ch.query_all( + f"SELECT DISTINCT toString(source_id) AS source_id, approval_enc FROM {TABLE} " + "WHERE org = {org:String} AND namespace = {ns:String} AND name = {name:String} " + "AND credential_mode = 'crawl' AND approval_enc != '' LIMIT {limit:UInt32}", + params={"org": org, "ns": namespace, "name": name, "limit": maximum + 1}, + ) + if len(rows) > maximum: + raise RuntimeError("crawl_approval_history_bound") + return rows + + +async def reseal_crawl_approval( + org: str, + namespace: str, + source_id: str, + before: str, + after: str, + *, + ch: _CHClient, +) -> None: + where = ( + "org = {org:String} AND namespace = {ns:String} AND source_id = {id:String} " + "AND credential_mode = 'crawl' AND approval_enc = {old:String}" + ) + params = {"org": org, "ns": namespace, "id": source_id, "old": before, "new": after} + await ch.execute( + f"ALTER TABLE {TABLE} UPDATE approval_enc = {{new:String}} WHERE {where}", + params=params, + settings={"mutations_sync": 2}, + ) + residue = await ch.query_one(f"SELECT count() AS n FROM {TABLE} WHERE {where}", params=params) + if residue is None or residue["n"]: + raise RuntimeError("crawl_approval_reseal_residue") + + +async def reseal_source_ciphertext( + org: str, + namespace: str, + name: str, + source_id: str, + column: str, + before: str, + after: str, + *, + ch: _CHClient, +) -> None: + if column not in {"credential_enc", "secret_enc"}: + raise ValueError("source ciphertext column refused") + where = ( + "org = {org:String} AND namespace = {ns:String} AND name = {name:String} " + f"AND source_id = {{id:String}} AND {column} = {{old:String}}" + ) + params = { + "org": org, + "ns": namespace, + "name": name, + "id": source_id, + "old": before, + "new": after, + } + # Copy no metadata: a late statement cannot restore canonical-person data. + await ch.execute( + f"ALTER TABLE loom.knowledge_sources UPDATE {column} = {{new:String}} WHERE {where}", + params=params, + settings={"mutations_sync": 2}, + ) + residue = await ch.query_one( + f"SELECT count() AS n FROM loom.knowledge_sources WHERE {where}", + params=params, + ) + if residue is None or residue["n"]: + raise RuntimeError("source_reseal_residue") + + +async def crawl_history( + org: str, *, maximum: int, ch: _CHClient | None = None +) -> list[KnowledgeSource]: + """Raw registrations include retired sources and earlier names, not only FINAL rows.""" + if not isinstance(org, str) or not org or type(maximum) is not int or maximum <= 0: + raise ValueError("crawl history scope refused") + with own_write_reads(): + rows = await (ch or _default_client()).query_all( + f"SELECT DISTINCT {_COLUMNS} FROM {TABLE} WHERE org = {{org:String}} " + "AND credential_mode = 'crawl' " + "ORDER BY namespace, name, source_id, updated_at LIMIT {limit:UInt32}", + params={"org": org, "limit": maximum + 1}, + ) + if len(rows) > maximum or any( + row["org"] != org or row["credential_mode"] != "crawl" for row in rows + ): + raise RuntimeError("crawl_history_unverified") + return [_source(row) for row in rows] + + def _items(raw: str) -> list[dict[str, Any]]: """The stored items, or [] (the source's config names them).""" try: @@ -67,6 +192,18 @@ def _items(raw: str) -> list[dict[str, Any]]: return [i for i in data if isinstance(i, dict)] if isinstance(data, list) else [] +def _crawl_options(row: dict[str, Any]) -> dict[str, Any]: + if row.get("credential_mode") != "crawl": + return {} + raw = str(row.get("options_json") or "{}") + if row.get("active"): + return json_object(Options.parse(json.loads(raw)).stored()) + # Preserve a retired source's remaining selectors during full-row reseals. + # Erasure may have removed its directory subject or its final person selector. + without_selectors(raw, frozenset()) + return json_object(raw) + + def _source(row: dict[str, Any]) -> KnowledgeSource: created = row.get("created_at") updated = row.get("updated_at") @@ -87,6 +224,12 @@ def _source(row: dict[str, Any]) -> KnowledgeSource: credential_mode=str(row.get("credential_mode") or "shared"), owner_subject=str(row.get("owner_subject") or ""), connection_id=str(row.get("connection_id") or ""), + crawl_options=_crawl_options(row), + registrant_subject=str(row.get("registrant_subject") or ""), + approval_enc=str(row.get("approval_enc") or ""), + crawl_generation=str(row.get("crawl_generation") or ""), + crawl_customer_id=str(row.get("crawl_customer_id") or ""), + crawl_credential_hash=str(row.get("crawl_credential_hash") or ""), created_by=str(row.get("created_by") or ""), active=bool(row.get("active")), pending_delete=bool(row.get("pending_delete")), @@ -275,6 +418,17 @@ async def insert_source(src: KnowledgeSource, *, ch: _CHClient | None = None) -> writing replica's: the lease-serialized writes stay ordered while those clocks differ by less than the lease's handoff between writes.""" now = naive_utc_now_seconds() + options = json.dumps( + _crawl_options( + { + "credential_mode": src.credential_mode, + "active": src.active, + "options_json": json.dumps(src.crawl_options), + } + ), + sort_keys=True, + separators=(",", ":"), + ) row = { "org": src.org, "namespace": src.namespace, @@ -290,6 +444,12 @@ async def insert_source(src: KnowledgeSource, *, ch: _CHClient | None = None) -> "credential_mode": src.credential_mode, "owner_subject": src.owner_subject, "connection_id": src.connection_id, + "options_json": options, + "registrant_subject": src.registrant_subject, + "approval_enc": src.approval_enc, + "crawl_generation": src.crawl_generation, + "crawl_customer_id": src.crawl_customer_id, + "crawl_credential_hash": src.crawl_credential_hash, "created_by": src.created_by, "active": src.active, "pending_delete": src.pending_delete, diff --git a/server/main.py b/server/main.py index 777b0b78ac..d32f42c3c4 100644 --- a/server/main.py +++ b/server/main.py @@ -551,14 +551,19 @@ def attach_browser_middleware(target: FastAPI) -> None: from server.auth import login as _login # noqa: E402 from server.auth import oauth as _oauth # noqa: E402 +from server.auth import person_erasure as _person_erasure # noqa: E402 from server.auth import scim_routes as _scim_routes # noqa: E402 from server.auth import self_signup as _self_signup # noqa: E402 +from server.erasure.app import crawl_person as _crawl_erasure # noqa: E402 from server.identity.app import agent_claim as _agent_claim # noqa: E402 from server.identity.app import claim_release as _claim_release # noqa: E402 from server.identity.app import password_link as _password_link # noqa: E402 from server.identity.app import scim_groups as _scim_groups # noqa: E402 from server.knowledge.app import connection_routes as _connected_accounts # noqa: E402 +from server.knowledge.app import crawl_capture as _crawl_capture # noqa: E402 +from server.knowledge.app import crawl_person_erasure as _crawl_person # noqa: E402 +_person_erasure.bind(_crawl_capture.prepare, _crawl_person.finish, _crawl_erasure.finalize) _self_signup.attach(app) _login.attach(app) _connected_accounts.attach(app) diff --git a/server/settings/catalog.py b/server/settings/catalog.py index c2a2f43d87..36a3ce6749 100644 --- a/server/settings/catalog.py +++ b/server/settings/catalog.py @@ -2451,6 +2451,26 @@ class KnowledgeSettings(BaseModel): knowledge_connected_registry_scan_limit: int = Field(default=4096, ge=1) # Registrations per page during root-secret resealing. knowledge_connected_registry_page_size: int = Field(default=256, ge=1, le=4096) + # Bytes read from one delegated Directory or OAuth response; larger responses refuse. + knowledge_crawl_directory_max_response_bytes: int = Field(default=1024 * 1024, ge=1024) + # Directory membership pages across the entire expansion, including nested groups. + knowledge_crawl_directory_max_pages: int = Field(default=256, ge=1) + # Groups across one complete crawl expansion; the total-person ceiling stays fixed. + knowledge_crawl_max_groups: int = Field(default=2000, ge=1) + # Distinct files across every person's complete listing; a larger union publishes nothing. + knowledge_crawl_max_files: int = Field(default=10000, ge=1) + # Maximum full control-registry scan for cleanup; a larger scan refuses partial work. + knowledge_crawl_registry_scan_limit: int = Field(default=4096, ge=1) + # Org control rows per root-secret reseal page. + knowledge_crawl_registry_page_size: int = Field(default=256, ge=1, le=4096) + # Seconds the renewed org exclusion lease lives; namespace publication leases follow it. + knowledge_crawl_lease_ttl_s: float = Field(default=60.0, gt=0) + # Seconds a crawl control change waits for an existing org exclusion lease. + knowledge_crawl_lease_wait_s: float = Field(default=5.0, ge=0) + # Whole pre-delete Directory identity capture across a person's existing crawl sources. + knowledge_crawl_identity_capture_timeout_s: float = Field(default=30.0, gt=0) + # Whole local canonical-person crawl cleanup, including publisher fencing and residue. + knowledge_crawl_person_cleanup_timeout_s: float = Field(default=60.0, gt=0) # Seconds one connected-account OAuth request may take, including its response. # The caller's remaining deadline can shorten this bound. knowledge_connected_oauth_timeout_s: float = Field(default=30.0, gt=0) diff --git a/tests/analyzer/shapes.txt b/tests/analyzer/shapes.txt index 955a96a71a..faafecc5bc 100644 --- a/tests/analyzer/shapes.txt +++ b/tests/analyzer/shapes.txt @@ -869,6 +869,7 @@ 65c31ffcda22 SELECT count() AS n FROM loom.entity_edge_rows WHERE org = {org:String} AND namespace = {ns:String} AND ? 65e630aa6409 SELECT count() AS n FROM loom.code_files FINAL WHERE org = {org:String} AND namespace = {ns:String} AND tombstoned_at IS NULL AND index_error != ? 663c96bc8c2e SELECT count() AS cnt, sum(numeric_value) AS s, avg(numeric_value) AS a, min(occurred_at) AS first_at, max(occurred_at) AS last_at FROM loom.derived_facts WHERE org = {org:String} AND namespace = {ns: +666058857b23 SELECT namespace, source_id, started_at, crawl_exclusion_generation, crawl_generation FROM loom.knowledge_source_listings FINAL WHERE org = {org:String} AND (namespace, source_id) IN {pairs:Array(Tupl 666fcf41eb4a SELECT concat(?, toString(c.session_id), ?, toString(c.chunk_index)) AS memory_key, ? AS memory_type, substring(c.content, ?, ?) AS content_excerpt, c.content AS content_full, c.agent AS agent, c.user 66a80f4706a4 ALTER TABLE loom.operator_edges DELETE WHERE (org = {org:String} AND namespace IN {ns:Array(String)}) AND ({erasure_mutation_tag:String} = {erasure_mutation_tag:String}) SETTINGS mutations_sync = ? 66b1526f73ec SELECT e.memory_key AS memory_key, e.memory_type AS memory_type, e.content_excerpt AS content_excerpt, e.status AS status, e.version AS version, cosineDistance(e.embedding, {q:Array(Float32)}) AS dist @@ -1038,7 +1039,6 @@ 7b240e4ee7d6 SELECT tier, edge_kind, to_symbol_id, from_symbol_id, path, line, argMax(from_qualified_name, version) AS from_qualified_name, argMax(to_qualified_name, version) AS to_qualified_name, argMax(confidenc 7b29c9cf5ab6 SELECT org, namespace, agent, user_id, thread_id, title, metadata, run_id, created_at, updated_at, tombstoned_at FROM loom.conversation_threads FINAL WHERE org = {org:String} AND namespace = {scope:St 7b31f76d8d46 SELECT entity, max(degree) AS degree FROM loom.entity_degree WHERE org = {o:String} AND namespace = {n:String} GROUP BY entity ORDER BY degree DESC, entity LIMIT {lim:UInt32} -7b5336f3029c SELECT namespace, source_id, started_at FROM loom.knowledge_source_listings FINAL WHERE org = {org:String} AND (namespace, source_id) IN {pairs:Array(Tuple(String, String))} LIMIT {limit:UInt32} 7b590beeb043 SELECT count() AS n FROM loom.llm_response_cache WHERE org = {org:String} AND namespace = {ns:String} AND ? 7b6e39c58a8e SELECT agent, user_id, session_id, memory_key, memory_type, content_excerpt, toString(created_at) AS created_at, toString(valid_at) AS valid_at, version FROM loom.memory_embeddings FINAL WHERE org = { 7b81be391890 SELECT count() AS n FROM loom.memory_versions WHERE org = {org:String} AND user_id = {uid:String} @@ -1079,7 +1079,6 @@ 7dfbb777603e ALTER TABLE loom.memory_evidence DELETE WHERE (org = {org:String} AND namespace IN {ns:Array(String)}) AND ({erasure_mutation_tag:String} = {erasure_mutation_tag:String}) SETTINGS mutations_sync = ? 7e34abf20d24 ALTER TABLE loom.memory_record_versions DELETE WHERE (org = {org:String} AND namespace IN {ns:Array(String)}) AND ({erasure_mutation_tag:String} = {erasure_mutation_tag:String}) SETTINGS mutations_syn 7e726e6f73c0 SELECT model, count() AS turns, uniqExact(session_id) AS sessions, sum(toUInt64(input_tokens)) AS input_tokens, sum(toUInt64(cache_read_tokens)) AS cache_read_tokens, sum(toUInt64(cache_creation_5m_to -7eccab6dfda3 SELECT org, namespace, name, source_id, kind, source_kind, config_json, items_json, credential_enc, secret_enc, access, credential_mode, owner_subject, connection_id, created_by, active, pending_delet 7ed9e5f6ceb3 ALTER TABLE loom.memory_chunk_sources UPDATE tombstoned_at = {now:DateTime} WHERE (org = {org:String} AND tombstoned_at IS NULL AND namespace = {ns:String}) AND ({erasure_mutation_tag:String} = {erasu 7ef1d533ac81 SELECT count() AS n FROM loom.memory_evidence WHERE org = {org:String} AND namespace = {ns:String} AND ? 7f22d53d8b16 SELECT uniqExact(k) AS n FROM ( SELECT k FROM (SELECT arrayJoin(memories_used) AS k FROM loom.agent_runs FINAL WHERE org = {org:String} AND namespace = {ns:String} AND agent = {agent:String} AND start @@ -1982,6 +1981,7 @@ df886269fbb5 SELECT org, namespace FROM loom.namespace_grants FINAL WHERE role = df9a592b3176 SELECT min(version) AS v FROM loom.memory_versions WHERE org = {org:String} AND namespace = {ns:String} dfab46389b75 SELECT count() AS n FROM loom.memory_record_versions WHERE org = {org:String} AND namespace = {ns:String} AND ? dfb389c3eaaa SELECT count() AS n FROM loom.memory_records FINAL WHERE org = {org:String} AND namespace = {ns:String} AND tombstoned_at IS NULL +dfbb9c9e86b1 SELECT org, namespace, name, source_id, kind, source_kind, config_json, items_json, credential_enc, secret_enc, access, credential_mode, owner_subject, connection_id, options_json, registrant_subject, dfdd19d591c8 SELECT memory_id, org, namespace, agent, user_id, session_id, memory_key, memory_type, value, ? AS value_truncated, confidence, version, pinned, char_limit, created_at, updated_at, expires_at, valid_a e01b3f138368 ALTER TABLE loom.memory_ask_precedents DELETE WHERE (org = {org:String} AND hasAny(arrayConcat(window_keys, cited_keys), {keys:Array(String)})) AND ({erasure_mutation_tag:String} = {erasure_mutation_t e026c7f5b6eb SELECT memory_type, toString(status) AS status, source_class, count() AS rows, min(updated_at) AS first_at, max(updated_at) AS last_at FROM loom.memory FINAL WHERE org = {org:String} AND namespace = { @@ -2049,6 +2049,7 @@ e6f6f6bc74e2 SELECT count() AS n FROM loom.memory_chunks FINAL WHERE org = {org: e7331ee31d1b SELECT concat(?, toString(c.session_id), ?, toString(c.chunk_index)) AS memory_key, ? AS memory_type, substring(c.content, ?, ?) AS content_excerpt, c.content AS content_full, c.agent AS agent, c.user e747c3134919 ALTER TABLE loom.memory_record_fields UPDATE tombstoned_at = {now:DateTime} WHERE (org = {org:String} AND tombstoned_at IS NULL AND record_id IN {ids:Array(UUID)}) AND ({erasure_mutation_tag:String} = e75f4a024984 SELECT count() AS n FROM loom.derived_facts WHERE org = {org:String} AND namespace = {ns:String} AND ? +e764b7e8412c -- schemas/schema.sql -- -- Reference dump of `loom.*` and `loom_telemetry.*`: what replaying schemas/migrations/*.sql -- produces, auto-generated by scripts/ops/dump_schema.py from a live -- ClickHou e77afbb3fb28 SELECT count() AS n FROM loom.chunks WHERE org = {org:String} AND namespace IN {ns:Array(String)} AND doc_superseded_at IS NULL AND (org, namespace, embedding_model, source_uri, doc_version) NOT IN (S e77b590b9f54 SELECT org, namespace, name, source_id, provider, generation, fingerprint, provider_id, status, secret_enc, agent, user_id, session_id, actor FROM loom.knowledge_source_push FINAL WHERE org = {org:Str e77c1e05f3c4 SELECT count() AS n FROM loom.blob_uploads WHERE org = {org:String} AND namespace IN {ns:Array(String)} AND ? diff --git a/tests/integration/crawl_native_privacy.py b/tests/integration/crawl_native_privacy.py new file mode 100644 index 0000000000..4b7ed119ab --- /dev/null +++ b/tests/integration/crawl_native_privacy.py @@ -0,0 +1,402 @@ +"""Native crawl publication and HTTP ACL isolation on an exclusively owned backend. + +Run this module explicitly in the crawl-native-privacy workflow. The full app and +inline workers share one process; ordinary integration jobs use an external server. +Verified SSO contacts are trusted fixture inputs, not enrollment coverage. +""" + +from __future__ import annotations + +import json +import uuid +from collections.abc import AsyncIterator, Iterator +from datetime import UTC, datetime +from typing import Any +from urllib.parse import parse_qs, unquote + +import httpx +import jwt +import pytest +from cryptography.hazmat.primitives import serialization +from cryptography.hazmat.primitives.asymmetric import rsa + +from tests.integration._eventually import eventually +from tests.integration._resources import unique_org + + +# This process exercises normal boot instead of the integration suite's in-process +# shortcuts for tests that do not start an app. +@pytest.fixture(autouse=True) +def _seed_test_api_key() -> Iterator[None]: + yield + + +@pytest.fixture(autouse=True) +def _code_registry_read() -> Iterator[None]: + yield + + +@pytest.fixture(autouse=True) +def _quick_intent_marks() -> Iterator[None]: + yield + + +@pytest.fixture +async def provider(monkeypatch: pytest.MonkeyPatch) -> AsyncIterator[dict[str, Any]]: + from server.clients import gdrive + from server.clients import google_workspace as google + + private = rsa.generate_private_key(public_exponent=65537, key_size=2048) + pem = private.private_bytes( + serialization.Encoding.PEM, + serialization.PrivateFormat.PKCS8, + serialization.NoEncryption(), + ).decode() + canary = uuid.uuid4().hex + content = f"# Quarterly renewal plan\n\nScope owners and release readiness. {canary}\n" + directory = "directory@example.test" + reader = "reader-a@example.test" + outside_reader = "reader-c@example.test" + file_id = "native-privacy-document" + metadata = { + "id": file_id, + "version": "1", + "name": "Quarterly renewal plan", + "mimeType": "application/vnd.google-apps.document", + "modifiedTime": datetime.now(UTC).isoformat(timespec="milliseconds").replace("+00:00", "Z"), + "webViewLink": f"https://docs.google.com/document/d/{file_id}", + "permissions": [ + {"type": "user", "role": "reader", "emailAddress": address} + for address in (reader, outside_reader) + ], + "owners": [{"me": True, "emailAddress": reader}], + } + scopes: set[str] = set() + exported: list[str] = [] + unexpected: list[str] = [] + + async def handle(request: httpx.Request) -> httpx.Response: + if request.url.host == "oauth2.googleapis.com": + assert request.method == "POST" and request.url.path == "/token" + form = parse_qs(request.content.decode()) + assert form["grant_type"] == ["urn:ietf:params:oauth:grant-type:jwt-bearer"] + claims = jwt.decode( + form["assertion"][0], + private.public_key(), + algorithms=["RS256"], + audience="https://oauth2.googleapis.com/token", + ) + assert claims["iss"] == "service@example.test" + scope = claims["scope"] + scopes.add(scope) + if scope in {s.value for s in gdrive.WriteScope}: + assert claims["sub"] == directory + return httpx.Response(400, json={"error": "unauthorized_client"}) + assert scope in {s.value for s in gdrive.ReadScope} + assert claims["sub"] in {directory, reader} + drive = claims["sub"] == reader + assert not drive or scope == gdrive.ReadScope.DRIVE.value + return httpx.Response( + 200, + json={ + "access_token": "native-drive" if drive else "native-directory", + "token_type": "Bearer", + "expires_in": 3600, + "scope": scope, + }, + ) + assert request.method == "GET" + if request.url.host == "admin.googleapis.com": + assert request.headers["authorization"] == "Bearer native-directory" + assert request.url.path.startswith("/admin/directory/v1/users/") + key = unquote(request.url.path.rsplit("/", 1)[1]) + assert key in {directory, reader, "provider-person"} + is_directory = key == directory + return httpx.Response( + 200, + json={ + "id": "provider-directory" if is_directory else "provider-person", + "primaryEmail": directory if is_directory else reader, + "aliases": [], + "customerId": "native-workspace", + "suspended": False, + "archived": False, + }, + ) + assert request.url.host == "www.googleapis.com", request.url.host + assert request.headers["authorization"] == "Bearer native-drive" + if request.url.path == "/drive/v3/files": + return httpx.Response(200, json={"files": [metadata], "incompleteSearch": False}) + if request.url.path == f"/drive/v3/files/{file_id}": + return httpx.Response(200, json=metadata) + assert request.url.path == f"/drive/v3/files/{file_id}/export", request.url.path + exported.append(file_id) + return httpx.Response(200, text=content) + + async def dispatch(request: httpx.Request) -> httpx.Response: + try: + return await handle(request) + except AssertionError: + unexpected.append(f"{request.method} {request.url.host}{request.url.path}") + raise + + async with httpx.AsyncClient(transport=httpx.MockTransport(dispatch)) as vendor: + monkeypatch.setattr(google, "get_client", lambda: vendor) + monkeypatch.setattr(gdrive, "get_client", lambda: vendor) + yield { + "canary": canary, + "content": content, + "reader": reader, + "outside_reader": outside_reader, + "directory": directory, + "scopes": scopes, + "exported": exported, + "unexpected": unexpected, + "credential": json.dumps( + { + "type": "service_account", + "client_email": "service@example.test", + "private_key": pem, + } + ), + } + assert not unexpected, unexpected + + +@pytest.fixture +async def native_app(provider: dict[str, Any]) -> AsyncIterator[httpx.AsyncClient]: + from server.main import app + + async with app.router.lifespan_context(app): + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=app), base_url="http://test" + ) as client: + + async def ready() -> bool: + assert not provider["unexpected"], provider["unexpected"] + result = await client.get("/v1/health/ready") + assert result.status_code in {200, 503}, result.text + return result.status_code == 200 + + await eventually(ready, max_wait=60, description="normal app readiness") + yield client + + +def _bearer(subject: str, org: str, namespaces: list[str], role: str) -> str: + from server.auth.jwt import mint_token, verify_token + from server.auth.reader_identity import reader_subject + + bearer = mint_token( + subject, + org, + namespaces, + [role], + is_service_account=False, + session_id=uuid.uuid4().hex, + ttl_seconds=1800, + ) + verified = verify_token(bearer) + assert verified.subject == subject + assert reader_subject(verified) == subject + return bearer + + +async def _post( + client: httpx.AsyncClient, bearer: str, op: str, body: dict[str, Any] +) -> httpx.Response: + return await client.post(f"/v1/{op}", json=body, headers={"Authorization": f"Bearer {bearer}"}) + + +@pytest.fixture +async def identities(native_app: httpx.AsyncClient, provider: dict[str, Any]) -> dict[str, Any]: + from server.auth.reader_identity import resolve_reader + from server.core.ch import get_client, set_query_tag + + set_query_tag("test.crawl_native_privacy.identities") + ch = get_client() + org, other_org = unique_org("crawl-native"), unique_org("crawl-other") + # Account login handles are distinct from gateway grant principals (user:*). + subjects = { + "admin": "registrant", + "a": "reader-a", + "b": "reader-b", + "c": "reader-c", + } + contacts = { + "admin": "registrant@example.test", + "a": provider["reader"], + "b": "reader-b@example.test", + "c": provider["outside_reader"], + } + # These actual rows represent already-verified upstream SSO contact evidence. + rows = [ + { + "org": org, + "namespace": f"identity-{name}", + "owner_email": subjects[name], + "contact_email": contact, + "password_hash": "", + "active": True, + } + for name, contact in contacts.items() + ] + rows.append( + { + "org": other_org, + "namespace": "identity-a", + "owner_email": subjects["a"], + "contact_email": contacts["a"], + "password_hash": "", + "active": True, + } + ) + await ch.insert("loom.namespaces", rows) + for row in rows: + identity = await resolve_reader(row["org"], row["owner_email"]) + assert identity is not None and identity.subject == row["owner_email"] + assert identity.emails == frozenset({row["contact_email"]}) + return {"org": org, "other_org": other_org, "subjects": subjects, "contacts": contacts} + + +@pytest.fixture +async def registered_crawl( + native_app: httpx.AsyncClient, identities: dict[str, Any], provider: dict[str, Any] +) -> None: + from server.core.ch import set_query_tag + + set_query_tag("test.crawl_native_privacy.registration") + org, subjects = identities["org"], identities["subjects"] + admin = _bearer(subjects["admin"], org, ["*"], "org_admin") + for namespace in ("col-crawl-documents", "col-other-documents"): + response = await _post( + native_app, admin, "collection.create", {"org": org, "namespace": namespace} + ) + assert response.status_code == 200, response.text + for who, namespace in ( + ("a", "col-crawl-documents"), + ("b", "col-crawl-documents"), + ("c", "col-other-documents"), + ): + response = await _post( + native_app, + admin, + "grants.create", + {"org": org, "namespace": namespace, "subject": subjects[who], "role": "viewer"}, + ) + assert response.status_code == 200, response.text + response = await _post( + native_app, + admin, + "knowledge.source.add", + { + "org": org, + "namespace": "col-crawl-documents", + "name": "private-documents", + "kind": "gdrive", + "credential_mode": "crawl", + "credential": provider["credential"], + "items": [{"type": "filter", "kinds": ["docs"]}], + "include": ["docs"], + "crawl_options": { + "directory_subject": provider["directory"], + "subjects": [{"type": "person", "email": provider["reader"]}], + }, + }, + ) + assert response.status_code == 200, response.text + response = await _post( + native_app, + admin, + "knowledge.source.sync_now", + {"org": org, "namespace": "col-crawl-documents", "name": "private-documents"}, + ) + assert response.status_code == 200, response.text + + +@pytest.fixture +async def published_crawl( + registered_crawl: None, identities: dict[str, Any], provider: dict[str, Any] +) -> None: + from server.clients import gdrive + from server.core.ch import get_client, set_query_tag + from server.knowledge.domain.sources import SYNC_WORKER_ACTOR + + set_query_tag("test.crawl_native_privacy.publication") + ch, org = get_client(), identities["org"] + + async def published() -> dict[str, Any] | None: + assert not provider["unexpected"], provider["unexpected"] + return await ch.query_one( + "SELECT namespace, content, access_json, actor FROM loom.documents FINAL " + "WHERE org = {org:String} AND actor = {actor:String} LIMIT 1", + params={"org": org, "actor": SYNC_WORKER_ACTOR}, + ) + + document = await eventually(published, max_wait=180, description="native crawl publication") + assert provider["canary"] in document["content"] + assert document["actor"] == SYNC_WORKER_ACTOR and document["namespace"].startswith("acl-") + acl = json.loads(document["access_json"]) + assert provider["reader"] in acl["emails"] + assert identities["contacts"]["b"] not in acl["emails"] + assert provider["outside_reader"] in acl["emails"] + assert provider["exported"] + assert provider["scopes"] == {s.value for s in (*gdrive.ReadScope, *gdrive.WriteScope)} + source = await ch.query_one( + "SELECT toString(source_id) AS source_id, crawl_generation " + "FROM loom.knowledge_sources FINAL WHERE org = {org:String} " + "AND namespace = {ns:String} AND name = {name:String} LIMIT 1", + params={"org": org, "ns": "col-crawl-documents", "name": "private-documents"}, + ) + assert source is not None and source["crawl_generation"] + + async def complete_listing() -> dict[str, Any] | None: + assert not provider["unexpected"], provider["unexpected"] + return await ch.query_one( + "SELECT crawl_generation, crawl_exclusion_generation " + "FROM loom.knowledge_source_listings FINAL WHERE org = {org:String} " + "AND namespace = {ns:String} AND source_id = {id:String} LIMIT 1", + params={"org": org, "ns": document["namespace"], "id": source["source_id"]}, + ) + + listing = await eventually(complete_listing, max_wait=30, description="complete native listing") + assert listing["crawl_generation"] == source["crawl_generation"] + assert listing["crawl_exclusion_generation"] + + +async def test_native_crawl_canary_obeys_document_acl_and_tenant_scope( + native_app: httpx.AsyncClient, + identities: dict[str, Any], + provider: dict[str, Any], + published_crawl: None, +) -> None: + from server.core.ch import set_query_tag + + set_query_tag("test.crawl_native_privacy.reads") + org, subjects = identities["org"], identities["subjects"] + query = { + "org": org, + "namespace": "col-crawl-documents", + "query": "quarterly renewal plan scope owners release readiness", + } + allowed = _bearer(subjects["a"], org, ["col-crawl-documents"], "viewer") + + async def admitted() -> bool: + assert not provider["unexpected"], provider["unexpected"] + result = await _post(native_app, allowed, "knowledge.search", query) + assert result.status_code == 200, result.text + return provider["canary"] in result.text + + assert await eventually(admitted, max_wait=30, description="authorized HTTP canary") + denied = _bearer(subjects["b"], org, ["col-crawl-documents"], "viewer") + response = await _post(native_app, denied, "knowledge.search", query) + assert response.status_code == 200, response.text + assert provider["canary"] not in response.text + # C has a document ACL entry but only another namespace's grant. A's other-org + # identity also has a matching contact, so the document ACL alone cannot deny it. + for bearer in ( + _bearer(subjects["c"], org, ["col-other-documents"], "viewer"), + _bearer(subjects["a"], identities["other_org"], ["*"], "viewer"), + ): + response = await _post(native_app, bearer, "knowledge.search", query) + assert response.status_code in {200, 403, 404}, response.text + assert provider["canary"] not in response.text diff --git a/tests/unit/snapshots/settings_defaults.json b/tests/unit/snapshots/settings_defaults.json index a44ad873f5..38d0ec39dc 100644 --- a/tests/unit/snapshots/settings_defaults.json +++ b/tests/unit/snapshots/settings_defaults.json @@ -573,6 +573,16 @@ "knowledge_connected_sources_max_per_namespace": 64, "knowledge_connected_sources_max_per_person": 8, "knowledge_connected_stop_timeout_s": 20.0, + "knowledge_crawl_directory_max_pages": 256, + "knowledge_crawl_directory_max_response_bytes": 1048576, + "knowledge_crawl_identity_capture_timeout_s": 30.0, + "knowledge_crawl_lease_ttl_s": 60.0, + "knowledge_crawl_lease_wait_s": 5.0, + "knowledge_crawl_max_files": 10000, + "knowledge_crawl_max_groups": 2000, + "knowledge_crawl_person_cleanup_timeout_s": 60.0, + "knowledge_crawl_registry_page_size": 256, + "knowledge_crawl_registry_scan_limit": 4096, "knowledge_gaps_enabled": false, "knowledge_gaps_interval_seconds": 86400, "knowledge_gaps_max_askers_per_group": 10000, diff --git a/tests/unit/test_cell_control_census.py b/tests/unit/test_cell_control_census.py index b80384b90e..9bc24069ca 100644 --- a/tests/unit/test_cell_control_census.py +++ b/tests/unit/test_cell_control_census.py @@ -181,6 +181,8 @@ def test_cell_control_migrations_are_classified(): ("copy_sql", "INSERT INTO", "loom.{dst}"): 1, ("apply", "DROP TABLE", "loom.{twin}"): 2, }, + # The token selects a source metadata column, not a cell control table. + "server/erasure/store/crawl_controls.py": {("controls", "UPDATE", "{field}"): 1}, "server/erasure/store/mutations.py": { ("run_mutation", "ALTER TABLE", "{table}"): 1, ("run_lightweight_delete", "DELETE FROM", "{table}"): 1, diff --git a/tests/unit/test_cell_scan_tables.py b/tests/unit/test_cell_scan_tables.py index c558ec5e27..5fd561106f 100644 --- a/tests/unit/test_cell_scan_tables.py +++ b/tests/unit/test_cell_scan_tables.py @@ -92,6 +92,8 @@ "loom.graph_projections", "loom.idempotency", "loom.knowledge_gaps", + "loom.knowledge_crawl_exclusion_keys", + "loom.knowledge_crawl_exclusions", "loom.knowledge_group_members", "loom.knowledge_ingest_cancels", "loom.knowledge_ingestions", diff --git a/tests/unit/test_claim_write_settlement.py b/tests/unit/test_claim_write_settlement.py new file mode 100644 index 0000000000..097f086fec --- /dev/null +++ b/tests/unit/test_claim_write_settlement.py @@ -0,0 +1,129 @@ +"""A cleanup claimant cannot pass a sent INSERT that may still append raw metadata.""" + +from __future__ import annotations + +import asyncio +from types import SimpleNamespace +from typing import Any + +import pytest + +from server.core import ch as ch_module +from server.core import leases, rmt_carry +from server.settings import settings +from tests.unit.clock_fake import ManualClock + +pytestmark = pytest.mark.asyncio + + +class Keeper: + """External claim acquisition/release and server-side expiry, not the claim protocol.""" + + def __init__(self, monkeypatch: pytest.MonkeyPatch) -> None: + self.clock = ManualClock(1000.0) + self.keys: dict[str, tuple[leases.Lease, float]] = {} + self.serial = 0 + monkeypatch.setattr(leases, "time", SimpleNamespace(monotonic=self.clock.monotonic)) + monkeypatch.setattr(rmt_carry, "time", SimpleNamespace(monotonic=self.clock.monotonic)) + monkeypatch.setattr(settings, "ch_timeout_write_s", 5) + monkeypatch.setattr(leases, "take", self.take) + + async def take(self, name: str, *, ttl_s: float, wait_s: float, ch: Any): + assert wait_s >= 0 + if name in self.keys and self.keys[name][1] >= self.clock(): + return None + self.serial += 1 + lease = leases.Lease(name, ttl_s=ttl_s, ch=self, holder=str(self.serial)) + lease._renewed = self.clock() + lease._sent = True + self.keys[name] = lease, self.clock() + ttl_s + return lease + + async def execute(self, sql: str, *, params: dict[str, Any]) -> None: + assert sql == leases.LOCK_RELEASE + name = params["name"] + current = self.keys.get(name) + if current is not None and current[0].holder == params["me"]: + del self.keys[name] + + def send(self, name: str) -> float: + deadline = rmt_carry.clock_send_deadline((self.clock(), self.clock())) + rmt_carry.note_send(name, deadline, ch_module.write_limit_s()) + return deadline + + +async def test_acknowledged_sends_release_without_waiting(monkeypatch): + keeper = Keeper(monkeypatch) + async with leases.holding("claim", ttl_s=60, settle_writes=True) as held: + assert held + keeper.send("write") + rmt_carry.note_ended("write") # A server answer, including its own error, ended it. + assert keeper.keys == {} and keeper.clock.sleeps == [] + + +async def test_cancelled_unanswered_send_keeps_keeper_key_until_its_safe_expiry(monkeypatch): + keeper = Keeper(monkeypatch) + with pytest.raises(asyncio.CancelledError): + async with leases.holding("claim", ttl_s=60, settle_writes=True): + deadline = keeper.send("write") + raise asyncio.CancelledError + async with leases.holding("claim", ttl_s=60, settle_writes=True) as held: + assert not held + assert deadline + rmt_carry.send_bound_s() - 15 <= keeper.keys["claim"][1] + keeper.clock.advance(61) + async with leases.holding("claim", ttl_s=60, settle_writes=True) as held: + assert held + assert keeper.keys == {} and keeper.clock.sleeps == [] + + +async def test_handled_lost_reply_cannot_turn_an_unsettled_claim_into_success(monkeypatch): + keeper = Keeper(monkeypatch) + with pytest.raises(leases.LeaseUnavailable, match="unanswered"): + async with leases.holding("claim", ttl_s=60, settle_writes=True): + keeper.send("lost-reply") + assert "claim" in keeper.keys + + +async def test_nested_claim_cannot_move_outer_send_deadline_or_release_its_unknown_send( + monkeypatch, +): + keeper = Keeper(monkeypatch) + with pytest.raises(asyncio.CancelledError): + async with leases.holding("outer", ttl_s=60, settle_writes=True): + outer = rmt_carry.clock_send_deadline((keeper.clock(), keeper.clock())) + keeper.clock.advance(20) + async with leases.holding("inner", ttl_s=60, settle_writes=True): + inner = keeper.send("late-write") + assert inner > outer + assert inner + 5 + 5 <= keeper.keys["outer"][1] + raise asyncio.CancelledError + assert set(keeper.keys) == {"outer", "inner"} + keeper.clock.advance(41) + async with leases.holding("outer", ttl_s=60, settle_writes=True) as held: + assert held + async with leases.holding("inner", ttl_s=60, settle_writes=True) as held: + assert not held + + +async def test_impossible_receiver_bound_refuses_before_body_without_retaining_empty_claim( + monkeypatch, +): + keeper = Keeper(monkeypatch) + with pytest.raises(leases.ClaimExpired): + async with leases.holding("claim", ttl_s=10, settle_writes=True): + pytest.fail("no work fits before the receiver's completion barrier") + assert keeper.keys == {} + + +async def test_default_write_timeout_still_leaves_a_full_claim_work_window(monkeypatch): + keeper = Keeper(monkeypatch) + monkeypatch.setattr(settings, "ch_timeout_write_s", 60) + assert ch_module.write_limit_s() == 60 and rmt_carry.send_bound_s() == 80 + async with leases.holding("claim", ttl_s=60, settle_writes=True) as held: + assert held and ch_module.write_limit_s() == 9 + assert rmt_carry.send_bound_s() == 29 + keeper.clock.advance(23) + deadline = keeper.send("near-work-end") + assert deadline + ch_module.write_limit_s() + 5 < keeper.keys["claim"][1] + rmt_carry.note_ended("near-work-end") + assert keeper.keys == {} and keeper.clock.sleeps == [] diff --git a/tests/unit/test_clickhouse_mutation_inventory.py b/tests/unit/test_clickhouse_mutation_inventory.py index 7bf18dcb36..cb5cd8429a 100644 --- a/tests/unit/test_clickhouse_mutation_inventory.py +++ b/tests/unit/test_clickhouse_mutation_inventory.py @@ -19,6 +19,13 @@ # the inventory honest without trying to parse Python string concatenation. _EXPECTED_MUTATIONS = Counter( { + # Reseal historical SCIM cleanup capsules under the identity lease, matching + # the exact SCIM row id and old ciphertext; synchronous residue checks keep + # the captured erasure identity unchanged before the old root is retired. + ("server/auth/person_erasure.py", "scim_users", "UPDATE"): 1, + # Root-secret rotation changes only exact old ciphertext, including raw + # versions; no late full-row insert can revive canonical-person metadata. + ("server/knowledge/store/sources.py", "knowledge_sources", "UPDATE"): 1, # The contradiction repair's edge close, end_edges, which ends a dropped fact's # edge with its triples (only when no live row keeps the pair), and the undo of # a close's own ends (reopen_closed_by). diff --git a/tests/unit/test_connected_account_restore.py b/tests/unit/test_connected_account_restore.py index 3a498f8066..5daa08c0fa 100644 --- a/tests/unit/test_connected_account_restore.py +++ b/tests/unit/test_connected_account_restore.py @@ -29,6 +29,9 @@ def __init__(self, monkeypatch, http): self.account_history = [self.rows[_KEY]] self.source_history = [self.source] monkeypatch.setattr(accounts.store, "_default_client", lambda: self) + from server.knowledge.app import crawl_restore + + monkeypatch.setattr(crawl_restore, "get_client", lambda: self) async def put(self, row, **kwargs): await super().put(row, **kwargs) @@ -43,6 +46,10 @@ async def erase(self, org, subjects, *, kind, **kwargs): ] async def query_all(self, query, params, **kwargs): + if query.startswith("SELECT DISTINCT org FROM (SELECT org FROM loom.knowledge_sources "): + assert "WHERE credential_mode = 'crawl' UNION ALL " in query + assert "FROM loom.knowledge_crawl_exclusion_keys)" in query + return [] # This connection-only world contains no crawl source or refusal key. if "FROM loom.connected_accounts" in query: assert "FINAL" not in query assert "credential_enc" not in query and "account_email_enc" not in query diff --git a/tests/unit/test_connected_source_workers.py b/tests/unit/test_connected_source_workers.py index c521ab2c8d..19b4e42cea 100644 --- a/tests/unit/test_connected_source_workers.py +++ b/tests/unit/test_connected_source_workers.py @@ -215,6 +215,8 @@ async def test_native_connected_notion_turn_publishes_real_rows_then_a_guarded_l "org": "team", "namespace": _MIRROR, "source_id": "source-id", + "crawl_exclusion_generation": "", + "crawl_generation": "", "name": "documents", } assert isinstance(listing["started_at"], datetime) diff --git a/tests/unit/test_crawl_approval_reseal.py b/tests/unit/test_crawl_approval_reseal.py new file mode 100644 index 0000000000..36f42260ef --- /dev/null +++ b/tests/unit/test_crawl_approval_reseal.py @@ -0,0 +1,146 @@ +"""Root rotation rewrites all historical approval envelopes without replacing authority.""" + +from __future__ import annotations + +import copy +import dataclasses +import json + +import pytest + +from server.auth import source_registration as registration +from server.auth.jwt import AuthorizationClaims +from server.core.secrets_at_rest import seal, unseal +from server.knowledge.app import crawl_reseal +from server.operations.base import Role +from server.settings import settings +from tests.unit.test_crawl_restore import _FIRST, _SECOND, _HistoryWorld + +pytestmark = pytest.mark.asyncio + + +def _approval(source_id: str, generation: str, *, org: str = "org") -> tuple[str, str]: + scope = registration.Scope(org, "namespace", source_id, generation, "f" * 64) + claims = AuthorizationClaims( + subject="registrant", + org=org, + namespaces=("namespace",), + roles=(Role.ORG_ADMIN,), + is_service_account=False, + issued_at=1, + expires_at=2, + creator="original-creator", + ) + plain = json.dumps( + { + "format": 1, + "scope": dataclasses.asdict(scope), + "registrant": "registrant", + "claims": dataclasses.asdict(claims), + }, + sort_keys=True, + ) + return seal(registration.PURPOSE, plain), plain + + +async def _history(monkeypatch): + world = _HistoryWorld(monkeypatch) + monkeypatch.setattr(crawl_reseal, "get_client", lambda: world) + monkeypatch.setattr(settings, "knowledge_crawl_registry_page_size", 1) + first, _ = _approval(_FIRST, "earlier-generation") + second, _ = _approval(_FIRST, "later-generation") + third, _ = _approval(_SECOND, "other-source-generation") + await world.seed(approval_enc=first) + await world.seed(approval_enc=first) + await world.seed(approval_enc=second) + await world.seed(name="secondary", source_id=_SECOND, approval_enc=third) + await world.seed(name="shared", credential_mode="shared", approval_enc="unrelated") + return world + + +def _rotate(monkeypatch): + monkeypatch.setattr(settings, "jwt_secret_previous", settings.jwt_secret) + monkeypatch.setattr( + settings, "jwt_secret", "replacement-root-sealing-secret-which-is-long-enough" + ) + + +async def test_approval_reseal_reaches_every_raw_version_and_retains_exact_source_identity( + monkeypatch, +): + world = await _history(monkeypatch) + before = copy.deepcopy(world.sources) + plaintext = [unseal(registration.PURPOSE, r["approval_enc"]) for r in before[:4]] + _rotate(monkeypatch) + assert await crawl_reseal.approvals(dry_run=False, ch=world) == (3, []) + monkeypatch.setattr(settings, "jwt_secret_previous", "") + for old, row, plain in zip(before[:4], world.sources[:4], plaintext, strict=True): + assert row["approval_enc"] != old["approval_enc"] + assert unseal(registration.PURPOSE, row["approval_enc"], warn=False) == plain + assert not unseal(registration.PURPOSE, old["approval_enc"], warn=False) + frame = json.loads(plain) + scope = registration.Scope(**frame["scope"]) + approval = registration.load(row["approval_enc"], scope, registrant="registrant") + assert approval.scope.source_id == row["source_id"] + assert approval.scope == scope and approval.registrant == frame["registrant"] + assert approval.claims.roles == (Role.ORG_ADMIN,) + assert ( + approval.claims.subject == "registrant" + and approval.claims.creator == "original-creator" + ) + assert approval.claims.expires_at == 2 + assert {k: v for k, v in row.items() if k != "approval_enc"} == { + k: v for k, v in old.items() if k != "approval_enc" + } + assert world.sources[-1] == before[-1] + assert len(world.page_reads) >= 3 and not world.retirements + + +async def test_approval_reseal_dry_run_counts_old_envelopes_without_writing_history(monkeypatch): + world = await _history(monkeypatch) + before = copy.deepcopy(world.sources) + _rotate(monkeypatch) + assert await crawl_reseal.approvals(dry_run=True, ch=world) == (3, []) + assert world.sources == before and not world.executed and not world.retirements + + +async def test_unreadable_approval_is_reported_without_resetting_its_source(monkeypatch): + world = _HistoryWorld(monkeypatch) + monkeypatch.setattr(crawl_reseal, "get_client", lambda: world) + await world.seed(approval_enc="unreadable-ciphertext") + before = copy.deepcopy(world.sources) + _rotate(monkeypatch) + assert await crawl_reseal.approvals(dry_run=False, ch=world) == (0, ["org/namespace/documents"]) + assert world.sources == before and not world.executed + + +async def test_approval_copied_from_another_scope_cannot_be_resealed_as_local_authority( + monkeypatch, +): + world = _HistoryWorld(monkeypatch) + monkeypatch.setattr(crawl_reseal, "get_client", lambda: world) + foreign, _ = _approval(_FIRST, "generation", org="another-org") + await world.seed(approval_enc=foreign) + before = copy.deepcopy(world.sources) + _rotate(monkeypatch) + with pytest.raises(registration.Refused): + await crawl_reseal.approvals(dry_run=False, ch=world) + assert world.sources == before and not world.executed + + +async def test_ignored_history_reseal_fails_its_raw_ciphertext_residue_check(monkeypatch): + world = await _history(monkeypatch) + before = copy.deepcopy(world.sources) + world.ignore_mutation = True + _rotate(monkeypatch) + with pytest.raises(RuntimeError, match="crawl_approval_reseal_residue"): + await crawl_reseal.approvals(dry_run=False, ch=world) + assert world.executed and world.sources == before + + +async def test_approval_reseal_refuses_another_target_without_touching_history(monkeypatch): + world = await _history(monkeypatch) + before = copy.deepcopy((world.sources, world.page_reads, world.executed)) + with pytest.raises(RuntimeError, match="crawl_reseal_target_mismatch"): + await crawl_reseal.approvals(dry_run=False, ch=object()) + assert (world.sources, world.page_reads, world.executed) == before diff --git a/tests/unit/test_crawl_authority.py b/tests/unit/test_crawl_authority.py new file mode 100644 index 0000000000..36e2d003e9 --- /dev/null +++ b/tests/unit/test_crawl_authority.py @@ -0,0 +1,412 @@ +"""Native Drive calls keep source, signer and provider-person authority across awaits.""" + +from __future__ import annotations + +import asyncio +import dataclasses +import hashlib +import json +import time +from typing import Any +from urllib.parse import parse_qs + +import httpx +import jwt +import pytest + +from server.auth import scim +from server.auth import source_registration as registration +from server.auth.jwt import verify_token +from server.clients import gdrive +from server.clients import google_workspace as google +from server.clients._vendor import CredentialRefused, VendorError +from server.core.errors import ValidationError +from server.core.secrets_at_rest import seal +from server.knowledge.app import crawl_authority, crawl_exclusions, crawl_expansion +from server.knowledge.app import sources as source_ops +from server.knowledge.domain.source_crawl import Person +from server.knowledge.domain.sources import KnowledgeSource +from server.knowledge.store import crawl_exclusions as exclusion_store +from server.operations.base import Role +from server.settings import settings +from tests.unit.principals import acting_as +from tests.unit.test_connected_source_authority import _World as _AuthorityWorld +from tests.unit.test_crawl_registration import _token +from tests.unit.test_crawl_signing_authority import key_view as key_view +from tests.unit.test_google_workspace import account as account + +pytestmark = [pytest.mark.asyncio, pytest.mark.claim_cap] +_BOUNDS = crawl_authority.Bounds(crawl_expansion.Limits(16384, 10, 10), 60, 0) +_NATIVE_SCIM_READ = scim.deactivated + + +class _World(_AuthorityWorld): + def __init__(self, monkeypatch: pytest.MonkeyPatch, http: httpx.AsyncClient, account: Any): + super().__init__(monkeypatch, http) + self.grant = Role.ADMIN + self.exclusion_keys: dict[str, dict[str, Any]] = {} + self.exclusion_rows: dict[tuple[str, str], bool] = {} + self.provider_calls: list[httpx.Request] = [] + self.claims_sent: list[dict[str, Any]] = [] + self.storage_version = 0 + self.person_erasure_pending = False + self.ended_people: set[str] = set() + self.account = account + self.customer = "workspace" + self.person_email = "one@example.com" + self.after_grant: Any = lambda: None + self.after_native: Any = lambda: None + monkeypatch.setattr(google, "get_client", lambda: http) + monkeypatch.setattr(gdrive, "get_client", lambda: http) + monkeypatch.setattr(crawl_authority, "get_client", lambda: self) + monkeypatch.setattr(scim, "deactivated", _NATIVE_SCIM_READ) + monkeypatch.setattr(scim, "get_client", lambda: self) + service = account[0] + self.credential = json.dumps( + { + "type": "service_account", + "client_email": service.client_email, + "private_key": service.private_key, + } + ) + self.source = KnowledgeSource( + org="team", + namespace="shared-space", + name="documents", + kind="gdrive", + source_id="crawl-source", + config={"include": ["docs"]}, + items=[{"type": "file", "id": "file"}], + access="mirrored", + credential_mode="crawl", + secret_enc=seal(source_ops._SECRET_PURPOSE, self.credential), + crawl_options={ + "directory_subject": "directory@example.com", + "subjects": [{"type": "person", "email": "one@example.com"}], + }, + crawl_generation="crawl-generation", + crawl_customer_id="workspace", + crawl_credential_hash=hashlib.sha256(self.credential.encode()).hexdigest(), + created_by="audit-attribution-is-not-authority", + ) + + async def query_one(self, query: str, params: dict[str, Any] | None = None, **kwargs: Any): + if query.startswith("SELECT 1 AS pending FROM loom.scim_users FINAL"): + assert params is None + return {"pending": 1} if self.person_erasure_pending else None + assert params is not None + if query.startswith("SELECT 1 AS x FROM loom.scim_users FINAL"): + assert set(params) == {"id"} + ended = self.ended_people | ({"person@example.com"} if self.scim_ended else set()) + return {"x": 1} if params["id"] in {scim.user_id(email) for email in ended} else None + if query.startswith("SELECT greatest(") and any( + f"FROM {table} " in query for table in (exclusion_store.KEYS, exclusion_store.REFUSALS) + ): + self.storage_version += 1 + return {"v": self.storage_version} + if f"FROM {exclusion_store.KEYS} FINAL" in query: + return self.exclusion_keys.get(params["org"]) + if f"FROM {exclusion_store.REFUSALS} FINAL" in query: + return next( + ({"digest": d} for org, d in self.exclusion_rows if org == params["org"]), None + ) + return await super().query_one(query, params, **kwargs) + + async def query_all(self, query: str, params: dict[str, Any], **kwargs: Any): + if query.startswith("SELECT digest, excluded FROM loom.knowledge_crawl_exclusions FINAL"): + return [ + {"digest": d, "excluded": self.exclusion_rows[params["org"], d]} + for d in params["digests"] + if (params["org"], d) in self.exclusion_rows + ] + if f"FROM {exclusion_store.REFUSALS} FINAL" in query: + return [ + {"digest": d} + for d in params["digests"] + if self.exclusion_rows.get((params["org"], d)) + ] + return await super().query_all(query, params, **kwargs) + + async def insert(self, table: str, rows: list[dict[str, Any]]) -> None: + assert self.holders.get(asyncio.current_task(), 0) + for row in rows: + if table == exclusion_store.KEYS: + self.exclusion_keys[row["org"]] = dict(row) + else: + assert table == exclusion_store.REFUSALS + self.exclusion_rows[row["org"], row["digest"]] = row["excluded"] + + async def setup(self) -> None: + assert self.source is not None + auth = verify_token(_token()) + with acting_as(auth): + registrant, approval = await registration.capture( + auth, crawl_authority.scope(self.source) + ) + self.source = dataclasses.replace( + self.source, registrant_subject=registrant, approval_enc=approval + ) + permit = await registration.current_permit(crawl_authority.scope(self.source)) + await crawl_exclusions.initialize("team", authority=permit, ttl_s=60, wait_s=0, ch=self) + + async def handle(self, request: httpx.Request) -> httpx.Response: + self.provider_calls.append(request) + if request.url.host == "oauth2.googleapis.com": + form = parse_qs(request.content.decode()) + assert form["grant_type"] == ["urn:ietf:params:oauth:grant-type:jwt-bearer"] + claims = jwt.decode( + form["assertion"][0], + self.account[1], + algorithms=["RS256"], + audience="https://oauth2.googleapis.com/token", + ) + self.claims_sent.append(claims) + if claims["scope"] in {write.value for write in gdrive.WriteScope}: + assert claims["sub"] == "directory@example.com" + return httpx.Response(400, json={"error": "unauthorized_client"}) + drive = claims["sub"] == "one@example.com" + assert not drive or claims["scope"] == gdrive.ReadScope.DRIVE.value + assert claims["sub"] == ("one@example.com" if drive else "directory@example.com") + if drive: + self.after_grant() + return httpx.Response( + 200, + json={ + "access_token": "private-person-access" + if drive + else "private-directory-access", + "token_type": "Bearer", + "expires_in": 60, + }, + ) + if request.url.host == "admin.googleapis.com": + assert request.headers["authorization"] == "Bearer private-directory-access" + key = request.url.path.rsplit("/", 1)[1] + directory = key == "directory@example.com" + assert directory or key in {"one@example.com", "provider-one"} + return httpx.Response( + 200, + json={ + "id": "directory-user" if directory else "provider-one", + "primaryEmail": "directory@example.com" if directory else self.person_email, + "aliases": [], + "customerId": self.customer, + "suspended": False, + "archived": False, + }, + ) + assert ( + request.url.host == "www.googleapis.com" and request.url.path == "/drive/v3/files/file" + ) + assert request.headers["authorization"] == "Bearer private-person-access" + self.native_requests.append(request) + self.after_native() + return httpx.Response(200, json={"id": "file", "name": "private-document"}) + + +@pytest.fixture +async def crawl_world(monkeypatch: pytest.MonkeyPatch, account: Any, key_view: list[str]): + async def handle(request: httpx.Request) -> httpx.Response: + return await state.handle(request) + + async with httpx.AsyncClient(transport=httpx.MockTransport(handle)) as http: + state = _World(monkeypatch, http, account) + await state.setup() + yield state + + +async def _ready(state: _World): + assert state.source is not None + authority = await crawl_authority.resolve( + state.source, deadline=time.monotonic() + 10, bounds=_BOUNDS + ) + expanded = await authority.prepare() + (person,) = expanded.people + return authority, person + + +async def test_real_checked_drive_uses_only_the_expanded_provider_person( + crawl_world: _World, +) -> None: + authority, person = await _ready(crawl_world) + access = await authority.drive(person) + assert await gdrive.file_meta(access, "file") == {"id": "file", "name": "private-document"} + assert len(crawl_world.native_requests) == 1 + assert {c["sub"] for c in crawl_world.claims_sent} == { + "directory@example.com", + "one@example.com", + } + assert all( + c["sub"] not in {"person", "person@example.com", crawl_world.source.created_by} + for c in crawl_world.claims_sent + ) + with pytest.raises(TypeError): + str(access) + + +async def test_unexpanded_or_email_shaped_loom_subject_cannot_select_delegation( + crawl_world: _World, +) -> None: + authority, _person = await _ready(crawl_world) + before = len(crawl_world.provider_calls) + with pytest.raises(CredentialRefused): + await authority.drive(Person("workspace", "person", "person@example.com", ())) + assert len(crawl_world.provider_calls) == before and not crawl_world.native_requests + + +async def test_provider_address_rebind_during_grant_stops_before_native_drive( + crawl_world: _World, +) -> None: + authority, person = await _ready(crawl_world) + crawl_world.after_grant = lambda: setattr(crawl_world, "person_email", "renamed@example.com") + with pytest.raises(CredentialRefused, match="person changed"): + await authority.drive(person) + assert not crawl_world.native_requests + + +@pytest.mark.parametrize("loss", ["grant", "registration", "exclusion"]) +async def test_native_response_never_returns_after_current_authority_loss( + crawl_world: _World, loss: str +) -> None: + authority, person = await _ready(crawl_world) + + def changed() -> None: + if loss == "grant": + crawl_world.grant = Role.WRITER + elif loss == "registration": + crawl_world.source = dataclasses.replace( + crawl_world.source, crawl_generation="replacement" + ) + else: + crawl_world.fences[crawl_exclusions._fence("team")] = "blocked:another-erasure" + + crawl_world.after_native = changed + with pytest.raises((PermissionError, VendorError)): + await gdrive.file_meta(await authority.drive(person), "file") + assert len(crawl_world.native_requests) == 1 + + +async def test_non_provider_wait_is_checked_before_result_delivery(crawl_world: _World) -> None: + authority, person = await _ready(crawl_world) + entered = [] + + async def model_wait(_access: Any) -> str: + entered.append(True) + crawl_world.grant = Role.WRITER + return "private-model-result" + + with pytest.raises(PermissionError): + await authority.run(person, model_wait) + assert entered and not crawl_world.native_requests + + +async def test_modified_selection_without_new_approval_makes_zero_provider_calls( + crawl_world: _World, +) -> None: + assert crawl_world.source is not None + crawl_world.source = dataclasses.replace( + crawl_world.source, items=[{"type": "file", "id": "other-file"}] + ) + with pytest.raises(registration.Refused): + await crawl_authority.resolve( + crawl_world.source, deadline=time.monotonic() + 10, bounds=_BOUNDS + ) + assert not crawl_world.provider_calls + + +async def test_different_sealed_key_cannot_inherit_the_approved_key_hash( + crawl_world: _World, +) -> None: + assert crawl_world.source is not None + crawl_world.source = dataclasses.replace( + crawl_world.source, secret_enc=seal(source_ops._SECRET_PURPOSE, "different-secret") + ) + with pytest.raises(CredentialRefused): + await crawl_authority.resolve( + crawl_world.source, deadline=time.monotonic() + 10, bounds=_BOUNDS + ) + assert not crawl_world.provider_calls + + +async def test_workspace_change_is_refused_before_any_drive_grant(crawl_world: _World) -> None: + crawl_world.customer = "different-workspace" + with pytest.raises(CredentialRefused, match="workspace changed"): + await _ready(crawl_world) + assert all(c["scope"] != gdrive.ReadScope.DRIVE.value for c in crawl_world.claims_sent) + assert not crawl_world.native_requests + + +async def test_ordinary_credential_selection_never_falls_back_to_a_crawl_key( + crawl_world: _World, +) -> None: + assert crawl_world.source is not None + assert source_ops.credential_of(crawl_world.source) == "" + with pytest.raises(ValidationError): + await source_ops.credential_for(crawl_world.source, deadline=time.monotonic() + 10) + assert not crawl_world.provider_calls + + +async def test_unfinished_person_erasure_refuses_before_unsealing_or_provider_work( + crawl_world: _World, +) -> None: + from server.core.errors import BackendUnavailableError + + crawl_world.person_erasure_pending = True + assert crawl_world.source is not None + # Even an unusable provider key is never reached before the pending-person gate. + crawl_world.source = dataclasses.replace(crawl_world.source, secret_enc="unreadable-key") + with pytest.raises(BackendUnavailableError, match="canonical-person cleanup"): + await crawl_authority.resolve( + crawl_world.source, deadline=time.monotonic() + 10, bounds=_BOUNDS + ) + assert not crawl_world.provider_calls and not crawl_world.native_requests + + +async def test_retained_native_checkout_stops_when_target_person_is_deactivated( + crawl_world: _World, +) -> None: + authority, person = await _ready(crawl_world) + access = await authority.drive(person) + before = len(crawl_world.provider_calls) + crawl_world.ended_people.add(person.email) + with pytest.raises(CredentialRefused, match="person unavailable"): + await gdrive.file_meta(access, "file") + assert len(crawl_world.provider_calls) == before and not crawl_world.native_requests + + +async def test_configured_crawl_lease_ttl_reaches_the_real_publication_guard( + crawl_world: _World, + monkeypatch: pytest.MonkeyPatch, +) -> None: + from contextlib import asynccontextmanager + + from server.core import leases + + monkeypatch.setattr(settings, "knowledge_crawl_lease_ttl_s", 17) + original = leases.cluster_lease + acquired = [] + + @asynccontextmanager + async def record(key: str, **kwargs: Any): + acquired.append((key, kwargs["ttl_s"])) + async with original(key, **kwargs): + yield + + @asynccontextmanager + async def holding(key: str, **kwargs: Any): + assert key == "knowledge-source:team:shared-space" + async with crawl_world.lease(key, **kwargs): + yield True + + monkeypatch.setattr(leases, "cluster_lease", record) + monkeypatch.setattr(leases, "holding", holding) + assert crawl_world.source is not None + authority = await crawl_authority.resolve( + crawl_world.source, + deadline=time.monotonic() + 10, + bounds=crawl_authority.configured_bounds(), + ) + async with authority.publication_guard(): + assert crawl_world.holders.get(asyncio.current_task(), 0) == 2 + assert (crawl_exclusions._fence("team"), 17) in acquired + assert not crawl_world.holders.get(asyncio.current_task(), 0) diff --git a/tests/unit/test_crawl_composed_flow.py b/tests/unit/test_crawl_composed_flow.py new file mode 100644 index 0000000000..014990170f --- /dev/null +++ b/tests/unit/test_crawl_composed_flow.py @@ -0,0 +1,423 @@ +"""Real registration, native worker publication and indexed authority share one fence.""" + +from __future__ import annotations + +import asyncio +import dataclasses +import time +from typing import Any + +import httpx +import pytest + +from server.auth.jwt import verify_token +from server.clients import gdrive +from server.core import instances, request_work +from server.core.errors import ValidationError +from server.knowledge.api.models_sources import KnowledgeSourceAddInput, KnowledgeSourceUpdateInput +from server.knowledge.app import ( + crawl_exclusions, + crawl_sources, + mirrors, + private_sync, +) +from server.knowledge.app import source_access as reads +from server.knowledge.app import source_sync as sync +from server.knowledge.app import sources as ops +from server.knowledge.domain.access_list import from_json +from server.knowledge.domain.sources import SYNC_WORKER_ACTOR, mirror_uri +from server.knowledge.store import crawl_publication, documents +from server.knowledge.store import mirrors as mirror_store +from server.knowledge.store import sources as source_store +from server.memory.domain.provenance import pipeline_actor +from server.operations.base import Role +from tests.unit.principals import acting_as +from tests.unit.test_connected_source_publication import _MIRROR +from tests.unit.test_connected_source_publication import _World as _PublicationWorld +from tests.unit.test_crawl_authority import _World as _CrawlWorld +from tests.unit.test_crawl_registration import _token +from tests.unit.test_crawl_signing_authority import key_view as key_view +from tests.unit.test_crawl_worker_observation import _file +from tests.unit.test_google_workspace import account as account + +pytestmark = [pytest.mark.asyncio, pytest.mark.claim_cap] + + +class _World(_CrawlWorld, _PublicationWorld): + """Reuse real authority and upsert fixtures, extending only SQL and provider I/O.""" + + def __init__(self, monkeypatch: pytest.MonkeyPatch, http: httpx.AsyncClient, account: Any): + super().__init__(monkeypatch, http, account) + self.source = None + self.listings: list[dict[str, Any]] = [] + self.states: list[dict[str, Any]] = [] + self.lose_during_embedding = False + self.lose_during_proof = False + self.unhandled_reads: list[tuple[str, tuple[str, ...]]] = [] + monkeypatch.setattr(crawl_sources, "get_client", lambda: self) + monkeypatch.setattr(crawl_exclusions, "get_client", lambda: self) + monkeypatch.setattr(instances, "read_live", self.read_live) + monkeypatch.setattr(mirrors, "mirrors_of", self.mirror_rows) + monkeypatch.setattr(mirror_store, "_default_client", lambda: self) + monkeypatch.setattr(crawl_publication, "_default_client", lambda: self) + monkeypatch.setattr(reads.store, "_default_client", lambda: self) + monkeypatch.setattr(reads.mirrors_repo, "mirrors_of", self.mirror_rows) + monkeypatch.setattr(reads.pg, "enabled", lambda: False) + monkeypatch.setattr(source_store, "every_copy", self.every_copy) + monkeypatch.setattr(source_store, "read_states", self.read_states) + monkeypatch.setattr(documents, "publications_owed", self.publications_owed) + monkeypatch.setattr(private_sync, "write_private_cost_event_for", self.cost_event) + + async def read_live(self, ch: Any, *, all_deployments: bool): + assert ch is self and all_deployments + return [{"schema_version": 114}] + + async def cost_event(self, **values: Any) -> None: + assert values["org"] == "team" and values["namespace"] == _MIRROR + + def assert_storage_leases(self) -> None: + assert self.held.get(asyncio.current_task(), []) == [ + crawl_exclusions._fence("team"), + "knowledge-source:team:shared-space", + ] + + async def no_prior_document(self, org: str, namespace: str, uri: str, **kwargs: Any): + assert (org, namespace) == ("team", _MIRROR) + assert set(kwargs) == {"include_superseded"} + return next((row for row in self.document_writes if row["source_uri"] == uri), None) + + async def every_copy(self, org: str, mirror: str, tag: str): + assert self.source is not None + assert (org, mirror, tag) == ("team", _MIRROR, "source-id:" + self.source.source_id) + return [] + + async def read_states(self, org: str, namespace: str, names: list[str]): + assert self.source is not None and org == "team" + assert (namespace, names) in ( + (_MIRROR, [self.source.source_id]), + (self.source.namespace, [self.source.name]), + ) + found = {} + for row in self.states: + if row["namespace"] == namespace: + for name in names: + if row["state_key"] == source_store.state_key(name): + found[name] = {"state_value": row["state_value"], "detail": row["detail"]} + return found + + async def publications_owed(self, org: str, mirror: str, suffix: str) -> bool: + assert self.source is not None + assert (org, mirror, suffix) == ("team", _MIRROR, "#" + self.source.source_id) + return False + + async def mirror_rows(self, org: str, namespaces: list[str], **kwargs: Any): + assert org == "team" and kwargs == {"live_only": True} + return {_MIRROR: "shared-space"} if "shared-space" in namespaces else {} + + async def query_one(self, query: str, params: dict[str, Any] | None = None, **kwargs: Any): + if query.startswith("SELECT count() AS n FROM loom.knowledge_sources FINAL"): + assert params == {"org": "team", "ns": "shared-space", "connected": False, "owner": ""} + return {"n": int(self.source is not None and self.source.active)} + if query.startswith("SELECT count() AS n FROM loom.documents FINAL"): + assert params is not None and params["org"] == "team" + return {"n": 0} + return await super().query_one(query, params, **kwargs) + + async def query_all(self, query: str, params: dict[str, Any], **kwargs: Any): + if "FROM loom.knowledge_group_members FINAL" in query: + assert params["org"] == "team" + return [] + if query.startswith("SELECT namespace, name, credential_mode"): + assert params == {"org": "team", "names": ["shared-space"]} + return ( + [] + if self.source is None + else [ + { + "namespace": self.source.namespace, + "name": self.source.name, + "credential_mode": self.source.credential_mode, + "source_id": self.source.source_id, + } + ] + ) + if "FROM loom.knowledge_source_listings FINAL" in query: + assert params["org"] == "team" + return [ + row + for row in self.listings + if (row["namespace"], row["source_id"]) in params["pairs"] + ] + if "FROM loom.documents FINAL" in query: + assert params["org"] == "team" + return [ + row + for row in self.document_writes + if (row["namespace"], row["source_uri"]) in params["pairs"] + ] + try: + return await super().query_all(query, params, **kwargs) + except AssertionError: + # Preserve fail-closed behavior; surface only the unexpected fake-I/O + # projection and parameter names, never substitute admission evidence. + self.unhandled_reads.append((" ".join(query.split()), tuple(sorted(params)))) + raise + + async def insert(self, table: str, rows: list[dict[str, Any]]) -> None: + if table == source_store.TABLE: + self.assert_storage_leases() + [row] = rows + self.source = source_store._source(row) + self.request = self.request.model_copy( + update={"source_uri": mirror_uri(_file()["webViewLink"], self.source.source_id)} + ) + elif table == source_store.STATE_TABLE: + self.assert_storage_leases() + self.states.extend(dict(row) for row in rows) + elif table == mirror_store.LISTINGS: + self.assert_storage_leases() + self.listings = [dict(row) for row in rows] + else: + await super().insert(table, rows) + + async def handle(self, request: httpx.Request) -> httpx.Response: + if request.url.host == "www.googleapis.com": + assert not self.holders.get(asyncio.current_task(), 0) + assert request.headers["authorization"] == "Bearer private-person-access" + self.native_requests.append(request) + if request.url.path == "/drive/v3/files/file/export": + return httpx.Response( + 200, text="# Notes\n\nThe delegated source publishes this revision." + ) + file = _file() + file["permissions"] = [ + {"type": "user", "role": "reader", "emailAddress": "person@example.com"} + ] + if request.url.path == "/drive/v3/files": + return httpx.Response(200, json={"files": [file], "incompleteSearch": False}) + assert request.url.path == "/drive/v3/files/file" + return httpx.Response(200, json=file) + response = await super().handle(request) + if self.lose_during_proof and self.claims_sent: + self.grant = Role.VIEWER + return response + + async def embed_many(self, texts: list[str]) -> list[list[float]]: + result = await super().embed_many(texts) + if self.lose_during_embedding: + self.grant = Role.VIEWER + return result + + +@pytest.fixture +async def world(monkeypatch: pytest.MonkeyPatch, account: Any, key_view: list[str]): + async def provider(request: httpx.Request) -> httpx.Response: + return await state.handle(request) + + marker = request_work._current.set(None) + request_work.begin() + try: + async with httpx.AsyncClient(transport=httpx.MockTransport(provider)) as http: + state = _World(monkeypatch, http, account) + yield state + finally: + request_work._current.reset(marker) + + +def _input(world: _World) -> KnowledgeSourceAddInput: + return KnowledgeSourceAddInput( + org="team", + namespace="shared-space", + name="documents", + kind="gdrive", + credential_mode="crawl", + credential=world.credential, + items=[{"type": "filter", "kinds": ["docs"]}], + include=["docs"], + crawl_options={ + "directory_subject": "directory@example.com", + "subjects": [{"type": "person", "email": "one@example.com"}], + }, + ) + + +async def _register(world: _World) -> None: + with acting_as(verify_token(_token())): + result = await ops.handle_knowledge_source_add(_input(world)) + assert result.name == "documents" and not result.dry_run + assert world.source is not None and world.source.crawl_customer_id == "workspace" + + +async def _turn(world: _World) -> sync.TurnResult: + assert world.source is not None + now = time.monotonic() + run = sync.Pass(sync.PassBudget(now, now + 30), 10) + with acting_as(None), pipeline_actor(SYNC_WORKER_ACTOR), private_sync.scope(): + return await sync._turn_of(world.source)(world.source, {}, run) + + +async def test_registration_native_publication_and_indexed_reader_share_exact_generation( + world: _World, +) -> None: + await _register(world) + assert {claim["scope"] for claim in world.claims_sent} == { + scope.value for scope in (*gdrive.ReadScope, *gdrive.WriteScope) + } + turn = await _turn(world) + assert turn.status == "ok" and turn.complete and turn.changed == 1 + assert len(world.document_writes) == len(world.chunk_writes) == len(world.listings) == 1 + [document] = world.document_writes + assert document["actor"] == SYNC_WORKER_ACTOR + assert from_json(document["access_json"]).owner == "" + assert "person@example.com" in from_json(document["access_json"]).emails + assert world.source is not None + assert world.listings[0]["crawl_generation"] == world.source.crawl_generation + candidate = {key: document[key] for key in ("namespace", "source_uri", "revision")} + candidate["doc_version"] = document["version"] + before = len(world.provider_calls), len(world.native_requests) + with acting_as(verify_token(_token())): + plan = await reads.prepare("team", ["shared-space"]) + assert plan.sources == {(_MIRROR, world.source.source_id): "shared-space"}, ( + world.unhandled_reads, + world.listings, + ) + checked = await plan.check([candidate], chunks=True) + assert checked, world.unhandled_reads + [admitted] = checked + assert admitted.document == document and plan.current(admitted) + assert (len(world.provider_calls), len(world.native_requests)) == before + + +async def test_current_administrator_loss_during_registration_never_publishes_source( + world: _World, +) -> None: + world.lose_during_proof = True + with acting_as(verify_token(_token())), pytest.raises((ValidationError, PermissionError)): + await ops.handle_knowledge_source_add(_input(world)) + assert world.provider_calls and world.source is None and not world.document_writes + + +async def test_actual_embedding_wait_cannot_publish_after_registrant_loses_namespace_admin( + world: _World, +) -> None: + await _register(world) + world.lose_during_embedding = True + with pytest.raises(PermissionError): + await _turn(world) + assert ( + world.embedded + and not world.document_writes + and not world.chunk_writes + and not world.listings + ) + + +@pytest.mark.parametrize("change", ["pending", "epoch", "generation"]) +async def test_indexed_evidence_is_removed_after_person_or_registration_fence_changes( + world: _World, change: str +) -> None: + await _register(world) + assert (await _turn(world)).status == "ok" + [document] = world.document_writes + candidate = {key: document[key] for key in ("namespace", "source_uri", "revision")} + candidate["doc_version"] = document["version"] + with acting_as(verify_token(_token())): + plan = await reads.prepare("team", ["shared-space"]) + assert plan.sources == {(_MIRROR, world.source.source_id): "shared-space"}, ( + world.unhandled_reads, + world.listings, + ) + checked = await plan.check([candidate], chunks=True) + assert checked, world.unhandled_reads + [admitted] = checked + if change == "pending": + world.person_erasure_pending = True + elif change == "epoch": + world.fences[crawl_exclusions._fence("team")] = "blocked:person-cleanup" + else: + assert world.source is not None + world.source = dataclasses.replace(world.source, crawl_generation="replaced") + assert await plan.check([candidate], chunks=True) == [] + assert not plan.current(admitted) + + +async def test_reapproved_generation_cannot_reuse_an_older_completed_listing(world: _World) -> None: + await _register(world) + assert (await _turn(world)).status == "ok" + [document] = world.document_writes + candidate = {key: document[key] for key in ("namespace", "source_uri", "revision")} + candidate["doc_version"] = document["version"] + assert world.source is not None + before = world.source.crawl_generation + with acting_as(verify_token(_token())): + await ops.handle_knowledge_source_update( + KnowledgeSourceUpdateInput( + org="team", + namespace="shared-space", + name="documents", + include=["docs", "pdf"], + ) + ) + assert world.source.crawl_generation != before + plan = await reads.prepare("team", ["shared-space"]) + assert plan.sources == {} and await plan.check([candidate], chunks=True) == [] + + +async def test_reactivation_does_not_readmit_until_an_explicit_verified_update( + world: _World, +) -> None: + from server.knowledge.domain.source_crawl import Person + + await _register(world) + index = await crawl_exclusions.Index.open("team", ch=world) + person = Person("workspace", "provider-one", "one@example.com", ()) + for digest in index.person_digests(person): + world.exclusion_rows["team", digest] = True + assert await index.contains(person) + before = await index.version() + with acting_as(verify_token(_token())): + ordinary = await ops.handle_knowledge_source_update( + KnowledgeSourceUpdateInput( + org="team", + namespace="shared-space", + name="documents", + include=["docs", "pdf"], + ) + ) + assert ordinary.crawl_proof is not None and ordinary.crawl_proof.members == 0 + assert await index.contains( + person + ) # Editing the selectors or signing in is no readmission. + explicit = await ops.handle_knowledge_source_update( + KnowledgeSourceUpdateInput( + org="team", + namespace="shared-space", + name="documents", + crawl_readmit=["one@example.com"], + ) + ) + current = await crawl_exclusions.Index.open("team", ch=world) + assert await current.version() != before and not await current.contains(person) + assert explicit.crawl_proof is not None and explicit.crawl_proof.members == 1 + assert not world.listings and not world.document_writes # No old content was republished. + assert "one@example.com" not in explicit.crawl_proof.model_dump_json() + assert set(explicit.crawl_proof.read_scopes) == {scope.value for scope in gdrive.ReadScope} + + +async def test_still_deactivated_person_cannot_be_explicitly_readmitted(world: _World) -> None: + await _register(world) + before = world.source + index = await crawl_exclusions.Index.open("team", ch=world) + epoch = await index.version() + world.ended_people.add("one@example.com") + with acting_as(verify_token(_token())), pytest.raises(ValidationError): + await ops.handle_knowledge_source_update( + KnowledgeSourceUpdateInput( + org="team", + namespace="shared-space", + name="documents", + crawl_readmit=["one@example.com"], + ) + ) + assert world.source == before and await index.version() == epoch + assert not world.native_requests and not world.document_writes and not world.listings diff --git a/tests/unit/test_crawl_delegation.py b/tests/unit/test_crawl_delegation.py new file mode 100644 index 0000000000..a338c0c81f --- /dev/null +++ b/tests/unit/test_crawl_delegation.py @@ -0,0 +1,173 @@ +"""Read admission requires each distinct write grant's explicit denial.""" + +from __future__ import annotations + +import time +from typing import Any +from urllib.parse import parse_qs + +import httpx +import jwt +import pytest + +from server.clients import gdrive +from server.clients import google_workspace as google +from server.clients._vendor import CredentialRefused, TooLarge, VendorError +from server.knowledge.app import crawl_delegation +from tests.unit.test_google_workspace import _Body, _http +from tests.unit.test_google_workspace import account as account + +pytestmark = pytest.mark.asyncio +_SUBJECT = "directory@example.com" + + +class _Standing: + def __init__(self) -> None: + self.live = True + + async def validate(self) -> None: + if not self.live: + raise PermissionError("source stopped") + + +def _claims(request: httpx.Request, account: Any) -> dict[str, Any]: + assert str(request.url) == "https://oauth2.googleapis.com/token" + assert not request.headers.get("authorization") + form = parse_qs(request.content.decode()) + assert set(form) == {"grant_type", "assertion"} + assert form["grant_type"] == ["urn:ietf:params:oauth:grant-type:jwt-bearer"] + claims = jwt.decode( + form["assertion"][0], account[1], algorithms=["RS256"], audience=str(request.url) + ) + assert set(claims) == {"iss", "sub", "scope", "aud", "iat", "exp"} + assert claims["iss"] == account[0].client_email and claims["sub"] == _SUBJECT + assert claims["exp"] - claims["iat"] == 600 + return claims + + +async def test_all_three_reads_and_four_distinct_write_denials_are_required( + monkeypatch: pytest.MonkeyPatch, account: Any +) -> None: + scopes: list[str] = [] + + def response(request: httpx.Request) -> httpx.Response: + scope = _claims(request, account)["scope"] + scopes.append(scope) + if scope in {read.value for read in gdrive.ReadScope}: + return httpx.Response( + 200, + json={"access_token": "private", "token_type": "Bearer", "expires_in": 60}, + ) + return httpx.Response(400, json={"error": "unauthorized_client"}) + + async with _http(monkeypatch, response) as calls: + await crawl_delegation.require_read_only( + account[0], + subject=_SUBJECT, + standing=_Standing(), + deadline=time.monotonic() + 10, + max_bytes=4096, + ) + assert len(calls) == 7 + assert scopes == [*(s.value for s in gdrive.ReadScope), *(s.value for s in gdrive.WriteScope)] + assert gdrive.WriteScope.DIRECTORY_GROUPS.value != gdrive.WriteScope.DIRECTORY_MEMBERS.value + + +@pytest.mark.parametrize("scope", list(gdrive.WriteScope)) +async def test_each_successful_write_grant_refuses_admission( + monkeypatch: pytest.MonkeyPatch, account: Any, scope: gdrive.WriteScope +) -> None: + def response(request: httpx.Request) -> httpx.Response: + assert _claims(request, account)["scope"] == scope.value + return httpx.Response(200, json={"access_token": "secret-write-token"}) + + async with _http(monkeypatch, response) as calls: + with pytest.raises(CredentialRefused) as error: + await google.require_write_denied( + account[0], + subject=_SUBJECT, + scope=scope, + deadline=time.monotonic() + 10, + max_bytes=4096, + ) + assert len(calls) == 1 and "secret-write-token" not in str(error.value) + + +@pytest.mark.parametrize( + "status,error", + [ + (400, "invalid_grant"), + (400, "access_denied"), + (403, "unauthorized_client"), + (429, "unauthorized_client"), + (503, "unauthorized_client"), + (302, "unauthorized_client"), + ], +) +async def test_uncertain_or_non_scope_failure_is_never_read_only_proof( + monkeypatch: pytest.MonkeyPatch, account: Any, status: int, error: str +) -> None: + async with _http( + monkeypatch, + lambda _: httpx.Response( + status, + json={"error": error, "error_description": "secret-private-response"}, + headers={"location": "https://untrusted.example/redirect"}, + ), + ) as calls: + with pytest.raises(VendorError) as failure: + await google.require_write_denied( + account[0], + subject=_SUBJECT, + scope=gdrive.WriteScope.DIRECTORY_MEMBERS, + deadline=time.monotonic() + 10, + max_bytes=4096, + ) + assert len(calls) == 1 and "secret-private-response" not in str(failure.value) + + +async def test_oversized_denial_closes_stream_without_accepting_it( + monkeypatch: pytest.MonkeyPatch, account: Any +) -> None: + body = _Body(b'{"error":"unauthorized_client",', b'"private":"' + b"s" * 100 + b'"}') + async with _http(monkeypatch, lambda _: httpx.Response(400, stream=body)): + with pytest.raises(TooLarge): + await google.require_write_denied( + account[0], + subject=_SUBJECT, + scope=gdrive.WriteScope.DRIVE, + deadline=time.monotonic() + 10, + max_bytes=32, + ) + assert body.closed + + +async def test_standing_lost_on_a_denial_stops_before_the_next_scope( + monkeypatch: pytest.MonkeyPatch, account: Any +) -> None: + standing = _Standing() + + def response(request: httpx.Request) -> httpx.Response: + scope = _claims(request, account)["scope"] + if scope in {read.value for read in gdrive.ReadScope}: + return httpx.Response( + 200, + json={ + "access_token": "private", + "token_type": "Bearer", + "expires_in": 60, + }, + ) + standing.live = False + return httpx.Response(400, json={"error": "unauthorized_client"}) + + async with _http(monkeypatch, response) as calls: + with pytest.raises(PermissionError): + await crawl_delegation.require_read_only( + account[0], + subject=_SUBJECT, + standing=standing, + deadline=time.monotonic() + 10, + max_bytes=4096, + ) + assert len(calls) == 4 diff --git a/tests/unit/test_crawl_exclusions.py b/tests/unit/test_crawl_exclusions.py new file mode 100644 index 0000000000..1f5b04ba22 --- /dev/null +++ b/tests/unit/test_crawl_exclusions.py @@ -0,0 +1,348 @@ +"""Actual exclusion rows, sealing, CAS fencing and crash repair with external I/O fakes.""" + +from __future__ import annotations + +import copy +import json +from contextlib import asynccontextmanager +from typing import Any + +import pytest + +from server.auth import claims +from server.clients._vendor import VendorError +from server.core import leases +from server.core.secrets_at_rest import resealed +from server.knowledge.app import crawl_exclusions as exclusions +from server.knowledge.domain.source_crawl import Person +from server.knowledge.store import crawl_exclusions as store +from server.settings import settings + +pytestmark = pytest.mark.asyncio +_PERSON = Person("workspace", "provider-user", "person@example.com", ("alias@example.com",)) + + +class _World: + def __init__(self, monkeypatch: pytest.MonkeyPatch) -> None: + self.keys: dict[str, dict[str, Any]] = {} + self.rows: dict[tuple[str, str], bool] = {} + self.keeper: dict[str, str] = {} + self.held = False + self.authorized = True + self.cleanup_fails = False + self.cleaned = 0 + self.queries = 0 + self.fail_key_write = False + self.version = 0 + monkeypatch.setattr( + settings, "jwt_secret", "first-root-sealing-secret-which-is-long-enough" + ) + monkeypatch.setattr(settings, "jwt_secret_previous", "") + monkeypatch.setattr(claims, "get", self.claim_get) + monkeypatch.setattr(claims, "swap", self.claim_swap) + monkeypatch.setattr(leases, "cluster_lease", self.lease) + + async def claim_get(self, ch: Any, key: str) -> str | None: + assert ch is self and key.startswith("crawl-exclusions:") + return self.keeper.get(key) + + async def claim_swap(self, ch: Any, key: str, old: str | None, new: str, expiry: int): + assert ch is self and self.held and expiry == 2**32 - 1 + if self.keeper.get(key) == old: + self.keeper[key] = new + return self.keeper.get(key) + + @asynccontextmanager + async def lease(self, name: str, *, ttl_s: float, wait_s: float, ch: Any): + assert ch is self and name.startswith("crawl-exclusions:") and not self.held + assert (ttl_s, wait_s) == (60, 0) + self.held = True + try: + yield + finally: + self.held = False + + async def query_one(self, sql: str, *, params: dict[str, Any]): + self.queries += 1 + if sql.startswith("SELECT greatest("): + assert self.held and (f"FROM {store.KEYS} " in sql or f"FROM {store.REFUSALS} " in sql) + self.version += 1 + return {"v": self.version} + assert set(params) == {"org"} + if f"FROM {store.KEYS} FINAL" in sql: + return copy.deepcopy(self.keys.get(params["org"])) + assert f"FROM {store.REFUSALS} FINAL" in sql and "LIMIT 1" in sql + return next(({"digest": d} for org, d in self.rows if org == params["org"]), None) + + async def query_all(self, sql: str, *, params: dict[str, Any]): + self.queries += 1 + if sql.startswith("SELECT digest, excluded FROM loom.knowledge_crawl_exclusions FINAL"): + assert params["limit"] == len(set(params["digests"])) + return [ + {"digest": d, "excluded": self.rows[params["org"], d]} + for d in params["digests"] + if (params["org"], d) in self.rows + ] + assert f"FROM {store.REFUSALS} FINAL" in sql and "AND excluded = 1" in sql + assert params["limit"] == len(params["digests"]) + return [{"digest": d} for d in params["digests"] if self.rows.get((params["org"], d))] + + async def insert(self, table: str, rows: list[dict[str, Any]]) -> None: + assert self.held + for row in rows: + if table == store.KEYS: + assert set(row) == { + "org", + "key_id", + "key_enc", + "generation", + "pending_id", + "pending_digests", + "pending_exclude", + "version", + } + self.keys[row["org"]] = copy.deepcopy(row) + if self.fail_key_write: + self.fail_key_write = False + raise RuntimeError("key write reply lost") + else: + assert table == store.REFUSALS and set(row) == { + "org", + "digest", + "excluded", + "version", + } + self.rows[row["org"], row["digest"]] = row["excluded"] + + async def validate(self, org: str) -> None: + assert org in {"org", "other-org"} + if not self.authorized: + raise PermissionError("org authority ended") + + async def cleanup(self) -> None: + assert self.held + assert any(v.startswith("blocked:") for v in self.keeper.values()) + if self.cleanup_fails: + raise RuntimeError("local cleanup incomplete") + self.cleaned += 1 + + async def start(self, org: str = "org") -> exclusions.Index: + return await exclusions.initialize(org, authority=self, ttl_s=60, wait_s=0, ch=self) + + async def change(self, index: exclusions.Index, *, exclude: bool = True) -> exclusions.Index: + return await index.change( + (_PERSON,), + exclude=exclude, + ttl_s=60, + wait_s=0, + authority=self, + cleanup=self.cleanup, + ) + + +async def test_exclusion_persists_only_keyed_refusal_and_stable_provider_and_alias_matches( + monkeypatch: pytest.MonkeyPatch, +) -> None: + world = _World(monkeypatch) + old = await world.start() + current = await world.change(old) + assert await current.contains(_PERSON) + assert await current.contains( + Person("workspace", "provider-user", "new-address@example.com", ()) + ) + assert await current.contains(Person("other-workspace", "different-user", _PERSON.email, ())) + assert await current.excluded_addresses((_PERSON.email, "other@example.com")) == {_PERSON.email} + assert world.cleaned == 1 and not world.keys["org"]["pending_id"] + encoded = json.dumps([world.keys, list(world.rows), world.keeper]) + assert all( + value not in encoded + for value in (_PERSON.email, _PERSON.id, _PERSON.customer, _PERSON.aliases[0]) + ) + assert all(len(digest) == 64 for org, digest in world.rows if org == "org") + with pytest.raises(VendorError, match="changed"): + await old.contains(_PERSON) + + +async def test_cleanup_failure_blocks_old_and_new_readers_and_restart_repairs_real_manifest( + monkeypatch: pytest.MonkeyPatch, +) -> None: + world = _World(monkeypatch) + index = await world.start() + world.cleanup_fails = True + with pytest.raises(RuntimeError, match="cleanup incomplete"): + await world.change(index) + assert world.keys["org"]["pending_id"] and all(world.rows.values()) + assert next(iter(world.keeper.values())).startswith("blocked:") + with pytest.raises(VendorError): + await index.contains(_PERSON) + with pytest.raises(VendorError): + await exclusions.Index.open("org", ch=world) + world.cleanup_fails = False + repaired = await exclusions.repair( + "org", + ttl_s=60, + wait_s=0, + authority=world, + cleanup=world.cleanup, + ch=world, + ) + assert await repaired.contains(_PERSON) and world.cleaned == 1 + assert world.keys["org"]["pending_id"] == "" + + +async def test_rescan_cannot_readmit_but_explicit_checked_readmission_changes_generation( + monkeypatch: pytest.MonkeyPatch, +) -> None: + world = _World(monkeypatch) + current = await world.change(await world.start()) + rescanned = await exclusions.Index.open("org", ch=world) + assert await rescanned.contains(_PERSON) + prior_version = await current.version() + readmitted = await world.change(current, exclude=False) + assert not await readmitted.contains(_PERSON) + assert await readmitted.version() != prior_version + assert world.cleaned == 2 + with pytest.raises(VendorError): + await rescanned.version() + + +async def test_same_address_in_other_org_does_not_inherit_person_refusal( + monkeypatch: pytest.MonkeyPatch, +) -> None: + world = _World(monkeypatch) + current = await world.change(await world.start()) + other = await world.start("other-org") + assert await current.contains(_PERSON) and not await other.contains(_PERSON) + assert set(current.person_digests(_PERSON)).isdisjoint(other.person_digests(_PERSON)) + + +async def test_root_reseal_preserves_stable_exclusion_key_and_refusal_identity( + monkeypatch: pytest.MonkeyPatch, +) -> None: + world = _World(monkeypatch) + current = await world.change(await world.start()) + before = current.person_digests(_PERSON) + previous = str(settings.jwt_secret) + monkeypatch.setattr(settings, "jwt_secret_previous", previous) + monkeypatch.setattr(settings, "jwt_secret", "second-root-sealing-secret-which-is-long-enough") + ciphertext = resealed(exclusions.KEY_PURPOSE, world.keys["org"]["key_enc"]) + assert ciphertext is not None + world.keys["org"]["key_enc"] = ( + ciphertext # External maintenance I/O; actual seal/open remain real. + ) + monkeypatch.setattr(settings, "jwt_secret_previous", "") + reopened = await exclusions.Index.open("org", ch=world) + assert reopened.person_digests(_PERSON) == before and await reopened.contains(_PERSON) + + +async def test_unreadable_or_lost_key_never_creates_a_new_key_over_existing_refusals( + monkeypatch: pytest.MonkeyPatch, +) -> None: + world = _World(monkeypatch) + await world.change(await world.start()) + world.keys.pop("org") + before = copy.deepcopy(world.rows) + with pytest.raises(VendorError, match="key unavailable"): + await world.start() + assert world.rows == before and "org" not in world.keys + + +async def test_authority_loss_before_mutation_writes_no_exclusion_or_new_fence( + monkeypatch: pytest.MonkeyPatch, +) -> None: + world = _World(monkeypatch) + index = await world.start() + before = copy.deepcopy((world.keys, world.rows, world.keeper)) + world.authorized = False + with pytest.raises(PermissionError): + await world.change(index) + assert (world.keys, world.rows, world.keeper) == before and world.cleaned == 0 + + +async def test_lost_initial_key_write_reply_reuses_the_persisted_key_on_retry( + monkeypatch: pytest.MonkeyPatch, +) -> None: + world = _World(monkeypatch) + world.fail_key_write = True + with pytest.raises(RuntimeError, match="reply lost"): + await world.start() + key_id = world.keys["org"]["key_id"] + assert next(iter(world.keeper.values())).startswith("initializing:") + current = await world.start() + assert world.keys["org"]["key_id"] == key_id + assert not await current.contains(_PERSON) and not world.rows + + +async def test_late_old_manifest_cannot_overwrite_a_newer_explicit_readmission( + monkeypatch: pytest.MonkeyPatch, +) -> None: + world = _World(monkeypatch) + index = await world.start() + saved: dict[str, Any] = {} + actual_cleanup = world.cleanup + + async def record_cleanup() -> None: + saved.update(copy.deepcopy(world.keys["org"])) + await actual_cleanup() + + current = await index.change( + (_PERSON,), + exclude=True, + ttl_s=60, + wait_s=0, + authority=world, + cleanup=record_cleanup, + ) + current = await world.change(current, exclude=False) + assert not await current.contains(_PERSON) + before = copy.deepcopy((world.rows, world.keeper)) + world.keys["org"] = saved # An old ClickHouse write is visible after the newer mutation. + with pytest.raises(VendorError, match="repair unavailable"): + await exclusions.repair( + "org", + ttl_s=60, + wait_s=0, + authority=world, + cleanup=world.cleanup, + ch=world, + ) + assert (world.rows, world.keeper) == before and world.cleaned == 2 + with pytest.raises(VendorError): + await current.version() + + +async def test_lost_refusal_write_cannot_release_the_completed_cleanup_fence( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from server.core.errors import BackendUnavailableError + + world = _World(monkeypatch) + index = await world.start() + + async def cleanup() -> None: + await world.cleanup() + # The external store lost the just-written refusal while cleanup completed. + world.rows.clear() + + with pytest.raises(BackendUnavailableError): + await index.change( + (_PERSON,), exclude=True, ttl_s=60, wait_s=0, authority=world, cleanup=cleanup + ) + assert world.cleaned == 1 and world.keys["org"]["pending_id"] + assert next(iter(world.keeper.values())).startswith("blocked:") + with pytest.raises(VendorError): + await exclusions.Index.open("org", ch=world) + + +@pytest.mark.parametrize("generation", ["active", "blocked", "initializing"]) +async def test_missing_registry_cannot_replace_a_surviving_keeper_generation( + monkeypatch: pytest.MonkeyPatch, generation: str +) -> None: + world = _World(monkeypatch) + fence = exclusions._fence("org") + world.keeper[fence] = generation + ":retained-key:retained-generation" + before = copy.deepcopy((world.keys, world.rows, world.keeper, world.version)) + with pytest.raises(VendorError, match="key unavailable"): + await world.start() + assert (world.keys, world.rows, world.keeper, world.version) == before + assert not world.cleaned and not world.held diff --git a/tests/unit/test_crawl_expansion.py b/tests/unit/test_crawl_expansion.py new file mode 100644 index 0000000000..3f80acb9e5 --- /dev/null +++ b/tests/unit/test_crawl_expansion.py @@ -0,0 +1,278 @@ +"""Actual Directory HTTP and crawl identity rules, without a provider or database.""" + +from __future__ import annotations + +import time +from typing import Any +from urllib.parse import parse_qs + +import httpx +import jwt +import pytest + +from server.auth import scim +from server.clients._vendor import CredentialRefused, TooLarge, VendorError +from server.knowledge.app import crawl_authority +from server.knowledge.app.crawl_expansion import Limits, expand +from server.knowledge.domain.source_crawl import Options, Person +from server.settings import settings +from tests.unit.test_google_workspace import _http +from tests.unit.test_google_workspace import account as account + +pytestmark = pytest.mark.asyncio +_NATIVE_SCIM_READ = scim.deactivated + + +@pytest.fixture(autouse=True) +def scim_rows(monkeypatch: pytest.MonkeyPatch) -> set[str]: + ended: set[str] = set() + + class Rows: + async def query_one(self, sql: str, *, params: dict[str, Any]): + assert sql == ( + "SELECT 1 AS x FROM loom.scim_users FINAL WHERE id = {id:String} AND active = 0" + ) + return {"x": 1} if params["id"] in ended else None + + rows = Rows() + monkeypatch.setattr(scim, "deactivated", _NATIVE_SCIM_READ) + monkeypatch.setattr(scim, "get_client", lambda: rows) + return ended + + +class _Standing: + current = True + + async def validate(self) -> None: + if not self.current: + raise PermissionError("source authority ended") + + +class _Exclusions: + epoch = "first" + + def __init__(self) -> None: + self.denied: set[str] = set() + self.observed: list[Person] = [] + + async def version(self) -> str: + return self.epoch + + async def contains(self, person: Person) -> bool: + self.observed.append(person) + return person.id in self.denied + + +class _Directory: + def __init__(self) -> None: + self.members: dict[str, list[dict[str, Any]]] = { + "team@example.com": [ + {"type": "USER", "id": "one", "email": "one@example.com"}, + {"type": "GROUP", "id": "nested", "email": "nested@example.com"}, + ], + "nested": [{"type": "USER", "id": "two", "email": "two@example.com"}], + } + self.customer = "workspace" + self.after_user: Any = lambda: None + + def answer(self, request: httpx.Request) -> httpx.Response: + if request.url.host == "oauth2.googleapis.com": + form = parse_qs(request.content.decode()) + claims = jwt.decode(form["assertion"][0], options={"verify_signature": False}) + assert claims["sub"] == "directory@example.com" + return httpx.Response( + 200, + json={"access_token": "private-token", "token_type": "Bearer", "expires_in": 60}, + ) + assert request.url.host == "admin.googleapis.com" + assert request.headers["authorization"] == "Bearer private-token" + if "/groups/" in request.url.path: + key = request.url.path.split("/groups/", 1)[1].removesuffix("/members") + return httpx.Response(200, json={"members": self.members[key]}) + key = request.url.path.rsplit("/", 1)[1] + identifier = {"one@example.com": "one", "alias@example.com": "one"}.get(key, key) + principal = key == "directory@example.com" + self.after_user() + return httpx.Response( + 200, + json={ + "id": identifier, + "primaryEmail": key if principal else f"{identifier}@example.com", + "aliases": ["alias@example.com"] if identifier == "one" else [], + "customerId": "workspace" if principal else self.customer, + "suspended": False, + "archived": False, + }, + ) + + +def _options(*subjects: tuple[str, str]) -> Options: + return Options.parse( + { + "directory_subject": "directory@example.com", + "subjects": [{"type": kind, "email": email} for kind, email in subjects], + } + ) + + +async def _run( + options: Options, + account: Any, + standing: _Standing, + exclusions: _Exclusions, + *, + limits: Limits | None = None, +): + return await expand( + options, + account[0], + standing=standing, + exclusions=exclusions, + limits=limits or Limits(body_bytes=16384, member_pages=10, groups=10), + deadline=time.monotonic() + 10, + ) + + +async def test_current_scim_deletion_excludes_a_person_and_every_group_entry( + monkeypatch: pytest.MonkeyPatch, account: Any, scim_rows: set[str] +) -> None: + directory = _Directory() + scim_rows.add(scim.user_id("alias@example.com")) + async with _http(monkeypatch, directory.answer): + result = await _run( + _options(("person", "one@example.com"), ("group", "team@example.com")), + account, + _Standing(), + _Exclusions(), + ) + assert [person.id for person in result.people] == ["two"] + assert all("one@example.com" not in members for _, members in result.groups) + + +async def test_complete_nested_groups_and_explicit_alias_share_stable_provider_identity( + monkeypatch: pytest.MonkeyPatch, + account: Any, +) -> None: + directory = _Directory() + exclusions = _Exclusions() + async with _http(monkeypatch, directory.answer) as calls: + result = await _run( + _options(("group", "team@example.com"), ("person", "alias@example.com")), + account, + _Standing(), + exclusions, + ) + assert [(p.customer, p.id, p.email) for p in result.people] == [ + ("workspace", "one", "one@example.com"), + ("workspace", "two", "two@example.com"), + ] + assert dict(result.groups) == { + "nested@example.com": ("two@example.com",), + "team@example.com": ("one@example.com", "two@example.com"), + } + assert result.exclusion_version == "first" + assert sum(r.url.path.endswith("/users/one") for r in calls) == 1 + assert not any(r.url.path.endswith("/users/alias@example.com") for r in calls) + assert "example.com" not in repr(result) and "example.com" not in repr(result.people[0]) + + +async def test_excluded_person_is_absent_from_direct_and_all_nested_membership( + monkeypatch: pytest.MonkeyPatch, + account: Any, +) -> None: + directory, exclusions = _Directory(), _Exclusions() + exclusions.denied.add("two") + async with _http(monkeypatch, directory.answer): + result = await _run( + _options(("group", "team@example.com"), ("person", "two@example.com")), + account, + _Standing(), + exclusions, + ) + assert [p.id for p in result.people] == ["one"] + assert dict(result.groups) == { + "nested@example.com": (), + "team@example.com": ("one@example.com",), + } + assert {p.id for p in exclusions.observed} == {"one", "two"} + + +@pytest.mark.parametrize("failure", ["cycle", "customer", "external", "group-bound"]) +async def test_unproven_or_incomplete_expansion_never_returns_partial_readers( + monkeypatch: pytest.MonkeyPatch, + account: Any, + failure: str, +) -> None: + directory = _Directory() + monkeypatch.setattr( + settings, "knowledge_crawl_max_groups", 1 if failure == "group-bound" else 10 + ) + limits = crawl_authority.configured_bounds().directory + if failure == "cycle": + directory.members["nested"] = [{"type": "GROUP", "id": "team", "email": "team@example.com"}] + elif failure == "customer": + directory.members["nested"] = [{"type": "CUSTOMER", "id": "workspace"}] + elif failure == "external": + directory.customer = "foreign-workspace" + async with _http(monkeypatch, directory.answer): + with pytest.raises((VendorError, CredentialRefused, TooLarge)): + await _run( + _options(("group", "team@example.com")), + account, + _Standing(), + _Exclusions(), + limits=limits, + ) + + +@pytest.mark.parametrize("change", ["standing", "exclusion"]) +async def test_loss_during_directory_response_stops_before_another_provider_dispatch( + monkeypatch: pytest.MonkeyPatch, + account: Any, + change: str, +) -> None: + directory, standing, exclusions = _Directory(), _Standing(), _Exclusions() + + def changed() -> None: + if change == "standing": + standing.current = False + else: + exclusions.epoch = "second" + + directory.after_user = changed + async with _http(monkeypatch, directory.answer) as calls: + with pytest.raises((PermissionError, VendorError)): + await _run(_options(("group", "team@example.com")), account, standing, exclusions) + assert len(calls) == 2 # Token and identity response; no members or person request. + + +async def test_directory_page_budget_is_global_across_selected_groups( + monkeypatch: pytest.MonkeyPatch, + account: Any, +) -> None: + directory = _Directory() + directory.members["team@example.com"] = [] + directory.members["other@example.com"] = [] + monkeypatch.setattr(settings, "knowledge_crawl_directory_max_pages", 1) + async with _http(monkeypatch, directory.answer) as calls: + with pytest.raises(TooLarge, match="page bound"): + await _run( + _options(("group", "team@example.com"), ("group", "other@example.com")), + account, + _Standing(), + _Exclusions(), + limits=crawl_authority.configured_bounds().directory, + ) + assert sum("/groups/" in r.url.path for r in calls) == 1 + + +async def test_unavailable_exclusion_state_refuses_before_token_exchange( + monkeypatch: pytest.MonkeyPatch, + account: Any, +) -> None: + exclusions = _Exclusions() + exclusions.epoch = "" + async with _http(monkeypatch, lambda r: pytest.fail("no HTTP admitted")) as calls: + with pytest.raises(VendorError, match="exclusions unavailable"): + await _run(_options(("person", "one@example.com")), account, _Standing(), exclusions) + assert calls == [] diff --git a/tests/unit/test_crawl_fleet_migration.py b/tests/unit/test_crawl_fleet_migration.py new file mode 100644 index 0000000000..a9d4dfe7a7 --- /dev/null +++ b/tests/unit/test_crawl_fleet_migration.py @@ -0,0 +1,60 @@ +"""Crawl admission waits for the migration that supplies its authorization state.""" + +from __future__ import annotations + +from unittest.mock import AsyncMock + +import pytest + +from server.core.errors import OverloadedError +from server.knowledge.app import crawl_sources +from server.knowledge.domain.sources import KnowledgeSource + + +@pytest.mark.asyncio +@pytest.mark.parametrize("supported", [112, 113, 114]) +async def test_registration_requires_the_crawl_migration_on_every_process( + monkeypatch: pytest.MonkeyPatch, supported: int +) -> None: + client = object() + permit = object() + scope = object() + required_versions: list[int] = [] + validation = AsyncMock() + + async def fleet_at(ch: object, required: int) -> bool: + assert ch is client + required_versions.append(required) + return supported >= required + + monkeypatch.setattr(crawl_sources, "get_client", lambda: client) + monkeypatch.setattr(crawl_sources.crawl_authority, "scope", lambda _source: scope) + monkeypatch.setattr(crawl_sources._Draft, "validate", validation) + monkeypatch.setattr( + crawl_sources.source_registration, "current_permit", AsyncMock(return_value=permit) + ) + monkeypatch.setattr(crawl_sources.machines, "fleet_at", fleet_at) + source = KnowledgeSource("org", "namespace", "name", "gdrive", "source") + + if supported < 114: + with pytest.raises(OverloadedError, match="not every process enforces crawl authority"): + await crawl_sources._draft(source, None) + assert validation.await_count == 1 + else: + draft = await crawl_sources._draft(source, None) + assert draft.source is source and draft.permit is permit + assert validation.await_count == 2 + assert required_versions == [114] + + +@pytest.mark.asyncio +async def test_composed_fixture_advertises_the_crawl_migration( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from server.core import instances + from tests.unit.test_crawl_composed_flow import _World + + world = _World.__new__(_World) + monkeypatch.setattr(instances, "read_live", world.read_live) + + assert await crawl_sources.machines.fleet_at(world, 114) diff --git a/tests/unit/test_crawl_person_acl.py b/tests/unit/test_crawl_person_acl.py new file mode 100644 index 0000000000..51deaeff7b --- /dev/null +++ b/tests/unit/test_crawl_person_acl.py @@ -0,0 +1,225 @@ +"""Person erasure redacts raw ACL history while preserving other writers and text.""" + +from __future__ import annotations + +import copy +from contextlib import asynccontextmanager +from typing import Any + +import pytest + +from server.erasure.store import crawl_person as store +from server.knowledge.domain.access_list import AccessList, from_json, to_json +from server.knowledge.domain.source_crawl import without_people +from server.knowledge.domain.sources import SYNC_WORKER_ACTOR +from server.settings import settings + +pytestmark = pytest.mark.asyncio +_ERASE = frozenset({"gone@example.com"}) +_OLD = to_json(AccessList(emails=("gone@example.com", "kept@example.com"), anyone=True)) +_NEW = to_json(AccessList(emails=("kept@example.com",), anyone=True)) + + +def _row(**extra: Any) -> dict[str, Any]: + return { + "org": "org", + "namespace": "mirror", + "actor": SYNC_WORKER_ACTOR, + "tags": ["source-id:source"], + "access_json": _OLD, + "text": "shared document", + **extra, + } + + +class _World: + def __init__(self, monkeypatch: pytest.MonkeyPatch, *, postgres: bool = False) -> None: + self.documents = [_row(version=1), _row(version=2)] + self.heads = [_row()] if postgres else [] + self.deliveries = ( + [{"row": _row(), "lease_token": 7, "left_s": 3.0, "targets": 5}] if postgres else [] + ) + self.events: list[str] = [] + self.waits: list[list[float]] = [] + self.old_publisher = False + self.cancel_wait = False + self.ignore_mutation = False + monkeypatch.setattr(store.pg, "enabled", lambda: postgres) + monkeypatch.setattr(store.pg, "transaction", self.transaction) + monkeypatch.setattr(store.pg, "wait_out", self.wait_out) + + @asynccontextmanager + async def transaction(self): + self.events.append("transaction") + yield self + self.events.append("commit") + + @staticmethod + def selected(row: dict[str, Any], args: Any) -> bool: + org, namespace, actor, tag = args[:4] + return (row["org"], row["namespace"], row["actor"]) == ( + org, + namespace, + actor, + ) and tag in row["tags"] + + async def wait_out(self, leases: list[float]) -> None: + assert self.events[-1] == "commit" + self.waits.append(leases) + self.events.append("wait") + if self.cancel_wait: + self.cancel_wait = False + raise RuntimeError("interrupted after durable redaction") + if self.old_publisher: + self.documents.append(_row(version=3)) + self.old_publisher = False + + async def fetch(self, sql: str, *args: Any): + assert " FINAL" not in sql + if "AS left_s FROM loom.knowledge_deliveries d" in sql: + assert "FOR UPDATE" in sql and "d.fenced_until" in sql + return [ + {"left_s": d["left_s"]} for d in self.deliveries if self.selected(d["row"], args) + ] + assert sql.startswith("SELECT DISTINCT row->>'access_json'") + rows = [*self.heads, *(d["row"] for d in self.deliveries)] + return [ + {"access_json": value} + for value in sorted({r["access_json"] for r in rows if self.selected(r, args)}) + ][: args[4]] + + async def execute(self, sql: str, *args: Any, params: dict[str, Any] | None = None): + if params is not None: + assert "ALTER TABLE loom.documents" in sql + assert "UPDATE access_json" in sql and "DELETE" not in sql + assert "erasure_mutation_tag" in params + self.events.append("mutation") + if not self.ignore_mutation: + for row in self.documents: + if ( + self.selected(row, self.scope(params)) + and row["access_json"] == params["before"] + ): + row["access_json"] = params["after"] + return + if sql.startswith("SELECT pg_advisory_xact_lock"): + assert args == ("org", "mirror") + return + if sql.startswith("SELECT 1 FROM loom.knowledge_heads"): + assert "ORDER BY org, namespace, source_uri FOR UPDATE" in sql + return + assert sql.startswith("UPDATE loom.knowledge_") and "jsonb_set" in sql + if "knowledge_heads" in sql: + for row in self.heads: + if self.selected(row, args) and row["access_json"] == args[4]: + row["access_json"] = args[5] + else: + assert "lease_token = lease_token + 1" in sql and "GREATEST(fenced_until" in sql + for delivery in self.deliveries: + row = delivery["row"] + if self.selected(row, args) and row["access_json"] == args[4]: + row["access_json"] = args[5] + delivery["lease_token"] += 1 + + @staticmethod + def scope(params: dict[str, Any]) -> tuple[str, ...]: + return tuple(params[k] for k in ("org", "ns", "actor", "tag")) + + async def query_all(self, sql: str, *, params: dict[str, Any]): + if "FROM system.mutations" in sql: + return [ + {"is_done": 1, "parts_to_do": 0, "latest_fail_reason": "", "mutation_id": "owned"} + ] + assert sql.startswith("SELECT DISTINCT access_json FROM loom.documents") + assert "FINAL" not in sql + values = sorted( + {r["access_json"] for r in self.documents if self.selected(r, self.scope(params))} + ) + return [{"access_json": value} for value in values[: params["limit"]]] + + async def query_one(self, sql: str, *, params: dict[str, Any]): + assert "SELECT count() AS n FROM loom.documents" in sql and "FINAL" not in sql + return { + "n": sum( + self.selected(r, self.scope(params)) and r["access_json"] == params["before"] + for r in self.documents + ) + } + + +async def test_raw_versions_redacted_without_erasing_text_other_grants_or_foreign_sources( + monkeypatch, +): + world = _World(monkeypatch) + foreign = [ + _row(org="other"), + _row(namespace="other"), + _row(actor="someone"), + _row(tags=["source-id:other"]), + ] + world.documents.extend(copy.deepcopy(foreign)) + await store.scrub("org", "mirror", "source", _ERASE, ch=world) + assert world.documents == [ + _row(version=1, access_json=_NEW), + _row(version=2, access_json=_NEW), + *foreign, + ] + assert await store.residue("org", "mirror", "source", _ERASE, ch=world) == 0 + + +async def test_old_postgres_publisher_waits_before_clickhouse_scrub(monkeypatch): + world = _World(monkeypatch, postgres=True) + world.old_publisher = True + await store.scrub("org", "mirror", "source", _ERASE, ch=world) + assert world.waits == [[3.0]] and world.events.index("wait") < world.events.index("mutation") + assert all(r["access_json"] == _NEW and r["text"] == "shared document" for r in world.documents) + assert world.heads == [_row(access_json=_NEW)] + assert world.deliveries == [ + {"row": _row(access_json=_NEW), "lease_token": 8, "left_s": 3.0, "targets": 5} + ] + + +async def test_retry_waits_retained_delivery_even_when_its_acl_was_already_redacted(monkeypatch): + world = _World(monkeypatch, postgres=True) + world.cancel_wait = True + with pytest.raises(RuntimeError, match="interrupted"): + await store.scrub("org", "mirror", "source", _ERASE, ch=world) + assert world.deliveries[0]["row"]["access_json"] == _NEW + assert all(r["access_json"] == _OLD for r in world.documents) + world.old_publisher = True + await store.scrub("org", "mirror", "source", _ERASE, ch=world) + assert world.waits == [[3.0], [3.0]] and world.deliveries[0]["lease_token"] == 8 + assert all(r["access_json"] == _NEW for r in world.documents) + + +async def test_reported_mutation_completion_cannot_hide_raw_historical_residue(monkeypatch): + world = _World(monkeypatch) + world.ignore_mutation = True + with pytest.raises(RuntimeError, match="history_remains"): + await store.scrub("org", "mirror", "source", _ERASE, ch=world) + assert await store.residue("org", "mirror", "source", _ERASE, ch=world) == 1 + + +async def test_history_bound_refuses_without_partial_clickhouse_mutation(monkeypatch): + world = _World(monkeypatch) + monkeypatch.setattr(settings, "knowledge_crawl_registry_scan_limit", 1) + world.documents.append(_row(access_json=to_json(AccessList(emails=("gone@example.com",))))) + with pytest.raises(RuntimeError, match="history_bound"): + await store.scrub("org", "mirror", "source", _ERASE, ch=world) + assert not world.events + + +async def test_only_exact_person_entries_removed_and_unread_history_stays_closed(): + raw = to_json( + AccessList( + emails=("gone@example.com", "other@example.com"), + groups=("team@example.com",), + domains=("example.com",), + ) + ) + result = from_json(without_people(raw, _ERASE)) + assert result == AccessList( + emails=("other@example.com",), groups=("team@example.com",), domains=("example.com",) + ) + assert without_people("{broken", _ERASE) == "" + assert without_people(to_json(AccessList(owner="someone")), _ERASE) == "" diff --git a/tests/unit/test_crawl_person_capture.py b/tests/unit/test_crawl_person_capture.py new file mode 100644 index 0000000000..35543aca9a --- /dev/null +++ b/tests/unit/test_crawl_person_capture.py @@ -0,0 +1,113 @@ +"""Cleanup captures real verified canonical identities before their ties disappear.""" + +from __future__ import annotations + +import json +from typing import Any + +import pytest + +from server.auth import person_erasure as erasure +from server.core.errors import BackendUnavailableError +from server.core.secrets_at_rest import seal +from tests.unit.test_connected_source_authority import world as world + +pytestmark = pytest.mark.asyncio + + +async def test_capture_retains_verified_canonical_subject_not_email_or_memory_user_id( + world, +) -> None: + world, _http = world + encoded = await erasure.capture("scim-person-id", "person@example.com", {"person": {"team"}}) + captured = erasure.load("scim-person-id", encoded) + assert captured.id == "scim-person-id" + assert captured.people == ( + erasure.Person("team", "person", ("person@example.com",), ("person", "person@example.com")), + ) + assert "person@example.com" not in encoded and "person@example.com" not in repr(captured) + assert world.directory_reads > 0 and not world.native_requests and not world.writes + + +async def test_contact_spelling_without_actual_verified_tie_cannot_capture_another_person( + world, +) -> None: + world, _http = world + with pytest.raises(BackendUnavailableError): + await erasure.capture("scim-person-id", "unverified@example.com", {"person": {"team"}}) + assert not world.native_requests and not world.writes + + +async def test_capture_cannot_be_replayed_for_a_different_deleted_identity(world) -> None: + encoded = await erasure.capture("scim-person-id", "person@example.com", {"person": {"team"}}) + with pytest.raises(BackendUnavailableError): + erasure.load("another-scim-id", encoded) + + +@pytest.mark.parametrize( + "change", + [ + {"format": True}, + {"id": "another"}, + { + "people": [ + { + "org": "team", + "subject": "person", + "emails": ["Unverified@example.com"], + "spellings": ["person"], + } + ] + }, + { + "people": [ + { + "org": "team", + "subject": "person", + "emails": [], + "spellings": ["person"], + "user_id": "person", + } + ] + }, + ], +) +async def test_malformed_sealed_metadata_never_turns_into_cleanup_authority( + world, change: Any +) -> None: + value = { + "format": 2, + "id": "scim-person-id", + "people": [ + { + "org": "team", + "subject": "person", + "emails": ["person@example.com"], + "spellings": ["person", "person@example.com"], + }, + ], + "workspaces": [], + "prepared": False, + } | change + encoded = seal(erasure.PURPOSE, json.dumps(value)) + with pytest.raises(BackendUnavailableError): + erasure.load("scim-person-id", encoded) + + +@pytest.mark.parametrize("pending", [False, True]) +async def test_pending_historical_scim_erasure_is_a_required_uncached_crawl_gate( + pending: bool, +) -> None: + class Rows: + async def query_one(self, sql: str): + assert sql == ( + "SELECT 1 AS pending FROM loom.scim_users FINAL " + "WHERE (deleted = 1 OR crawl_erasure_required = 1) AND NOT crawl_erasure_done LIMIT 1" + ) + return {"pending": 1} if pending else None + + if pending: + with pytest.raises(BackendUnavailableError): + await erasure.require_settled(ch=Rows()) + else: + await erasure.require_settled(ch=Rows()) diff --git a/tests/unit/test_crawl_person_controls.py b/tests/unit/test_crawl_person_controls.py new file mode 100644 index 0000000000..92c05f879d --- /dev/null +++ b/tests/unit/test_crawl_person_controls.py @@ -0,0 +1,240 @@ +"""Raw control history loses exact person references, not another owner's signing key.""" + +from __future__ import annotations + +import copy +import json +from typing import Any + +import pytest + +from server.erasure.store import crawl_controls as store +from server.knowledge.domain.source_crawl import without_selectors +from server.knowledge.store import sources + +pytestmark = pytest.mark.asyncio +_EMAILS = frozenset({"gone@example.com"}) + + +def _options(*people: str, directory: str = "admin@example.com") -> str: + return json.dumps( + { + "directory_subject": directory, + "subjects": [ + *({"type": "person", "email": p} for p in people), + {"type": "group", "email": "team@example.com"}, + ], + }, + sort_keys=True, + separators=(",", ":"), + ) + + +def _row(**extra: Any) -> dict[str, Any]: + return { + "org": "org", + "namespace": "namespace", + "source_id": "source", + "name": "name", + "credential_mode": "crawl", + "source_kind": "gdrive", + "kind": "", + "options_json": _options("gone@example.com", "kept@example.com"), + "registrant_subject": "other", + "created_by": "other", + "active": True, + "secret_enc": "sealed-shared-service-principal", + "credential_enc": "", + "approval_enc": "sealed-original-approval", + "crawl_generation": "generation", + "crawl_credential_hash": "a" * 64, + **extra, + } + + +class _World: + def __init__(self) -> None: + self.rows = [_row(version=1), _row(version=2)] + self.groups: list[dict[str, Any]] = [] + self.executed: list[str] = [] + self.ignore_mutation = False + + @staticmethod + def scope(row: dict[str, Any], params: dict[str, Any]) -> bool: + return all( + row[field] == params[key] + for field, key in (("org", "org"), ("namespace", "ns"), ("source_id", "id")) + ) + + def selected(self, sql: str, params: dict[str, Any]): + rows = self.groups if "knowledge_group_members" in sql else self.rows + found = [r for r in rows if self.scope(r, params)] + if "credential_mode = 'crawl'" in sql: + found = [r for r in found if r["credential_mode"] == "crawl"] + if "options_json = {before:String}" in sql: + found = [r for r in found if r["options_json"] == params["before"]] + if ( + "(registrant_subject = {subject:String} OR created_by IN {spellings:Array(String)})" + in sql + ): + found = [ + r + for r in found + if r["registrant_subject"] == params["subject"] + or r["created_by"] in params["spellings"] + ] + else: + for key in ("registrant_subject", "created_by"): + if f"{key} = {{subject:String}}" in sql: + found = [r for r in found if r[key] == params["subject"]] + if "created_by IN {spellings:Array(String)}" in sql: + found = [r for r in found if r["created_by"] in params["spellings"]] + if "(active OR approval_enc != '')" in sql: + found = [r for r in found if r["active"] or r["approval_enc"]] + if "hasAny(members" in sql: + found = [r for r in found if set(r["members"]).intersection(params["emails"])] + return found + + async def query_all(self, sql: str, *, params: dict[str, Any]): + if "FROM system.mutations" in sql: + return [ + {"is_done": 1, "parts_to_do": 0, "latest_fail_reason": "", "mutation_id": "owned"} + ] + assert "SELECT DISTINCT options_json, registrant_subject" in sql and "FINAL" not in sql + keys = sorted( + {(r["options_json"], r["registrant_subject"]) for r in self.selected(sql, params)} + ) + return [ + dict(zip(("options_json", "registrant_subject"), k, strict=True)) + for k in keys[: params["limit"]] + ] + + async def query_one(self, sql: str, *, params: dict[str, Any]): + assert "SELECT count() AS n" in sql and "FINAL" not in sql + return {"n": len(self.selected(sql, params))} + + async def execute(self, sql: str, *, params: dict[str, Any]): + assert "ALTER TABLE" in sql and "DELETE" not in sql and "erasure_mutation_tag" in params + self.executed.append(sql) + if self.ignore_mutation: + return + # Interpret only SQL writes; all retirement/redaction decisions remain in production. + for row in self.selected(sql, params): + if "UPDATE active = 0" in sql: + row.update(active=False, approval_enc="", crawl_generation="") + elif "UPDATE options_json" in sql: + row["options_json"] = params["after"] + elif "UPDATE members" in sql: + row["members"] = [v for v in row["members"] if v not in params["emails"]] + else: + field = "registrant_subject" if "UPDATE registrant_subject" in sql else "created_by" + row[field] = "" + + +async def test_selector_history_retires_only_affected_approvals_preserving_other_people_and_key(): + world = _World() + latest = _row(version=3, options_json=_options("kept@example.com")) + foreign = _row(org="other") + world.rows.extend([copy.deepcopy(latest), copy.deepcopy(foreign)]) + assert await store.controls("org", "namespace", "source", "deleted", _EMAILS, ch=world) + for row in world.rows[:2]: + assert row["secret_enc"] == "sealed-shared-service-principal" + assert row["active"] is False and row["approval_enc"] == row["crawl_generation"] == "" + assert json.loads(row["options_json"])["subjects"] == [ + {"type": "person", "email": "kept@example.com"}, + {"type": "group", "email": "team@example.com"}, + ] + assert not sources._source(row).active + assert world.rows[2:] == [latest, foreign] + + +async def test_registrant_erasure_does_not_retire_newly_approved_other_registrant(): + world = _World() + world.rows = [ + _row( + registrant_subject="deleted", + created_by="deleted", + options_json=_options("kept@example.com"), + ), + _row( + registrant_subject="other", + created_by="deleted", + options_json=_options("kept@example.com"), + ), + ] + await store.controls("org", "namespace", "source", "deleted", _EMAILS, ch=world) + old, current = world.rows + assert old["active"] is False and old["registrant_subject"] == old["created_by"] == "" + assert current["active"] and current["approval_enc"] == "sealed-original-approval" + assert current["created_by"] == "" and current["registrant_subject"] == "other" + assert old["secret_enc"] == current["secret_enc"] == "sealed-shared-service-principal" + + +async def test_historical_groups_drop_only_exact_member_for_exact_source(): + world = _World() + base = { + "org": "org", + "namespace": "mirror", + "source_id": "source", + "group": "team@example.com", + "members": ["gone@example.com", "kept@example.com"], + } + foreign = {**base, "source_id": "other"} + world.groups = [{**base, "version": 1}, {**base, "version": 2}, copy.deepcopy(foreign)] + await store.memberships("org", "mirror", "source", _EMAILS, ch=world) + assert [r["members"] for r in world.groups[:2]] == [["kept@example.com"]] * 2 + assert world.groups[2] == foreign + + +async def test_successful_mutation_status_cannot_hide_control_residue(): + world = _World() + world.ignore_mutation = True + with pytest.raises(RuntimeError, match="history_remains"): + await store.controls("org", "namespace", "source", "deleted", _EMAILS, ch=world) + + +async def test_directory_subject_removal_has_valid_idempotent_retired_representation(): + raw = _options("kept@example.com", directory="gone@example.com") + after = without_selectors(raw, _EMAILS) + assert json.loads(after)["directory_subject"] == "" + assert without_selectors(after, _EMAILS) == after + source = sources._source(_row(options_json=after, active=False, approval_enc="")) + assert not source.active and source.crawl_options == json.loads(after) + with pytest.raises(ValueError, match="address refused"): + sources._source(_row(options_json=after, active=True)) + + +async def test_malformed_historical_options_refuse_without_destroying_another_owners_key(): + world = _World() + world.rows[0]["options_json"] = "{broken" + before = copy.deepcopy(world.rows) + with pytest.raises(ValueError): + await store.controls("org", "namespace", "source", "deleted", _EMAILS, ch=world) + assert world.rows == before and not world.executed + + +async def test_verified_creator_alias_is_redacted_without_treating_it_as_source_authority(): + world = _World() + current = _row(created_by="person@example.com", options_json=_options("kept@example.com")) + foreign = _row(created_by="unverified@example.com", options_json=_options("kept@example.com")) + world.rows = [current, foreign] + affected = await store.controls( + "org", + "namespace", + "source", + "canonical-person", + _EMAILS, + spellings=("canonical-person", "person@example.com"), + ch=world, + ) + assert ( + not affected + and current["created_by"] == "" + and foreign["created_by"] == "unverified@example.com" + ) + assert all( + row["active"] + and row["approval_enc"] == "sealed-original-approval" + and row["secret_enc"] == "sealed-shared-service-principal" + for row in world.rows + ) diff --git a/tests/unit/test_crawl_person_finalize.py b/tests/unit/test_crawl_person_finalize.py new file mode 100644 index 0000000000..ba68861b85 --- /dev/null +++ b/tests/unit/test_crawl_person_finalize.py @@ -0,0 +1,207 @@ +"""Completed deletion clears raw capture PII before its durable done receipt.""" + +from __future__ import annotations + +import asyncio +import copy +import json +from typing import Any + +import pytest + +from server.auth import person_erasure as erasure +from server.auth import scim +from server.clients._vendor import CredentialRefused +from server.core.errors import BackendUnavailableError +from server.core.secrets_at_rest import seal +from server.erasure.app import crawl_person +from server.erasure.store import crawl_controls +from server.knowledge.app import crawl_person_erasure +from server.settings import settings +from tests.unit.test_crawl_authority import crawl_world as crawl_world +from tests.unit.test_crawl_person_workspace import captured_world as captured_world +from tests.unit.test_crawl_signing_authority import key_view as key_view +from tests.unit.test_google_workspace import account as account + +pytestmark = pytest.mark.asyncio + + +@pytest.fixture +async def finalizing(captured_world, monkeypatch: pytest.MonkeyPatch): + world, canonical = captured_world + prepared = await erasure.prepare("scim-person-id", canonical) + world.pending.update( + id="scim-person-id", active=False, deleted=True, crawl_erasure_enc=prepared + ) + world.history_caps = [ + {"id": "scim-person-id", "crawl_erasure_enc": canonical, "done": False}, + {"id": "scim-person-id", "crawl_erasure_enc": prepared, "done": False}, + {"id": "other-id", "crawl_erasure_enc": "other-owner-encrypted-metadata", "done": False}, + ] + world.capsule_mutations = 0 + world.ignore_capsule_mutation = False + world.lose_reply = False + query_one, query_all = world.query_one, world.query_all + + async def one(sql: str, params: dict[str, Any] | None = None, **kwargs: Any): + if "SELECT count() AS n FROM loom.scim_users" in sql: + assert params == {"id": "scim-person-id"} and "FINAL" not in sql + return { + "n": sum( + r["id"] == params["id"] and (bool(r["crawl_erasure_enc"]) or not r["done"]) + for r in world.history_caps + ) + } + return await query_one(sql, params, **kwargs) + + async def all_rows(sql: str, params: dict[str, Any], **kwargs: Any): + if "FROM system.mutations" in sql: + return [ + {"is_done": 1, "parts_to_do": 0, "latest_fail_reason": "", "mutation_id": "owned"} + ] + if sql.startswith("SELECT DISTINCT crawl_erasure_enc FROM loom.scim_users"): + assert "FINAL" not in sql and params["id"] == "scim-person-id" + return [ + {"crawl_erasure_enc": v} + for v in sorted( + { + r["crawl_erasure_enc"] + for r in world.history_caps + if r["id"] == params["id"] and r["crawl_erasure_enc"] + } + ) + ][: params["limit"]] + return await query_all(sql, params, **kwargs) + + async def execute(sql: str, *, params: dict[str, Any]): + assert "ALTER TABLE loom.scim_users" in sql and "UPDATE crawl_erasure_enc = ''" in sql + assert "crawl_erasure_done = true" in sql and "crawl_erasure_required = true" in sql + assert params["id"] == "scim-person-id" and "erasure_mutation_tag" in params + world.capsule_mutations += 1 + if not world.ignore_capsule_mutation: + for row in world.history_caps: + if row["id"] == params["id"]: + row.update(crawl_erasure_enc="", done=True) + world.pending.update(crawl_erasure_enc="", crawl_erasure_done=True) + if world.lose_reply: + raise RuntimeError("lost mutation reply") + + monkeypatch.setattr(world, "query_one", one) + monkeypatch.setattr(world, "query_all", all_rows) + monkeypatch.setattr(world, "execute", execute, raising=False) + monkeypatch.setattr(crawl_controls, "_default_client", lambda: world) + monkeypatch.setattr(erasure, "_finalizer", crawl_person.finalize) + yield world, canonical, prepared + + +async def test_finalization_erases_canonical_and_provider_capsule_versions_but_not_another_id( + finalizing, +): + world, _canonical, prepared = finalizing + await erasure.finalize("scim-person-id", prepared) + assert world.capsule_mutations == 1 + assert all(r["crawl_erasure_enc"] == "" and r["done"] for r in world.history_caps[:2]) + assert world.history_caps[2] == { + "id": "other-id", + "crawl_erasure_enc": "other-owner-encrypted-metadata", + "done": False, + } + assert world.pending["crawl_erasure_done"] and world.pending["crawl_erasure_enc"] == "" + + +async def test_lost_reply_after_atomic_redaction_is_completed_on_actual_scim_retry(finalizing): + world, _canonical, prepared = finalizing + world.lose_reply = True + with pytest.raises(RuntimeError, match="lost mutation reply"): + await erasure.finalize("scim-person-id", prepared) + assert world.pending["crawl_erasure_done"] and world.pending["crawl_erasure_enc"] == "" + before = len(world.provider_calls) + await scim._finish_person_erasure(dict(world.pending)) + assert world.capsule_mutations == 1 and len(world.provider_calls) == before + + +async def test_done_mutation_status_does_not_override_nonzero_raw_capsule_residue(finalizing): + world, _canonical, prepared = finalizing + world.ignore_capsule_mutation = True + with pytest.raises(RuntimeError, match="capture_history_remains"): + await erasure.finalize("scim-person-id", prepared) + assert ( + not world.pending["crawl_erasure_done"] and world.pending["crawl_erasure_enc"] == prepared + ) + + +async def test_uncovered_historical_canonical_person_is_not_silently_destroyed(finalizing): + world, _canonical, prepared = finalizing + foreign = seal( + erasure.PURPOSE, + json.dumps( + { + "format": 2, + "id": "scim-person-id", + "prepared": False, + "workspaces": [], + "people": [ + { + "org": "other-org", + "subject": "another-person", + "spellings": ["another-person"], + "emails": ["other@example.com"], + } + ], + } + ), + ) + world.history_caps.append({"id": "scim-person-id", "crawl_erasure_enc": foreign, "done": False}) + with pytest.raises(CredentialRefused, match="verified repair"): + await erasure.finalize("scim-person-id", prepared) + assert world.capsule_mutations == 0 and not world.pending["crawl_erasure_done"] + + +async def test_replaced_current_capture_cannot_finalize_old_cleanup(finalizing): + world, _canonical, prepared = finalizing + world.pending["crawl_erasure_enc"] = "replacement" + with pytest.raises(BackendUnavailableError): + await erasure.finalize("scim-person-id", prepared) + assert world.capsule_mutations == 0 + + +async def test_missing_raw_current_capture_refuses_before_erasure_receipt(finalizing): + world, canonical, prepared = finalizing + world.history_caps = [{"id": "scim-person-id", "crawl_erasure_enc": canonical, "done": False}] + with pytest.raises(CredentialRefused, match="history unavailable"): + await erasure.finalize("scim-person-id", prepared) + assert world.capsule_mutations == 0 and not world.pending["crawl_erasure_done"] + + +async def test_configured_cleanup_deadline_cancels_external_io_without_completing_capsule( + finalizing, + monkeypatch: pytest.MonkeyPatch, +) -> None: + world, _canonical, prepared = finalizing + monkeypatch.setattr(settings, "knowledge_crawl_person_cleanup_timeout_s", 1.0) + monkeypatch.setattr(erasure, "_cleanup", crawl_person_erasure.finish) + monkeypatch.setattr(crawl_person_erasure, "get_client", lambda: world) + original = world.query_one + before = copy.deepcopy(world.exclusion_keys), world.source + entered = False + cancelled = False + + async def blocked(sql: str, params: dict[str, Any] | None = None, **kwargs: Any): + nonlocal entered, cancelled + if "FROM loom.knowledge_crawl_exclusion_keys FINAL" in sql: + assert params == {"org": "team"} + entered = True + try: + await asyncio.Event().wait() + finally: + cancelled = True + return await original(sql, params, **kwargs) + + monkeypatch.setattr(world, "query_one", blocked) + async with asyncio.timeout(3): + with pytest.raises(TimeoutError): + await erasure.finish("scim-person-id", prepared) + assert entered and cancelled + assert (world.exclusion_keys, world.source) == before + assert world.pending["crawl_erasure_enc"] == prepared + assert not world.pending["crawl_erasure_done"] and not world.capsule_mutations diff --git a/tests/unit/test_crawl_person_reseal.py b/tests/unit/test_crawl_person_reseal.py new file mode 100644 index 0000000000..7e6ef1f52d --- /dev/null +++ b/tests/unit/test_crawl_person_reseal.py @@ -0,0 +1,126 @@ +"""Raw canonical-person capsule history remains readable after root rotation.""" + +from __future__ import annotations + +import copy +from contextlib import asynccontextmanager +from typing import Any + +import pytest + +from server.auth import person_erasure as person +from server.auth import scim +from server.core.errors import BackendUnavailableError +from server.settings import settings + +pytestmark = pytest.mark.asyncio + + +class _Rows: + def __init__(self, monkeypatch: pytest.MonkeyPatch) -> None: + self.rows: list[dict[str, Any]] = [] + self.held = False + self.ignore_mutation = False + self.mutations = 0 + monkeypatch.setattr(person, "get_client", lambda: self) + monkeypatch.setattr(scim.leases, "holding", self.holding) + + @asynccontextmanager + async def holding(self, key: str, *, ttl_s: float, wait_s: float, settle_writes: bool): + assert key == "scim-user:managed-person" and ttl_s > wait_s >= 0 + assert settle_writes is True + assert not self.held + self.held = True + try: + yield True + finally: + self.held = False + + async def query_all(self, sql: str, *, params: dict[str, Any]): + if sql.startswith("SELECT DISTINCT id "): + assert "FINAL" not in sql and params["limit"] > 0 + return [{"id": "managed-person"}] if params["after"] < "managed-person" else [] + assert sql.startswith("SELECT DISTINCT crawl_erasure_enc ") + assert "FINAL" not in sql and self.held and params["id"] == "managed-person" + return [ + {"crawl_erasure_enc": value} + for value in sorted({r["crawl_erasure_enc"] for r in self.rows}) + ] + + async def execute(self, sql: str, *, params: dict[str, Any], settings: dict[str, int]): + assert self.held and settings == {"mutations_sync": 2} + assert sql == ( + "ALTER TABLE loom.scim_users UPDATE crawl_erasure_enc = {new:String} " + "WHERE id = {id:String} AND crawl_erasure_enc = {old:String}" + ) + self.mutations += 1 + if not self.ignore_mutation: + for row in self.rows: + if row["id"] == params["id"] and row["crawl_erasure_enc"] == params["old"]: + row["crawl_erasure_enc"] = params["new"] + + async def query_one(self, sql: str, *, params: dict[str, Any]): + assert "FINAL" not in sql and self.held + return { + "n": sum( + r["id"] == params["id"] and r["crawl_erasure_enc"] == params["old"] + for r in self.rows + ) + } + + +def _history(monkeypatch: pytest.MonkeyPatch) -> tuple[_Rows, list[person.Capture]]: + monkeypatch.setattr(settings, "jwt_secret", "first-person-history-root-secret-which-is-long") + monkeypatch.setattr(settings, "jwt_secret_previous", "") + state = _Rows(monkeypatch) + owner = person.Person("org", "canonical", ("contact@example.com",), ("canonical", "old-alias")) + captured = [ + person.Capture("managed-person", (owner,), prepared=ready) for ready in (False, True) + ] + state.rows = [ + { + "id": value.id, + "crawl_erasure_enc": person._seal(value), + "active": False, + "deleted": True, + "subjects": ["canonical"], + "version": index + 1, + } + for index, value in enumerate(captured) + ] + monkeypatch.setattr(settings, "jwt_secret_previous", settings.jwt_secret) + monkeypatch.setattr(settings, "jwt_secret", "next-person-history-root-secret-which-is-long") + return state, captured + + +async def test_all_historical_capsules_reseal_without_changing_ordinary_identity( + monkeypatch: pytest.MonkeyPatch, +) -> None: + rows, captures = _history(monkeypatch) + before = copy.deepcopy(rows.rows) + assert await person.reseal_registry(dry_run=False, ch=rows) == (2, []) + monkeypatch.setattr(settings, "jwt_secret_previous", "") + assert [person.load(row["id"], row["crawl_erasure_enc"]) for row in rows.rows] == captures + for old, new in zip(before, rows.rows, strict=True): + assert old["crawl_erasure_enc"] != new["crawl_erasure_enc"] + assert {key: value for key, value in old.items() if key != "crawl_erasure_enc"} == { + key: value for key, value in new.items() if key != "crawl_erasure_enc" + } + assert rows.mutations == 2 and not rows.held + + +async def test_dry_run_preserves_every_raw_capsule_version(monkeypatch: pytest.MonkeyPatch) -> None: + rows, _captures = _history(monkeypatch) + before = copy.deepcopy(rows.rows) + assert await person.reseal_registry(dry_run=True, ch=rows) == (2, []) + assert rows.rows == before and rows.mutations == 0 and not rows.held + + +async def test_unacknowledged_raw_history_mutation_refuses_root_retirement( + monkeypatch: pytest.MonkeyPatch, +) -> None: + rows, _captures = _history(monkeypatch) + rows.ignore_mutation = True + with pytest.raises(BackendUnavailableError): + await person.reseal_registry(dry_run=False, ch=rows) + assert rows.mutations == 1 and not rows.held diff --git a/tests/unit/test_crawl_person_workspace.py b/tests/unit/test_crawl_person_workspace.py new file mode 100644 index 0000000000..31a048bc80 --- /dev/null +++ b/tests/unit/test_crawl_person_workspace.py @@ -0,0 +1,230 @@ +"""Deletion captures stable Google ids through real source and Directory boundaries.""" + +from __future__ import annotations + +import dataclasses +import time +from typing import Any + +import httpx +import pytest + +from server.auth import person_erasure as erasure +from server.clients import gdrive +from server.clients import google_workspace as google +from server.clients._vendor import CredentialRefused +from server.core.errors import BackendUnavailableError +from server.knowledge.app import crawl_capture +from server.knowledge.store import sources +from server.settings import settings +from tests.unit.test_crawl_authority import crawl_world as crawl_world +from tests.unit.test_crawl_signing_authority import key_view as key_view +from tests.unit.test_google_workspace import account as account + +pytestmark = pytest.mark.asyncio + + +@pytest.fixture +async def captured_world(crawl_world, monkeypatch: pytest.MonkeyPatch): + world = crawl_world + monkeypatch.setattr(erasure, "get_client", lambda: world) + monkeypatch.setattr(crawl_capture, "get_client", lambda: world) + monkeypatch.setattr(erasure, "_preparer", crawl_capture.prepare) + original_query_one, original_query_all = world.query_one, world.query_all + original_handle = world.handle + world.pending = { + "active": True, + "deleted": False, + "crawl_erasure_required": True, + "crawl_erasure_done": False, + "crawl_erasure_enc": "", + } + world.after_person = lambda: None + world.current_contact = "person@example.com" + written: list[dict[str, Any]] = [] + + class Rows: + async def insert(self, table: str, rows: list[dict[str, Any]]) -> None: + assert table == sources.TABLE + written.extend(rows) + + await sources.insert_source(world.source, ch=Rows()) + world.history = written + + async def query_one(sql: str, params: dict[str, Any] | None = None, **kwargs: Any): + if sql.startswith("SELECT active, deleted, crawl_erasure_enc"): + assert params == {"id": "scim-person-id"} + return dict(world.pending) + return await original_query_one(sql, params, **kwargs) + + async def query_all(sql: str, params: dict[str, Any], **kwargs: Any): + if "FROM loom.knowledge_sources WHERE" in sql: + assert sql.startswith("SELECT DISTINCT") and "FINAL" not in sql + assert params["org"] == "team" + return world.history[: params["limit"]] + result = await original_query_all(sql, params, **kwargs) + if "FROM loom.namespaces" in sql: + return [{**row, "contact_email": world.current_contact} for row in result] + return result + + async def handle(request: httpx.Request): + if request.url.host == "admin.googleapis.com" and request.url.path.endswith( + "/person@example.com" + ): + world.provider_calls.append(request) + assert request.headers["authorization"] == "Bearer private-directory-access" + world.after_person() + return httpx.Response( + 200, + json={ + "id": "stable-google-user", + "primaryEmail": "person@example.com", + "aliases": ["old-address@example.com"], + "customerId": "workspace", + "suspended": False, + "archived": False, + }, + ) + return await original_handle(request) + + monkeypatch.setattr(world, "query_one", query_one) + monkeypatch.setattr(world, "query_all", query_all) + monkeypatch.setattr(world, "handle", handle) + encoded = await erasure.capture("scim-person-id", "person@example.com", {"person": {"team"}}) + world.pending["crawl_erasure_enc"] = encoded + yield world, encoded + + +async def test_real_predelete_capture_keeps_google_id_distinct_from_canonical_subject( + captured_world, +): + world, encoded = captured_world + prepared = await erasure.prepare("scim-person-id", encoded) + result = erasure.load("scim-person-id", prepared) + assert result.prepared and result.people == ( + erasure.Person("team", "person", ("person@example.com",), ("person", "person@example.com")), + ) + assert result.workspaces == ( + erasure.Workspace( + "team", + "person", + "workspace", + "stable-google-user", + ("old-address@example.com", "person@example.com"), + ), + ) + assert {claim["scope"] for claim in world.claims_sent} == { + gdrive.ReadScope.DIRECTORY_USERS.value + } + assert {claim["sub"] for claim in world.claims_sent} == {"directory@example.com"} + assert not world.native_requests and not world.writes + assert world.pending["crawl_erasure_enc"] == encoded + assert "person@example.com" not in prepared and "stable-google-user" not in prepared + + +async def test_absent_crawl_history_does_not_invent_google_provider_configuration(captured_world): + world, encoded = captured_world + world.history = [] + prepared = erasure.load("scim-person-id", await erasure.prepare("scim-person-id", encoded)) + assert prepared.prepared and prepared.workspaces == () + assert not world.provider_calls and not world.native_requests + + +async def test_retired_customer_without_current_authority_cannot_use_another_workspaces_key( + captured_world, +): + world, encoded = captured_world + world.history.append( + dict(world.history[0], source_id="old-source", crawl_customer_id="old-workspace") + ) + with pytest.raises(CredentialRefused, match="verified repair"): + await erasure.prepare("scim-person-id", encoded) + assert not world.provider_calls and world.pending["crawl_erasure_enc"] == encoded + + +async def test_provider_observation_cannot_outlive_source_retirement(captured_world): + world, encoded = captured_world + world.after_person = lambda: setattr( + world, "source", dataclasses.replace(world.source, active=False) + ) + with pytest.raises(CredentialRefused): + await erasure.prepare("scim-person-id", encoded) + assert world.pending["crawl_erasure_enc"] == encoded + assert not erasure.load("scim-person-id", encoded).prepared and not world.native_requests + + +async def test_deleted_or_replaced_capture_never_resolves_a_potentially_reused_email( + captured_world, +): + world, encoded = captured_world + world.pending.update(active=False, deleted=True) + with pytest.raises(BackendUnavailableError): + await erasure.prepare("scim-person-id", encoded) + assert not world.provider_calls and world.pending["crawl_erasure_enc"] == encoded + + +async def test_capture_replacement_during_directory_response_cannot_publish_old_proof( + captured_world, +): + world, encoded = captured_world + world.after_person = lambda: world.pending.update(crawl_erasure_enc="different-capture") + with pytest.raises(BackendUnavailableError): + await erasure.prepare("scim-person-id", encoded) + assert not world.native_requests and not world.writes + + +async def test_canonical_contact_rebinding_during_directory_response_refuses_old_identity( + captured_world, +): + world, encoded = captured_world + world.after_person = lambda: setattr(world, "current_contact", "different-person@example.com") + with pytest.raises(BackendUnavailableError): + await erasure.prepare("scim-person-id", encoded) + assert world.pending["crawl_erasure_enc"] == encoded + assert not erasure.load("scim-person-id", encoded).prepared and not world.native_requests + + +@pytest.mark.parametrize("budget", [0.25, 1.0]) +async def test_configured_capture_deadline_rechecks_the_actual_directory_response( + captured_world, + monkeypatch: pytest.MonkeyPatch, + budget: float, +) -> None: + world, encoded = captured_world + clock = time.monotonic() + monkeypatch.setattr(settings, "knowledge_crawl_identity_capture_timeout_s", budget) + monkeypatch.setattr(crawl_capture, "_now", lambda: clock) + monkeypatch.setattr(google, "_now", lambda: clock) + + def spent() -> None: + nonlocal clock + clock += 0.5 + + world.after_person = spent + if budget < 0.5: + with pytest.raises(TimeoutError): + await erasure.prepare("scim-person-id", encoded) + else: + prepared = await erasure.prepare("scim-person-id", encoded) + captured = erasure.load("scim-person-id", prepared) + assert captured.prepared and captured.workspaces[0].user == "stable-google-user" + assert world.pending["crawl_erasure_enc"] == encoded + assert not world.native_requests and not world.writes + + +async def test_inactive_person_can_prepare_but_deleted_or_replaced_capsule_cannot( + captured_world, +) -> None: + world, encoded = captured_world + world.pending["active"] = False + prepared = await erasure.prepare("scim-person-id", encoded) + assert erasure.load("scim-person-id", prepared).prepared + assert world.pending["crawl_erasure_enc"] == encoded + before = len(world.provider_calls) + world.pending["deleted"] = True + with pytest.raises(BackendUnavailableError): + await erasure.prepare("scim-person-id", encoded) + world.pending.update(deleted=False, crawl_erasure_enc="replacement") + with pytest.raises(BackendUnavailableError): + await erasure.prepare("scim-person-id", encoded) + assert len(world.provider_calls) == before and not world.native_requests diff --git a/tests/unit/test_crawl_registration.py b/tests/unit/test_crawl_registration.py new file mode 100644 index 0000000000..b80b4e78b3 --- /dev/null +++ b/tests/unit/test_crawl_registration.py @@ -0,0 +1,231 @@ +"""Real JWT and shared authorization rules for an exact retained source approval.""" + +from __future__ import annotations + +import dataclasses +import json +import time +from typing import Any + +import jwt +import pytest + +from server.auth import authz +from server.auth import source_registration as registration +from server.auth.context import current_auth +from server.auth.jwt import AuthInfo, verify_token +from server.core.secrets_at_rest import seal, unseal +from server.operations.base import Role +from server.settings import settings +from tests.unit.principals import acting_as +from tests.unit.test_connected_source_authority import world as world +from tests.unit.test_crawl_signing_authority import _SECOND +from tests.unit.test_crawl_signing_authority import key_view as key_view + +pytestmark = [pytest.mark.asyncio, pytest.mark.claim_cap] +_SCOPE = registration.Scope("team", "shared-space", "crawl-source", "crawl-generation", "f" * 64) + + +def _token(**changes: Any) -> str: + now = int(time.time()) + return jwt.encode( + { + "sub": "person@example.com", + "org": "team", + "namespaces": ["shared-space"], + "roles": ["org_admin"], + "sa": False, + "iss": settings.jwt_issuer, + "aud": settings.jwt_audience, + "iat": now - 10, + "exp": now + 3600, + "jti": "registration-token", + **changes, + }, + _SECOND, + algorithm="HS256", + headers={"kid": "signer"}, + ) + + +async def _approval(state: Any, **changes: Any): + state.grant = Role.ADMIN + raw = _token(**changes) + auth = verify_token(raw) + with acting_as(auth): + registrant, encoded = await registration.capture(auth, _SCOPE) + return auth, registrant, encoded, raw + + +async def test_retained_metadata_is_exact_scoped_and_never_becomes_a_request_principal( + world: Any, + key_view: list[str], +) -> None: + state, _http = world + auth, registrant, encoded, raw = await _approval(state) + approval = registration.load(encoded, _SCOPE, registrant=registrant) + before = current_auth() + await approval.validate() + assert current_auth() is before + assert registrant == "person" and not isinstance(approval.claims, AuthInfo) + assert approval.claims.roles == auth.roles and approval.claims.expires_at == auth.expires_at + plain = unseal(registration.PURPOSE, encoded) + assert raw not in plain and _SECOND not in plain + assert json.loads(plain)["scope"] == dataclasses.asdict(_SCOPE) + with pytest.raises(PermissionError, match="invalid auth principal"): + await authz.check_authorized(approval.claims, Role.ADMIN, "team", "shared-space") + + +@pytest.mark.parametrize("field", ["creator", "impersonated_by", "parent_subjects"]) +async def test_historical_approval_inspection_finds_typed_person_lineage_without_granting_authority( + world: Any, + key_view: list[str], + field: str, +) -> None: + state, _http = world + _auth, _registrant, original, _raw = await _approval(state) + frame = json.loads(unseal(registration.PURPOSE, original)) + # These are historical typed metadata, not a live AuthInfo or a newly approved source. + frame["claims"][field] = ["deleted-alias"] if field == "parent_subjects" else "deleted-alias" + if field == "parent_subjects": + frame["claims"]["parent_iats"] = [frame["claims"]["issued_at"]] + encoded = seal(registration.PURPOSE, json.dumps(frame)) + before = current_auth() + assert registration.names_person( + encoded, + org="team", + namespace="shared-space", + source_id="crawl-source", + spellings=("deleted-alias",), + ) + assert not registration.names_person( + encoded, + org="team", + namespace="shared-space", + source_id="crawl-source", + spellings=("someone-else",), + ) + assert current_auth() is before + + +@pytest.mark.parametrize("field", ["org", "namespace", "source_id"]) +async def test_person_inspection_cannot_retire_an_approval_copied_from_another_scope( + world: Any, + key_view: list[str], + field: str, +) -> None: + state, _http = world + _auth, _registrant, encoded, _raw = await _approval(state) + scope = {"org": "team", "namespace": "shared-space", "source_id": "crawl-source"} + scope[field] = "foreign" + with pytest.raises(registration.Refused): + registration.names_person(encoded, **scope, spellings=("person", "person@example.com")) + + +@pytest.mark.parametrize("nested", [False, True]) +async def test_unknown_retained_claim_restrictions_refuse_instead_of_being_ignored( + world: Any, + key_view: list[str], + nested: bool, +) -> None: + state, _http = world + _auth, registrant, original, _raw = await _approval(state) + frame = json.loads(unseal(registration.PURPOSE, original)) + target = frame["claims"]["signing_authority"] if nested else frame["claims"] + target["future_restriction"] = {"deny": True} + encoded = seal(registration.PURPOSE, json.dumps(frame)) + with pytest.raises(registration.Refused): + registration.load(encoded, _SCOPE, registrant=registrant) + + +@pytest.mark.parametrize("field", ["namespace", "source_id", "generation", "org", "resource_hash"]) +async def test_copied_ciphertext_cannot_approve_another_source_or_generation( + world: Any, + key_view: list[str], + field: str, +) -> None: + state, _http = world + _auth, registrant, encoded, _raw = await _approval(state) + changed = dataclasses.replace(_SCOPE, **{field: "different"}) + with pytest.raises(registration.Refused): + registration.load(encoded, changed, registrant=registrant) + + +async def test_namespace_demotion_refuses_retained_named_claim_using_fresh_real_grants( + world: Any, + key_view: list[str], +) -> None: + state, _http = world + _auth, registrant, encoded, _raw = await _approval(state) + approval = registration.load(encoded, _SCOPE, registrant=registrant) + state.grant = Role.WRITER + with pytest.raises(PermissionError): + await approval.validate() + + +async def test_flat_wildcard_org_role_keeps_existing_additive_claim_semantics( + world: Any, + key_view: list[str], +) -> None: + state, _http = world + _auth, registrant, encoded, _raw = await _approval(state, namespaces=["*"]) + state.grant = Role.WRITER + await registration.load(encoded, _SCOPE, registrant=registrant).validate() + + +@pytest.mark.parametrize( + "change", + [ + {"roles": ["admin"]}, + {"roles": ["org_admin"], "ns_roles": {"shared-space": "viewer"}}, + ], +) +async def test_registration_requires_both_real_flat_org_role_and_namespace_admin( + world: Any, + key_view: list[str], + change: dict[str, Any], +) -> None: + state, _http = world + state.grant = Role.ADMIN + auth = verify_token(_token(**change)) + with acting_as(auth), pytest.raises(PermissionError): + await registration.capture(auth, _SCOPE) + + +@pytest.mark.parametrize("loss", ["signing-key", "token-revocation", "scim", "expiry"]) +async def test_registration_rechecks_actual_signing_token_and_person_authority( + monkeypatch: pytest.MonkeyPatch, + world: Any, + key_view: list[str], + loss: str, +) -> None: + state, _http = world + auth, registrant, encoded, _raw = await _approval(state) + approval = registration.load(encoded, _SCOPE, registrant=registrant) + if loss == "signing-key": + key_view.remove(_SECOND) + elif loss == "token-revocation": + state.cutoffs["registration-token"] = None + elif loss == "scim": + state.scim_ended = True + else: + assert auth.expires_at is not None + monkeypatch.setattr( + registration, "_now", lambda: auth.expires_at + settings.jwt_leeway_seconds + 1 + ) + with pytest.raises(PermissionError): + await approval.validate() + + +async def test_offline_verified_or_changed_presenter_cannot_create_persistent_approval( + world: Any, + key_view: list[str], +) -> None: + state, _http = world + state.grant = Role.ADMIN + offline = verify_token(_token(), secret=_SECOND) + with acting_as(offline), pytest.raises(registration.Refused): + await registration.capture(offline, _SCOPE) + auth = verify_token(_token()) + with acting_as(auth), pytest.raises(registration.Refused): + await registration.capture(dataclasses.replace(auth), _SCOPE) diff --git a/tests/unit/test_crawl_reseal.py b/tests/unit/test_crawl_reseal.py new file mode 100644 index 0000000000..722bf6018f --- /dev/null +++ b/tests/unit/test_crawl_reseal.py @@ -0,0 +1,94 @@ +"""Actual reseal writers preserve stable refusal keys and unfinished cleanup manifests.""" + +from __future__ import annotations + +import copy +from typing import Any + +import pytest + +from server.clients._vendor import VendorError +from server.core.secrets_at_rest import unseal +from server.knowledge.app import crawl_exclusions as exclusions +from server.knowledge.app import crawl_reseal +from server.knowledge.store import crawl_exclusions as store +from server.settings import settings +from tests.unit.test_crawl_exclusions import _PERSON +from tests.unit.test_crawl_exclusions import _World as _ExclusionWorld + +pytestmark = pytest.mark.asyncio + + +class _World(_ExclusionWorld): + def __init__(self, monkeypatch: pytest.MonkeyPatch) -> None: + super().__init__(monkeypatch) + monkeypatch.setattr(crawl_reseal, "get_client", lambda: self) + monkeypatch.setattr(settings, "knowledge_crawl_lease_wait_s", 0) + monkeypatch.setattr(settings, "knowledge_crawl_registry_page_size", 1) + + async def query_all(self, sql: str, *, params: dict[str, Any]): + if sql.startswith(f"SELECT org FROM {store.KEYS} FINAL"): + assert params["limit"] == 1 + return [{"org": org} for org in sorted(self.keys) if org > params["after"]][:1] + return await super().query_all(sql, params=params) + + +def _rotate(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(settings, "jwt_secret_previous", settings.jwt_secret) + monkeypatch.setattr(settings, "jwt_secret", "second-root-sealing-secret-which-is-long-enough") + + +async def test_rotation_preserves_the_same_key_and_pending_blocked_cleanup( + monkeypatch: pytest.MonkeyPatch, +) -> None: + world = _World(monkeypatch) + index = await world.start() + digests = index.person_digests(_PERSON) + world.cleanup_fails = True + with pytest.raises(RuntimeError, match="cleanup incomplete"): + await world.change(index) + before, fence = copy.deepcopy(world.keys["org"]), dict(world.keeper) + _rotate(monkeypatch) + assert await crawl_reseal.keys(dry_run=False, ch=world) == (1, []) + after = world.keys["org"] + assert after["key_enc"] != before["key_enc"] and after["version"] > before["version"] + for field in ("key_id", "generation", "pending_id", "pending_digests", "pending_exclude"): + assert after[field] == before[field] + assert world.keeper == fence and after["pending_id"] + assert exclusions._open_key(after) == exclusions._open_key(before) + assert index.person_digests(_PERSON) == digests + with pytest.raises(VendorError, match="cleanup incomplete"): + await exclusions.Index.open("org", ch=world) + monkeypatch.setattr(settings, "jwt_secret_previous", "") + assert unseal(exclusions.KEY_PURPOSE, after["key_enc"], warn=False) + + +async def test_dry_run_does_not_rewrite_key_or_manifest(monkeypatch: pytest.MonkeyPatch) -> None: + world = _World(monkeypatch) + await world.start() + before, fence = copy.deepcopy(world.keys), dict(world.keeper) + _rotate(monkeypatch) + assert await crawl_reseal.keys(dry_run=True, ch=world) == (1, []) + assert world.keys == before and world.keeper == fence + + +async def test_lost_key_is_reported_without_reinitializing_refusals( + monkeypatch: pytest.MonkeyPatch, +) -> None: + world = _World(monkeypatch) + index = await world.start() + await world.change(index) + world.keys["org"]["key_enc"] = "unreadable-sealed-value" + before, rows, fence = copy.deepcopy(world.keys), dict(world.rows), dict(world.keeper) + _rotate(monkeypatch) + assert await crawl_reseal.keys(dry_run=False, ch=world) == (0, ["org"]) + assert world.keys == before and world.rows == rows and world.keeper == fence + + +async def test_reseal_refuses_a_different_database_client_before_queries( + monkeypatch: pytest.MonkeyPatch, +) -> None: + world = _World(monkeypatch) + with pytest.raises(RuntimeError, match="target_mismatch"): + await crawl_reseal.keys(dry_run=False, ch=object()) + assert world.queries == 0 and not world.keys diff --git a/tests/unit/test_crawl_restore.py b/tests/unit/test_crawl_restore.py new file mode 100644 index 0000000000..7591d3463d --- /dev/null +++ b/tests/unit/test_crawl_restore.py @@ -0,0 +1,375 @@ +"""Restored crawl authority ends before the real reconciliation publishes held rows.""" + +from __future__ import annotations + +import copy +import json +from collections.abc import AsyncIterator +from contextlib import asynccontextmanager +from datetime import UTC, datetime +from typing import Any + +import pytest + +from server.cli import restore as cli_restore +from server.core import leases +from server.knowledge.app import connected_restore, crawl_restore +from server.knowledge.domain.sources import KnowledgeSource +from server.knowledge.store import mirrors, source_retirements +from server.knowledge.store import sources as source_store +from server.settings import settings +from tests.unit.test_connected_account_restore import _unrelated_restore_participants +from tests.unit.test_crawl_exclusions import _World as _ExclusionWorld + +pytestmark = pytest.mark.asyncio +_MIRROR = "acl-" + "a" * 24 +_FIRST = "00000000-0000-4000-8000-000000000001" +_SECOND = "00000000-0000-4000-8000-000000000002" + + +def _sql(value: str) -> str: + return " ".join(value.split()) + + +class _HistoryWorld(_ExclusionWorld): + """Raw source versions, exact leases and only the SQL used by the real stores.""" + + def __init__(self, monkeypatch: pytest.MonkeyPatch) -> None: + super().__init__(monkeypatch) + self.sources: list[dict[str, Any]] = [] + self.retirements: list[dict[str, Any]] = [] + self.scope: tuple[str, str] | None = None + self.seeding = False + self.ignore_mutation = False + self.mutation_tags: set[str] = set() + self.executed: list[str] = [] + self.page_reads: list[tuple[str, str, str]] = [] + monkeypatch.setattr(crawl_restore, "get_client", lambda: self) + monkeypatch.setattr(source_store, "_default_client", lambda: self) + monkeypatch.setattr(mirrors, "_default_client", lambda: self) + monkeypatch.setattr(source_retirements, "_default_client", lambda: self) + monkeypatch.setattr(leases, "holding", self.source_lease) + monkeypatch.setattr(settings, "knowledge_crawl_lease_wait_s", 0) + + @asynccontextmanager + async def source_lease( + self, name: str, *, ttl_s: float, wait_s: float, ch: Any = None, settle_writes: bool + ) -> AsyncIterator[bool]: + assert ch is None or ch is self + assert settle_writes is True + prefix, org, namespace = name.split(":") + assert prefix == "knowledge-source" and org == "org" and namespace == "namespace" + assert self.scope is None and ttl_s == 60.0 and wait_s == 5.0 + self.scope = org, namespace + try: + yield True + finally: + self.scope = None + + async def seed(self, **changes: Any) -> None: + self.seeding = True + try: + source = KnowledgeSource( + **( + { + "org": "org", + "namespace": "namespace", + "name": "documents", + "kind": "gdrive", + "source_id": _FIRST, + "items": [{"type": "file", "id": "file"}], + "credential_mode": "crawl", + "access": "mirrored", + "secret_enc": "retained-service-key", + "approval_enc": "historical-approval", + "registrant_subject": "registrant", + "crawl_generation": "generation", + "crawl_customer_id": "workspace", + "crawl_credential_hash": "a" * 64, + "crawl_options": { + "directory_subject": "directory@example.com", + "subjects": [{"type": "person", "email": "person@example.com"}], + }, + } + | changes + ) + ) + await source_store.insert_source(source, ch=self) + finally: + self.seeding = False + + def _crawl_rows(self, org: str) -> list[dict[str, Any]]: + return [r for r in self.sources if r["org"] == org and r["credential_mode"] == "crawl"] + + async def query_one(self, sql: str, *, params: dict[str, Any] | None = None): + compact = _sql(sql) + if compact.startswith("SELECT count() AS n FROM loom.worker_status "): + assert params == {"s": 2 * settings.supervisor_lease_ttl_s} + return {"n": 0} + if compact.startswith("SELECT count() AS n FROM loom.knowledge_sources "): + assert params is not None + self._count_query(compact, params) + return {"n": len(self._mutation_rows(compact, params))} + if compact.startswith( + "SELECT " + source_store._COLUMNS + " FROM loom.knowledge_sources FINAL " + ): + assert params is not None and set(params) == {"org", "ns", "name"} + assert compact.endswith("AND name = {name:String} LIMIT 1") + assert self.held and self.scope == (params["org"], params["ns"]) + found = [ + r + for r in self.sources + if (r["org"], r["namespace"], r["name"]) + == (params["org"], params["ns"], params["name"]) + ] + return copy.deepcopy(found[-1]) if found else None + return await super().query_one(sql, params=params) + + async def query_all(self, sql: str, *, params: dict[str, Any]): + compact = _sql(sql) + if "FROM system.mutations" in compact: + assert set(params) == {"db", "t", "pat"} and compact.endswith("LIMIT 1") + assert params["db"] == "loom" and params["t"] == "knowledge_sources" + assert any(tag in params["pat"] for tag in self.mutation_tags) + return [ + {"mutation_id": "owned", "is_done": 1, "parts_to_do": 0, "latest_fail_reason": ""} + ] + if compact.startswith("SELECT DISTINCT org FROM (SELECT org FROM loom.knowledge_sources "): + assert params == {"limit": settings.knowledge_crawl_registry_scan_limit + 1} + assert ( + compact + == "SELECT DISTINCT org FROM (SELECT org FROM loom.knowledge_sources WHERE credential_mode = 'crawl' UNION ALL SELECT org FROM loom.knowledge_crawl_exclusion_keys) ORDER BY org LIMIT {limit:UInt32}" + ) + orgs = sorted( + {r["org"] for r in self.sources if r["credential_mode"] == "crawl"} | set(self.keys) + ) + return [{"org": org} for org in orgs[: params["limit"]]] + if compact.startswith("SELECT DISTINCT " + source_store._COLUMNS): + assert " FINAL " not in compact and set(params) == {"org", "limit"} + assert params["limit"] == settings.knowledge_crawl_registry_scan_limit + 1 + assert compact.endswith( + "AND credential_mode = 'crawl' ORDER BY namespace, name, source_id, updated_at LIMIT {limit:UInt32}" + ) + columns = source_store._COLUMNS.split(", ") + unique = {tuple(row[k] for k in columns) for row in self._crawl_rows(params["org"])} + rows = [dict(zip(columns, values, strict=True)) for values in unique] + return sorted( + rows, key=lambda r: (r["namespace"], r["name"], r["source_id"], r["updated_at"]) + )[: params["limit"]] + if compact.startswith("SELECT mirror, namespace FROM loom.knowledge_mirrors FINAL "): + assert params == {"org": "org", "namespaces": ["namespace"]} + assert compact.endswith("AND NOT deleted ORDER BY mirror") + assert self.held and self.scope == ("org", "namespace") + return [{"mirror": _MIRROR, "namespace": "namespace"}] + if compact.startswith("SELECT DISTINCT org, namespace, name FROM loom.knowledge_sources "): + return self._approval_page(compact, params) + if compact.startswith("SELECT DISTINCT toString(source_id) AS source_id, approval_enc "): + return self._approval_values(compact, params) + return await super().query_all(sql, params=params) + + def _approval_page(self, sql: str, params: dict[str, Any]): + assert ( + sql + == "SELECT DISTINCT org, namespace, name FROM loom.knowledge_sources WHERE (org, namespace, name) > {after:Tuple(String, String, String)} AND approval_enc != '' ORDER BY org, namespace, name LIMIT {limit:UInt32}" + ) + assert set(params) == {"after", "limit"} + assert params["limit"] == settings.knowledge_crawl_registry_page_size + self.page_reads.append(params["after"]) + keys = sorted( + { + (r["org"], r["namespace"], r["name"]) + for r in self.sources + if r["approval_enc"] and (r["org"], r["namespace"], r["name"]) > params["after"] + } + ) + return [ + dict(zip(("org", "namespace", "name"), k, strict=True)) for k in keys[: params["limit"]] + ] + + def _approval_values(self, sql: str, params: dict[str, Any]): + assert ( + sql + == "SELECT DISTINCT toString(source_id) AS source_id, approval_enc FROM loom.knowledge_sources WHERE org = {org:String} AND namespace = {ns:String} AND name = {name:String} AND credential_mode = 'crawl' AND approval_enc != '' LIMIT {limit:UInt32}" + ) + assert set(params) == {"org", "ns", "name", "limit"} + assert params["limit"] == settings.knowledge_crawl_registry_scan_limit + 1 + assert self.scope == (params["org"], params["ns"]) + pairs = sorted( + { + (r["source_id"], r["approval_enc"]) + for r in self._crawl_rows(params["org"]) + if r["namespace"] == params["ns"] + and r["name"] == params["name"] + and r["approval_enc"] + } + ) + return [ + {"source_id": source, "approval_enc": encoded} + for source, encoded in pairs[: params["limit"]] + ] + + def _mutation_rows(self, sql: str, params: dict[str, Any]): + if set(params) >= {"org", "ns", "id", "old", "new"}: + assert "namespace = {ns:String} AND source_id = {id:String}" in sql + assert "credential_mode = 'crawl' AND approval_enc = {old:String}" in sql + return [ + r + for r in self._crawl_rows(params["org"]) + if r["namespace"] == params["ns"] + and r["source_id"] == params["id"] + and r["approval_enc"] == params["old"] + ] + assert set(params) <= {"org", "erasure_mutation_tag"} and "org" in params + assert "org = {org:String} AND credential_mode = 'crawl'" in sql + rows = self._crawl_rows(params["org"]) + if "(active OR approval_enc != '' OR crawl_generation != '')" in sql: + rows = [r for r in rows if r["active"] or r["approval_enc"] or r["crawl_generation"]] + return rows + + def _count_query(self, sql: str, params: dict[str, Any]) -> None: + prefix = "SELECT count() AS n FROM loom.knowledge_sources WHERE " + if "old" in params: + assert sql == ( + prefix + "org = {org:String} AND namespace = {ns:String} " + "AND source_id = {id:String} AND credential_mode = 'crawl' " + "AND approval_enc = {old:String}" + ) + else: + assert sql == ( + prefix + "org = {org:String} AND credential_mode = 'crawl' " + "AND (active OR approval_enc != '' OR crawl_generation != '')" + ) + + async def execute( + self, sql: str, *, params: dict[str, Any], settings: dict[str, Any] | None = None + ): + compact = _sql(sql) + assert compact.startswith("ALTER TABLE loom.knowledge_sources UPDATE ") + self.executed.append(compact) + rows = self._mutation_rows(compact, params) + if "UPDATE approval_enc = {new:String}" in compact: + assert compact == ( + "ALTER TABLE loom.knowledge_sources UPDATE approval_enc = {new:String} " + "WHERE org = {org:String} AND namespace = {ns:String} " + "AND source_id = {id:String} AND credential_mode = 'crawl' " + "AND approval_enc = {old:String}" + ) + assert settings == {"mutations_sync": 2} + assert self.scope == (params["org"], params["ns"]) + if not self.ignore_mutation: + for row in rows: + row["approval_enc"] = params["new"] + return + assert compact == ( + "ALTER TABLE loom.knowledge_sources UPDATE active = 0, approval_enc = '', " + "crawl_generation = '' WHERE (org = {org:String} AND credential_mode = 'crawl') " + "AND ({erasure_mutation_tag:String} = {erasure_mutation_tag:String}) " + "SETTINGS mutations_sync = 0" + ) + assert "SETTINGS mutations_sync = 0" in compact and settings is None + assert self.held and self.scope is None + self.mutation_tags.add(params["erasure_mutation_tag"]) + if not self.ignore_mutation: + for row in rows: + row.update(active=False, approval_enc="", crawl_generation="") + + async def insert(self, table: str, rows: list[dict[str, Any]]) -> None: + if table == source_store.TABLE: + if not self.seeding: + assert self.held and self.scope == ("org", "namespace") + self.sources.extend(copy.deepcopy(rows)) + elif table == source_retirements.TABLE: + assert self.held and self.scope == ("org", "namespace") + for row in rows: + item = source_retirements.from_row(row) + assert (item.org, item.namespace, item.mirror) == ("org", "namespace", _MIRROR) + assert row["state_value"] == "pending" + self.retirements.extend(copy.deepcopy(rows)) + else: + await super().insert(table, rows) + + +async def _restored(monkeypatch: pytest.MonkeyPatch) -> _HistoryWorld: + world = _HistoryWorld(monkeypatch) + await world.change(await world.start()) + await world.seed(approval_enc="older-approval", crawl_generation="older") + await world.seed(approval_enc="newer-approval", crawl_generation="newer") + await world.seed(source_id=_SECOND, approval_enc="replacement-approval") + await world.seed(name="shared", credential_mode="shared", approval_enc="unrelated") + return world + + +def _participants(monkeypatch: pytest.MonkeyPatch, before_publish: Any) -> None: + _unrelated_restore_participants(monkeypatch, before_publish) + + async def no_accounts(*, apply: bool, ch: Any) -> None: + assert isinstance(ch, _HistoryWorld) + + monkeypatch.setattr(connected_restore, "restore_all", no_accounts) + + +async def test_reconcile_invalidates_every_historical_crawl_approval_before_publication( + monkeypatch, +): + world = await _restored(monkeypatch) + shared = copy.deepcopy(world.sources[-1]) + refusal_state = copy.deepcopy((world.keys, world.rows, world.keeper)) + published = [] + + def before_publish(): + assert all( + not r["active"] and not r["approval_enc"] and not r["crawl_generation"] + for r in world._crawl_rows("org") + ) + assert {json.loads(r["detail"])["source_id"] for r in world.retirements} == { + _FIRST, + _SECOND, + } + published.append(True) + + _participants(monkeypatch, before_publish) + assert await cli_restore.reconcile("clickhouse", datetime.now(UTC), apply=True, ch=world) == 0 + assert published == [True, True] + assert world.sources[3] == shared + assert all(r["secret_enc"] == "retained-service-key" for r in world.sources) + assert (world.keys, world.rows, world.keeper) == refusal_state + + +@pytest.mark.parametrize("proof", ["missing-key", "mismatched-epoch"]) +async def test_unproved_refusal_state_keeps_restored_publication_held(monkeypatch, proof): + world = await _restored(monkeypatch) + if proof == "missing-key": + world.keys.pop("org") + else: + world.keeper[next(iter(world.keeper))] = "active:another-key:another-epoch" + before = copy.deepcopy(world.sources) + published = [] + _participants(monkeypatch, lambda: published.append(True)) + assert await cli_restore.reconcile("clickhouse", datetime.now(UTC), apply=True, ch=world) == 1 + assert world.sources == before and not world.retirements and not published + + +async def test_ignored_history_invalidation_cannot_report_restored_publication_safe(monkeypatch): + world = await _restored(monkeypatch) + world.ignore_mutation = True + published = [] + _participants(monkeypatch, lambda: published.append(True)) + assert await cli_restore.reconcile("clickhouse", datetime.now(UTC), apply=True, ch=world) == 1 + assert world.executed and world.retirements and not published + assert any(r["approval_enc"] for r in world._crawl_rows("org")) + + +async def test_crawl_restore_dry_run_preserves_history_and_refusal_proof(monkeypatch): + world = await _restored(monkeypatch) + before = copy.deepcopy((world.sources, world.keys, world.rows, world.keeper)) + await crawl_restore.restore_all(apply=False, ch=world) + assert (world.sources, world.keys, world.rows, world.keeper) == before + assert not world.retirements and not world.executed + + +async def test_crawl_restore_refuses_a_different_target_before_storage_access(monkeypatch): + world = await _restored(monkeypatch) + before = copy.deepcopy((world.sources, world.retirements, world.executed, world.queries)) + with pytest.raises(RuntimeError, match="crawl_restore_target_mismatch"): + await crawl_restore.restore_all(apply=True, ch=object()) + assert (world.sources, world.retirements, world.executed, world.queries) == before diff --git a/tests/unit/test_crawl_sent_write_settlement.py b/tests/unit/test_crawl_sent_write_settlement.py new file mode 100644 index 0000000000..8d1650831c --- /dev/null +++ b/tests/unit/test_crawl_sent_write_settlement.py @@ -0,0 +1,234 @@ +"""Real SCIM and source publishers cannot outlive a competing raw-history cleanup.""" + +from __future__ import annotations + +import asyncio +import copy +from contextlib import asynccontextmanager +from typing import Any + +import pytest + +from server.auth import person_erasure, scim +from server.core import leases +from server.core.errors import OverloadedError +from server.erasure.app import crawl_person +from server.erasure.store import crawl_controls +from server.knowledge.app import crawl_exclusions +from server.knowledge.app import sources as ops +from server.knowledge.store import sources as source_store +from tests.unit.test_claim_write_settlement import Keeper +from tests.unit.test_crawl_composed_flow import _register +from tests.unit.test_crawl_composed_flow import world as world +from tests.unit.test_crawl_person_controls import _World as ControlRows +from tests.unit.test_crawl_signing_authority import key_view as key_view +from tests.unit.test_google_workspace import account as account +from tests.unit.test_scim import _create, _fresh_world + +pytestmark = [pytest.mark.asyncio, pytest.mark.claim_cap] +_REAL_HELD = scim._held +_REAL_HOLDING = leases.holding + + +class Packet: + """External receiver: an already-started bounded INSERT or a late refused send.""" + + def __init__(self, keeper: Keeper, row: dict[str, Any]): + self.row = copy.deepcopy(row) + self.deadline = keeper.send("delayed-metadata") + + def land(self, keeper: Keeper, rows: list[dict[str, Any]]) -> None: + assert keeper.clock() <= self.deadline + 5, "the server execution limit is retained" + rows.append(copy.deepcopy(self.row)) + + +async def _sent_or_failed(task: asyncio.Task[Any], sent: asyncio.Event) -> None: + signal = asyncio.create_task(sent.wait()) + try: + done, _ = await asyncio.wait({task, signal}, return_when=asyncio.FIRST_COMPLETED) + if task in done: + await task + pytest.fail("publisher returned without the targeted send") + assert sent.is_set() + finally: + signal.cancel() + await asyncio.gather(signal, return_exceptions=True) + + +async def test_cancelled_real_scim_prepared_insert_blocks_finalization_until_safe_expiry( + monkeypatch: pytest.MonkeyPatch, +): + with _fresh_world() as state: + uid = await _create("a@example.com") + keeper = Keeper(monkeypatch) + original_insert = state.table.insert + sent = asyncio.Event() + packet: Packet | None = None + + @asynccontextmanager + async def held(name: str): + async with _REAL_HELD(name): + state.holding += 1 + try: + yield + finally: + state.holding -= 1 + + async def insert(table: str, rows: list[dict[str, Any]]): + nonlocal packet + if table == scim.TABLE and rows[0]["crawl_erasure_enc"] and packet is None: + capture = person_erasure.load(uid, rows[0]["crawl_erasure_enc"]) + if capture.prepared: + packet = Packet(keeper, rows[0]) + sent.set() + await asyncio.Future() # The client is cancelled; the receiver still owns it. + await original_insert(table, rows) + + original_one, original_all = state.table.query_one, state.table.query_all + + async def one(sql: str, params=None): + if sql.startswith("SELECT count() AS n FROM loom.scim_users"): + assert params == {"id": uid} and "FINAL" not in sql + return { + "n": sum( + row["id"] == uid + and (bool(row["crawl_erasure_enc"]) or not row["crawl_erasure_done"]) + for row in state.table.rows + ) + } + return await original_one(sql, params) + + async def all_rows(sql: str, params): + if sql.startswith("SELECT DISTINCT crawl_erasure_enc FROM loom.scim_users"): + assert params["id"] == uid and "FINAL" not in sql + return [ + {"crawl_erasure_enc": value} + for value in sorted( + { + row["crawl_erasure_enc"] + for row in state.table.rows + if row["id"] == uid and row["crawl_erasure_enc"] + } + ) + ][: params["limit"]] + if "FROM system.mutations" in sql: + return [ + {"is_done": 1, "parts_to_do": 0, "latest_fail_reason": "", "mutation_id": "own"} + ] + return await original_all(sql, params) + + async def execute(sql: str, *, params): + assert "UPDATE crawl_erasure_enc = ''" in sql and params["id"] == uid + for row in state.table.rows: + if row["id"] == uid: + row.update( + crawl_erasure_enc="", crawl_erasure_done=True, crawl_erasure_required=True + ) + + monkeypatch.setattr(scim, "_held", held) + monkeypatch.setattr(state.table, "insert", insert) + monkeypatch.setattr(state.table, "query_one", one) + monkeypatch.setattr(state.table, "query_all", all_rows) + monkeypatch.setattr(state.table, "execute", execute, raising=False) + monkeypatch.setattr(person_erasure, "_finalizer", crawl_person.finalize) + monkeypatch.setattr(crawl_controls, "_default_client", lambda: state.table) + task = asyncio.create_task(scim.delete(uid)) + await _sent_or_failed(task, sent) + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + assert packet is not None + assert not state.table.newest(uid)["crawl_erasure_done"] + with pytest.raises(scim.Busy): + await scim.delete(uid) + assert not state.person_cleanup.finalized + keeper.clock.advance(19) + packet.land(keeper, state.table.rows) + foreign = {**packet.row, "id": "unrelated", "crawl_erasure_enc": "foreign-capsule"} + state.table.rows.append(copy.deepcopy(foreign)) + keeper.clock.advance(42) + await scim.delete(uid) + assert all( + not row["crawl_erasure_enc"] and row["crawl_erasure_done"] + for row in state.table.rows + if row["id"] == uid + ) + assert state.table.newest(uid)["crawl_erasure_done"] + assert [row for row in state.table.rows if row["id"] == "unrelated"] == [foreign] + + +async def test_cancelled_real_registration_cannot_append_after_actual_source_cleanup( + world, + monkeypatch: pytest.MonkeyPatch, +): + keeper = Keeper(monkeypatch) + original_insert = world.insert + sent = asyncio.Event() + packet: Packet | None = None + + @asynccontextmanager + async def holding(key: str, **kwargs: Any): + async with _REAL_HOLDING(key, **kwargs) as acquired: + if acquired: + async with world.lease(key): + yield True + else: + yield False + + async def insert(table: str, rows: list[dict[str, Any]]): + nonlocal packet + if table == source_store.TABLE and packet is None: + world.assert_storage_leases() + packet = Packet(keeper, rows[0]) + sent.set() + await asyncio.Future() + await original_insert(table, rows) + + monkeypatch.setattr(leases, "holding", holding) + monkeypatch.setattr(world, "insert", insert) + task = asyncio.create_task(_register(world)) + await _sent_or_failed(task, sent) + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + assert packet is not None + controls = ControlRows() + controls.rows = [] + complete = False + + async def clean(): + nonlocal complete + async with crawl_exclusions.org_lease("team", ttl_s=60, wait_s=5): + async with ops.sources_lease("team", "shared-space"): + await crawl_controls.controls( + "team", + "shared-space", + str(packet.row["source_id"]), + packet.row["registrant_subject"], + frozenset({"one@example.com", "person@example.com"}), + spellings=(packet.row["registrant_subject"], "person@example.com"), + ch=controls, + ) + complete = True + + with pytest.raises(OverloadedError): + await clean() + assert not complete and not controls.executed + keeper.clock.advance(19) + packet.land(keeper, controls.rows) + foreign = {**packet.row, "namespace": "other-space"} + controls.rows.append(copy.deepcopy(foreign)) + keeper.clock.advance(42) + await clean() + assert complete + [erased] = [row for row in controls.rows if row["namespace"] == "shared-space"] + assert not erased["active"] and not erased["approval_enc"] and not erased["crawl_generation"] + assert not erased["registrant_subject"] + expected_creator = ( + "" + if packet.row["created_by"] in {packet.row["registrant_subject"], "person@example.com"} + else packet.row["created_by"] + ) + assert erased["created_by"] == expected_creator + assert "one@example.com" not in erased["options_json"] + assert [row for row in controls.rows if row["namespace"] == "other-space"] == [foreign] diff --git a/tests/unit/test_crawl_signing_authority.py b/tests/unit/test_crawl_signing_authority.py new file mode 100644 index 0000000000..7290515f51 --- /dev/null +++ b/tests/unit/test_crawl_signing_authority.py @@ -0,0 +1,89 @@ +"""A retained registration remembers the key that actually verified its claim.""" + +from __future__ import annotations + +import time + +import jwt +import pytest + +from server.auth import jwt as auth_jwt +from server.auth import keyring, revocation +from server.core import cell +from server.settings import settings + +_FIRST = "first-signing-material-with-more-than-thirty-two-bytes" +_SECOND = "second-signing-material-with-more-than-thirty-two-bytes" + + +def _token(secret: str) -> str: + return jwt.encode( + { + "sub": "registrant", + "org": "org", + "namespaces": ["team"], + "roles": ["org_admin"], + "sa": False, + "iss": settings.jwt_issuer, + "aud": settings.jwt_audience, + "iat": int(time.time()), + "jti": "registration-token", + }, + secret, + algorithm="HS256", + headers={"kid": "signer"}, + ) + + +@pytest.fixture +def key_view(monkeypatch: pytest.MonkeyPatch) -> list[str]: + # External snapshots are healthy; the signature/claim verifier remains real. + keys = [_FIRST, _SECOND] + monkeypatch.setattr(cell, "gate_reason", lambda: "") + monkeypatch.setattr(keyring, "loaded", lambda: True) + monkeypatch.setattr(keyring, "stale", lambda: False) + monkeypatch.setattr(keyring, "verification_keys", lambda kid, algorithm: list(keys)) + monkeypatch.setattr(revocation, "loaded", lambda: True) + monkeypatch.setattr(revocation, "snapshot_age_seconds", lambda: 0) + monkeypatch.setattr(revocation, "is_revoked", lambda *args, **kwargs: False) + return keys + + +def test_retained_authority_pins_successful_key_not_first_same_kid_candidate( + key_view: list[str], +) -> None: + auth = auth_jwt.verify_token(_token(_SECOND)) + authority = auth.signing_authority + assert authority is not None and authority.current() + assert (authority.issuer, authority.kid, authority.algorithm) == ( + settings.jwt_issuer, + "signer", + "HS256", + ) + key_view.remove(_SECOND) + assert not authority.current() # A different still-valid key under this kid cannot inherit it. + assert _FIRST not in repr(auth) and _SECOND not in repr(auth) + + +@pytest.mark.parametrize("failure", ["unloaded", "stale", "untrusted-issuer"]) +def test_retained_authority_requires_current_signing_and_issuer_configuration( + monkeypatch: pytest.MonkeyPatch, + key_view: list[str], + failure: str, +) -> None: + authority = auth_jwt.verify_token(_token(_SECOND)).signing_authority + assert authority is not None + if failure == "unloaded": + monkeypatch.setattr(keyring, "loaded", lambda: False) + elif failure == "stale": + monkeypatch.setattr(keyring, "stale", lambda: True) + else: + monkeypatch.setattr(settings, "jwt_issuer", "changed-issuer") + assert not authority.current() + + +def test_explicit_offline_secret_verification_creates_no_persistable_authority( + key_view: list[str], +) -> None: + auth = auth_jwt.verify_token(_token(_SECOND), secret=_SECOND) + assert auth.subject == "registrant" and auth.signing_authority is None diff --git a/tests/unit/test_crawl_source_state.py b/tests/unit/test_crawl_source_state.py new file mode 100644 index 0000000000..75d77c9b80 --- /dev/null +++ b/tests/unit/test_crawl_source_state.py @@ -0,0 +1,155 @@ +"""Real source row construction and complete multi-person union contracts.""" + +from __future__ import annotations + +import dataclasses +import json +from typing import Any + +import pytest + +from server.knowledge.domain.source_crawl import Expansion, Options, Person, Union +from server.knowledge.domain.sources import KnowledgeSource, mirror_synced +from server.knowledge.store import sources + + +def _options() -> dict[str, Any]: + return { + "directory_subject": "directory@example.com", + "subjects": [{"type": "group", "email": "team@example.com"}], + } + + +@pytest.mark.parametrize( + "change", + [ + {"directory_subject": "person@example.com\n"}, + {"subjects": []}, + {"subjects": [{"type": "customer", "email": "all@example.com"}]}, + {"subjects": [{"type": "person", "email": "person@example.com", "id": "guessed"}]}, + {"subjects": [{"type": "person", "email": "person@example.com"}] * 2}, + {"query": "all users"}, + ], +) +def test_crawl_options_refuse_untyped_or_ambiguous_admission(change: dict[str, Any]) -> None: + with pytest.raises(ValueError): + Options.parse(_options() | change) + + +@pytest.mark.asyncio +async def test_real_source_row_roundtrip_keeps_separate_crawl_authority_and_never_serves_it() -> ( + None +): + written: list[dict[str, Any]] = [] + + class Store: + async def insert(self, table: str, rows: list[dict[str, Any]]) -> None: + assert table == sources.TABLE + written.extend(rows) + + source = KnowledgeSource( + org="org", + namespace="team", + name="meeting-notes", + kind="gdrive", + source_id="a8895a80-d41c-4515-8c89-57fc1f7b4994", + config={"include": ["docs"]}, + credential_mode="crawl", + access="mirrored", + items=[{"type": "file", "id": "file-identifier"}], + crawl_options=_options(), + registrant_subject="canonical-admin", + approval_enc="sealed-approval", + crawl_generation="registration-generation", + crawl_customer_id="verified-workspace", + crawl_credential_hash="a" * 64, + secret_enc="sealed-delegated-key", + created_by="audit-principal", + ) + await sources.insert_source(source, ch=Store()) + (row,) = written + assert ( + row["kind"] == row["credential_enc"] == row["owner_subject"] == row["connection_id"] == "" + ) + assert json.loads(row["options_json"]) == _options() + restored = sources._source(row) + assert dataclasses.replace(restored, created_at=None, updated_at=None) == source + assert mirror_synced(restored) # Crawl writes remain confined to the checked mirror. + assert restored.registrant_subject != restored.created_by + + +def _people() -> tuple[Person, Person]: + return Person("customer", "one", "one@example.com", ()), Person( + "customer", "two", "two@example.com", () + ) + + +def _file(**changes: Any) -> dict[str, Any]: + return {"id": "file", "version": "1", "mimeType": "text/plain", **changes} + + +def test_one_person_absence_cannot_retire_another_persons_file() -> None: + one, two = _people() + union = Union(Expansion("customer", (one, two), (), "exclusions"), 100) + union.add(one, [_file(owners=[{"me": True, "emailAddress": one.email}])], whole=True) + with pytest.raises(ValueError, match="incomplete"): + union.complete() + union.add(two, [], whole=True) + (file,) = union.complete() + assert (file.id, file.access.emails, file.presenters) == ("file", (one.email,), (one.id,)) + assert file.access.owner == "" + + +def test_complete_union_uses_explicit_permissions_not_link_only_listing_as_person() -> None: + one, two = _people() + union = Union(Expansion("customer", (one, two), (), "exclusions"), 100) + union.add( + one, + [ + _file( + permissions=[ + {"type": "user", "role": "reader", "emailAddress": one.email}, + {"type": "anyone", "role": "reader", "allowFileDiscovery": False}, + ] + ) + ], + whole=True, + ) + union.add(two, [_file(permissions=[])], whole=True) + (file,) = union.complete() + assert file.presenters == (one.id, two.id) + assert file.access.emails == (one.email,) + assert not file.access.anyone and not file.access.owner + + +@pytest.mark.parametrize("change", [{"version": "2"}, {"modifiedTime": "new"}]) +def test_union_refuses_an_acl_observed_on_a_different_content_revision( + change: dict[str, Any], +) -> None: + one, two = _people() + union = Union(Expansion("customer", (one, two), (), "exclusions"), 100) + union.add(one, [_file(permissions=[])], whole=True) + with pytest.raises(ValueError, match="changed"): + union.add(two, [_file(permissions=[], **change)], whole=True) + with pytest.raises(ValueError, match="incomplete"): + union.complete() + + +def test_truncated_listing_never_completes_or_publishes_a_partial_union() -> None: + one, two = _people() + union = Union(Expansion("customer", (one, two), (), "exclusions"), 100) + with pytest.raises(ValueError, match="incomplete"): + union.add(one, [_file()], whole=False) + union.add(two, [], whole=True) + with pytest.raises(ValueError, match="incomplete"): + union.complete() + + +def test_global_file_bound_applies_to_the_union_not_each_person_separately() -> None: + one, two = _people() + union = Union(Expansion("customer", (one, two), (), "exclusions"), 1) + union.add(one, [_file(id="first")], whole=True) + with pytest.raises(ValueError, match="file bound"): + union.add(two, [_file(id="second")], whole=True) + with pytest.raises(ValueError, match="incomplete"): + union.complete() diff --git a/tests/unit/test_crawl_storage_contracts.py b/tests/unit/test_crawl_storage_contracts.py new file mode 100644 index 0000000000..360e82e707 --- /dev/null +++ b/tests/unit/test_crawl_storage_contracts.py @@ -0,0 +1,138 @@ +"""Every crawl control and membership version carries its complete declared row.""" + +from __future__ import annotations + +import copy +from datetime import UTC, datetime +from typing import Any + +import pytest + +from server.knowledge.domain.source_crawl import Expansion +from server.knowledge.domain.sources import KnowledgeSource +from server.knowledge.store import crawl_exclusions as exclusions +from server.knowledge.store import crawl_publication as publication +from tests.unit.schema_ddl import columns + +pytestmark = pytest.mark.asyncio + + +class _Rows: + def __init__(self) -> None: + self.version = 100 + self.inserted: list[tuple[str, list[dict[str, Any]]]] = [] + + async def query_one(self, sql: str, *, params: dict[str, Any]) -> dict[str, int]: + assert sql.startswith("SELECT greatest(") and "max(version) + 1" in sql + assert params["k0"] == "org" + self.version += 1 + return {"v": self.version} + + async def query_all(self, sql: str, *, params: dict[str, Any]): + assert "FROM loom.knowledge_group_members FINAL" in sql + assert params == {"org": "org", "ns": "mirror", "id": "source", "limit": 5} + return [{"group": "old@example.com"}] + + async def insert(self, table: str, rows: list[dict[str, Any]]) -> None: + assert all(set(row) == set(columns(table.removeprefix("loom."))) for row in rows) + self.inserted.append((table, copy.deepcopy(rows))) + + +async def test_refusal_key_rewrite_preserves_full_pending_manifest() -> None: + rows = _Rows() + await exclusions.key_put( + "org", + "a" * 64, + "sealed-key", + generation="generation", + pending_id="b" * 32, + pending_digests=("c" * 64,), + pending_exclude=True, + ch=rows, + ) + assert rows.inserted == [ + ( + exclusions.KEYS, + [ + { + "org": "org", + "key_id": "a" * 64, + "key_enc": "sealed-key", + "generation": "generation", + "pending_id": "b" * 32, + "pending_digests": ["c" * 64], + "pending_exclude": True, + "version": 101, + } + ], + ) + ] + + +async def test_refusal_and_explicit_readmission_each_write_complete_newer_rows() -> None: + rows = _Rows() + await exclusions.put("org", ["a" * 64, "a" * 64], exclude=True, ch=rows) + await exclusions.put("org", ["a" * 64], exclude=False, ch=rows) + assert rows.inserted == [ + ( + exclusions.REFUSALS, + [{"org": "org", "digest": "a" * 64, "excluded": True, "version": 101}], + ), + ( + exclusions.REFUSALS, + [{"org": "org", "digest": "a" * 64, "excluded": False, "version": 102}], + ), + ] + + +async def test_complete_group_replacement_clears_removed_members_without_losing_columns() -> None: + rows = _Rows() + source = KnowledgeSource("org", "namespace", "name", "gdrive", "source", mirror="mirror") + now = datetime.now(UTC) + expansion = Expansion("customer", (), (("new@example.com", ("person@example.com",)),), "fence") + await publication.groups(source, expansion, now, maximum=4, ch=rows) + assert rows.inserted == [ + ( + publication.GROUPS, + [ + { + "org": "org", + "namespace": "mirror", + "source_id": "source", + "group": "new@example.com", + "name": "name", + "members": ["person@example.com"], + "read_at": now, + "deleted": False, + "version": 101, + }, + { + "org": "org", + "namespace": "mirror", + "source_id": "source", + "group": "old@example.com", + "name": "name", + "members": [], + "read_at": now, + "deleted": True, + "version": 102, + }, + ], + ) + ] + + +def test_canonical_person_axis_is_explicit_and_disjoint_from_namespace_memory() -> None: + from server.erasure.store import census + + assert census.CANONICAL_PERSON_TABLES == { + "loom.scim_users": "crawl_person.finalize", + "loom.knowledge_crawl_exclusion_keys": "crawl_person.exclude", + "loom.knowledge_crawl_exclusions": "crawl_person.exclude", + } + for table in census.CANONICAL_PERSON_TABLES: + assert table not in census.NO_MEMORY_CONTENT + assert table not in census.CANONICAL_OWNER_TABLES + assert not any(substrate.table == table for substrate in census._CENSUS) + assert not any(table in tables for tables in census.STEP_ONLY_TABLES.values()) + assert census.classified_tables()[table] == "canonical_person" diff --git a/tests/unit/test_crawl_worker_observation.py b/tests/unit/test_crawl_worker_observation.py new file mode 100644 index 0000000000..8a6897aa0f --- /dev/null +++ b/tests/unit/test_crawl_worker_observation.py @@ -0,0 +1,137 @@ +"""Native item listings build one bounded observation before any publication.""" + +from __future__ import annotations + +import dataclasses +import time +from typing import Any + +import httpx +import pytest + +from server.clients._vendor import CredentialRefused, TooLarge +from server.code.app.index.pass_budget import PassBudget +from server.knowledge.app import crawl_authority, source_sync_crawl +from server.knowledge.app import source_sync as sync +from tests.unit.test_crawl_authority import _BOUNDS, _World +from tests.unit.test_crawl_authority import account as account +from tests.unit.test_crawl_authority import crawl_world as crawl_world +from tests.unit.test_crawl_authority import key_view as key_view + +pytestmark = [pytest.mark.asyncio, pytest.mark.claim_cap] + + +def _file(id: str = "file") -> dict[str, Any]: + return { + "id": id, + "version": "1", + "name": "Private meeting", + "mimeType": "application/vnd.google-apps.document", + "modifiedTime": "2026-01-01T00:00:00.000Z", + "webViewLink": "https://docs.google.com/document/d/" + id, + "permissions": [], + "owners": [{"me": True, "emailAddress": "one@example.com"}], + } + + +def _run() -> sync.Pass: + now = time.monotonic() + return sync.Pass(PassBudget(now, now + 10), 10) + + +async def _authority(world: _World) -> crawl_authority.Authority: + assert world.source is not None + return await crawl_authority.resolve( + world.source, deadline=time.monotonic() + 10, bounds=_BOUNDS + ) + + +async def test_actual_native_single_item_observation_keeps_ownership_without_owner_arm( + crawl_world: _World, +) -> None: + original = crawl_world.handle + + async def provider(request: httpx.Request) -> httpx.Response: + if request.url.path == "/drive/v3/files/file": + assert request.url.params["fields"].endswith("owners(me,emailAddress),trashed") + crawl_world.native_requests.append(request) + return httpx.Response(200, json=_file()) + return await original(request) + + crawl_world.handle = provider + authority = await _authority(crawl_world) + expansion, files = await source_sync_crawl.observe(authority, _run(), maximum_files=2) + (file,) = files + assert file.id == "file" and file.access.emails == ("one@example.com",) + assert not file.access.owner and expansion.people[0].id == "provider-one" + assert len(crawl_world.native_requests) == 1 + assert not crawl_world.writes # No memory, ACL, group, listing or retirement writes. + + +async def test_current_source_context_is_required_and_cleared_on_failed_work( + crawl_world: _World, +) -> None: + authority = await _authority(crawl_world) + expansion = await authority.prepare() + assert crawl_world.source is not None + with pytest.raises(CredentialRefused, match="checkout unavailable"): + await sync.require_current(crawl_world.source) + with pytest.raises(RuntimeError, match="extraction failed"): + async with authority.using(expansion.people): + await sync.require_current(crawl_world.source) + raise RuntimeError("extraction failed") + with pytest.raises(CredentialRefused, match="checkout unavailable"): + await sync.require_current(crawl_world.source) + + +@pytest.mark.parametrize("incomplete", [True, False]) +async def test_incomplete_search_or_global_file_cap_never_produces_union( + crawl_world: _World, + incomplete: bool, +) -> None: + assert crawl_world.source is not None + crawl_world.source = dataclasses.replace( + crawl_world.source, items=[{"type": "filter", "kinds": ["docs"]}] + ) + await crawl_world.setup() + original = crawl_world.handle + + async def provider(request: httpx.Request) -> httpx.Response: + if request.url.path == "/drive/v3/files": + crawl_world.native_requests.append(request) + second = "pageToken" in request.url.params + return httpx.Response( + 200, + json={ + "files": [_file("second" if second else "first")], + "incompleteSearch": incomplete, + **({} if incomplete or second else {"nextPageToken": "next"}), + }, + ) + return await original(request) + + crawl_world.handle = provider + with pytest.raises(ValueError if incomplete else TooLarge): + await source_sync_crawl.observe(await _authority(crawl_world), _run(), maximum_files=1) + assert len(crawl_world.native_requests) == (1 if incomplete else 2) + assert not crawl_world.writes + + +async def test_native_response_after_registrant_loss_is_not_a_complete_observation( + crawl_world: _World, +) -> None: + from server.operations.base import Role + + original = crawl_world.handle + + async def provider(request: httpx.Request) -> httpx.Response: + if request.url.path == "/drive/v3/files/file": + crawl_world.native_requests.append(request) + crawl_world.grant = Role.WRITER + return httpx.Response(200, json=_file()) + return await original(request) + + crawl_world.handle = provider + with pytest.raises(PermissionError): + await source_sync_crawl.observe(await _authority(crawl_world), _run(), maximum_files=2) + assert len(crawl_world.native_requests) == 1 and not crawl_world.writes diff --git a/tests/unit/test_crawl_worker_regressions.py b/tests/unit/test_crawl_worker_regressions.py new file mode 100644 index 0000000000..c863af1723 --- /dev/null +++ b/tests/unit/test_crawl_worker_regressions.py @@ -0,0 +1,332 @@ +"""Native crawl turns retain absence proofs, pass bounds and mirror retry state.""" + +from __future__ import annotations + +import dataclasses +import json +import time +from datetime import UTC, datetime +from types import SimpleNamespace +from typing import Any +from urllib.parse import parse_qs + +import httpx +import jwt +import pytest + +from server.clients import gdrive +from server.knowledge.app import crawl_authority, private_sync, source_sync_crawl +from server.knowledge.app import source_sync as sync +from server.knowledge.domain.sources import SYNC_WORKER_ACTOR, mirror_uri +from server.knowledge.store import documents +from server.memory.domain.provenance import pipeline_actor +from server.settings import settings +from tests.unit.principals import acting_as +from tests.unit.test_connected_source_publication import _MIRROR +from tests.unit.test_crawl_authority import _BOUNDS +from tests.unit.test_crawl_composed_flow import _register, _World +from tests.unit.test_crawl_composed_flow import account as account +from tests.unit.test_crawl_composed_flow import key_view as key_view +from tests.unit.test_crawl_composed_flow import world as world +from tests.unit.test_crawl_worker_observation import _file + +pytestmark = pytest.mark.asyncio + + +async def _work( + world: _World, *, left: int = 10, asked: dict[str, Any] | None = None +) -> tuple[sync.TurnResult, sync.Pass]: + assert world.source is not None + now = time.monotonic() + run = sync.Pass(sync.PassBudget(now, now + 20), left) + with acting_as(None), pipeline_actor(SYNC_WORKER_ACTOR), private_sync.scope(): + result = await source_sync_crawl.turn( + world.source, {"detail": json.dumps(asked or {})}, run, bounds=_BOUNDS + ) + return result, run + + +async def _persist(world: _World, result: sync.TurnResult) -> dict[str, str]: + assert world.source is not None and result.mirror == _MIRROR + source = dataclasses.replace(world.source, mirror=_MIRROR) + authority = await crawl_authority.resolve( + source, deadline=time.monotonic() + 20, bounds=_BOUNDS + ) + expansion = await authority.prepare() + old = await sync.mirror_state(source, _MIRROR) + with acting_as(None), pipeline_actor(SYNC_WORKER_ACTOR), private_sync.scope(): + async with authority.using(expansion.people), sync.publication_guard(source): + await sync._write_mirrored_state(source, {}, result, {}, old) + return await sync.mirror_state(source, _MIRROR) + + +def _copies(world: _World, monkeypatch: pytest.MonkeyPatch, count: int) -> list[str]: + assert world.source is not None + rows = [ + { + "file_id": "missing-" + str(i), + "source_uri": mirror_uri( + "https://docs.google.com/document/d/missing-" + str(i), world.source.source_id + ), + "searchable": True, + "source_revision": "old", + "access_json": "{}", + } + for i in range(count) + ] + + async def every_copy(org: str, mirror: str, tag: str): + assert world.source is not None + assert (org, mirror, tag) == ("team", _MIRROR, "source-id:" + world.source.source_id) + return rows + + checked: list[str] = [] + + async def foreign_document(org: str, namespace: str, uri: str, *, include_superseded: bool): + assert (org, namespace, include_superseded) == ("team", _MIRROR, False) + checked.append(uri) + # Retirement still runs its real ownership validator and preserves a foreign latest row. + return SimpleNamespace(actor="another-writer", tags=[]) + + monkeypatch.setattr(sync.store, "every_copy", every_copy) + monkeypatch.setattr(documents, "get_one", foreign_document) + return checked + + +async def test_missing_page_file_requires_native_positive_absence( + world: _World, monkeypatch: pytest.MonkeyPatch +) -> None: + await _register(world) + checked = _copies(world, monkeypatch, 1) + original = world.handle + lookups: list[str] = [] + + async def provider(request: httpx.Request) -> httpx.Response: + if request.url.host == "www.googleapis.com": + if request.url.path == "/drive/v3/files": + return httpx.Response(200, json={"files": [], "incompleteSearch": False}) + if request.url.path == "/drive/v3/files/missing-0": + lookups.append(request.url.path) + return httpx.Response(200, json=_file("missing-0")) + return await original(request) + + world.handle = provider + result, _ = await _work(world) + assert lookups == ["/drive/v3/files/missing-0"] + assert result.complete and not result.removed + assert ( + not checked + ) # Still reached by the configured docs filter; never considered for deletion. + + +@pytest.mark.parametrize("left", [0, 1]) +async def test_absence_checks_and_retire_attempts_share_the_document_budget( + world: _World, monkeypatch: pytest.MonkeyPatch, left: int +) -> None: + await _register(world) + checked = _copies(world, monkeypatch, 3) + original = world.handle + lookups: list[str] = [] + + async def provider(request: httpx.Request) -> httpx.Response: + if request.url.host == "www.googleapis.com": + if request.url.path == "/drive/v3/files": + return httpx.Response(200, json={"files": []}) + if request.url.path.startswith("/drive/v3/files/missing-"): + lookups.append(request.url.path) + return httpx.Response(404, json={"error": "notFound"}) + return await original(request) + + world.handle = provider + result, run = await _work(world, left=left) + assert run.left == 0 and result.cut and not result.complete + assert len(lookups) == len(checked) == left + assert not result.removed and not world.listings + + +async def test_empty_folder_query_needs_current_selected_folder_visibility(world: _World) -> None: + await _register(world) + assert world.source is not None + world.source = dataclasses.replace( + world.source, items=[{"type": "filter", "kinds": ["docs"], "in": "selected-folder"}] + ) + await world.setup() # Seal approval for the actual changed selection. + original = world.handle + checked: list[str] = [] + + async def provider(request: httpx.Request) -> httpx.Response: + if request.url.host == "www.googleapis.com": + if request.url.path == "/drive/v3/files": + return httpx.Response(200, json={"files": []}) + if request.url.path == "/drive/v3/files/selected-folder": + checked.append(request.url.path) + return httpx.Response(404, json={"error": "notFound"}) + return await original(request) + + world.handle = provider + result, _ = await _work(world) + assert checked == ["/drive/v3/files/selected-folder"] + assert not result.complete and not world.listings and not world.document_writes + + +async def test_consecutive_failed_exports_resume_the_mirror_wait_counter(world: _World) -> None: + await _register(world) + original = world.handle + + async def provider(request: httpx.Request) -> httpx.Response: + if request.url.path == "/drive/v3/files/file/export": + return httpx.Response(500, json={"error": "temporary"}) + return await original(request) + + world.handle = provider + first, _ = await _work(world) + assert first.waits["file"][1] == 1 and not first.complete + saved = await _persist(world, first) + assert json.loads(saved["detail"])["file_waits"]["file"][1] == 1 + second, _ = await _work(world) + assert second.waits["file"][1] == 2 and not second.complete + assert not world.document_writes + + +@pytest.mark.parametrize("explicit_full", [False, True]) +async def test_whole_enumeration_does_not_regrant_a_settled_revision_before_full_due( + world: _World, explicit_full: bool +) -> None: + await _register(world) + assert world.source is not None + now = datetime.now(UTC).replace(microsecond=0).isoformat() + detail = { + "full_listed_at": now, + "file_waits": {"file": [_file()["modifiedTime"], 3, "too_large"]}, + } + world.states.append( + { + "org": "team", + "namespace": _MIRROR, + "state_key": sync.store.state_key(world.source.source_id), + "state_value": "", + "detail": json.dumps(detail), + } + ) + original = world.handle + exports: list[str] = [] + + async def provider(request: httpx.Request) -> httpx.Response: + if request.url.path == "/drive/v3/files/file/export": + exports.append(request.url.path) + return await original(request) + + world.handle = provider + asked = ( + {"requested_at": "explicit-request", "full": True} + if explicit_full + else {"file_waits": {}, "full_listed_at": "forged-namespace-state"} + ) + result, _ = await _work(world, asked=asked) + assert result.documents_seen == 1 # Both paths still enumerate the whole union. + assert result.full is explicit_full + assert len(exports) == int(explicit_full) + if explicit_full: + assert result.waits_reset_for == "explicit-request" and result.changed == 1 + else: + assert result.waits == detail["file_waits"] and result.changed == 0 + + +async def test_one_presenters_absence_cannot_retire_another_presenters_reached_file( + world: _World, monkeypatch: pytest.MonkeyPatch +) -> None: + await _register(world) + assert world.source is not None + world.source = dataclasses.replace( + world.source, + crawl_options={ + **world.source.crawl_options, + "subjects": [ + {"type": "person", "email": "one@example.com"}, + {"type": "person", "email": "two@example.com"}, + ], + }, + ) + await world.setup() + checked = _copies(world, monkeypatch, 1) + original = world.handle + lookups: list[str] = [] + + async def provider(request: httpx.Request) -> httpx.Response: + if request.url.host == "admin.googleapis.com" and request.url.path.rsplit("/", 1)[1] in { + "two@example.com", + "provider-two", + }: + return httpx.Response( + 200, + json={ + "id": "provider-two", + "primaryEmail": "two@example.com", + "aliases": [], + "customerId": "workspace", + "suspended": False, + "archived": False, + }, + ) + if request.url.host == "oauth2.googleapis.com": + form = parse_qs(request.content.decode()) + claims = jwt.decode( + form["assertion"][0], + world.account[1], + algorithms=["RS256"], + audience="https://oauth2.googleapis.com/token", + ) + if claims["sub"] == "two@example.com": + assert claims["scope"] == gdrive.ReadScope.DRIVE.value + return httpx.Response( + 200, + json={ + "access_token": "second-person-access", + "token_type": "Bearer", + "expires_in": 60, + }, + ) + if request.url.host == "www.googleapis.com": + if request.url.path == "/drive/v3/files": + return httpx.Response(200, json={"files": []}) + if request.url.path == "/drive/v3/files/missing-0": + bearer = request.headers["authorization"] + lookups.append(bearer) + if bearer == "Bearer private-person-access": + return httpx.Response(404, json={"error": "notFound"}) + assert bearer == "Bearer second-person-access" + return httpx.Response(200, json=_file("missing-0")) + return await original(request) + + world.handle = provider + result, _ = await _work(world) + assert lookups == ["Bearer private-person-access", "Bearer second-person-access"] + assert result.complete and not result.removed and not checked + + +async def test_configured_file_bound_refuses_the_whole_native_turn_before_publication( + world: _World, + monkeypatch: pytest.MonkeyPatch, +) -> None: + await _register(world) + monkeypatch.setattr(settings, "knowledge_crawl_max_files", 1) + original = world.handle + listed = [] + + async def provider(request: httpx.Request) -> httpx.Response: + if request.url.host == "www.googleapis.com" and request.url.path == "/drive/v3/files": + listed.append(request) + return httpx.Response( + 200, + json={ + "files": [_file("first"), _file("second")], + "incompleteSearch": False, + }, + ) + return await original(request) + + world.handle = provider + result, _ = await _work(world) + assert listed and result.status != "ok" and not result.complete + assert not world.document_writes and not world.chunk_writes and not world.listings + assert not world.embedded diff --git a/tests/unit/test_every_table_is_classified.py b/tests/unit/test_every_table_is_classified.py index ecfefeae46..3f106013cf 100644 --- a/tests/unit/test_every_table_is_classified.py +++ b/tests/unit/test_every_table_is_classified.py @@ -1,6 +1,6 @@ """Every `loom.*` and `loom_telemetry.*` table is a census row, a documented -step-only table, canonical-owner cleanup or a NO_MEMORY_CONTENT entry with a reason. A CREATE TABLE -in schema.sql that is in none of the four lists fails +step-only table, canonical-owner or canonical-person cleanup, or NO_MEMORY_CONTENT entry. +A CREATE TABLE in schema.sql that is in none of the five lists fails here, so a new table is classified the day it lands. CH-free.""" from __future__ import annotations @@ -28,7 +28,7 @@ def test_every_schema_table_is_classified() -> None: classified = erasure_census.classified_tables() missing = sorted(_schema_tables() - set(classified)) assert not missing, ( - "tables in schemas/schema.sql that are in neither _CENSUS, STEP_ONLY_TABLES, CANONICAL_OWNER_TABLES nor " + "tables in schemas/schema.sql that are in neither _CENSUS, STEP_ONLY_TABLES, CANONICAL_OWNER_TABLES, CANONICAL_PERSON_TABLES nor " f"NO_MEMORY_CONTENT (server/erasure/store/census.py): {missing}" ) @@ -39,14 +39,22 @@ def test_the_registry_names_only_tables_that_exist() -> None: assert not {s.table for s in erasure_census._CENSUS} - tables assert not set(sum(erasure_census.STEP_ONLY_TABLES.values(), ())) - tables assert not set(erasure_census.CANONICAL_OWNER_TABLES) - tables + assert not set(erasure_census.CANONICAL_PERSON_TABLES) - tables assert not set(erasure_census.NO_MEMORY_CONTENT) - tables -def test_the_four_lists_are_disjoint_and_every_reason_is_written() -> None: +def test_the_five_lists_are_disjoint_and_every_reason_is_written() -> None: census = {s.table for s in erasure_census._CENSUS} step_only = set(sum(erasure_census.STEP_ONLY_TABLES.values(), ())) no_content = set(erasure_census.NO_MEMORY_CONTENT) canonical = set(erasure_census.CANONICAL_OWNER_TABLES) + people = set(erasure_census.CANONICAL_PERSON_TABLES) + assert not people & (census | step_only | no_content | canonical) + assert erasure_census.CANONICAL_PERSON_TABLES == { + "loom.scim_users": "crawl_person.finalize", + "loom.knowledge_crawl_exclusion_keys": "crawl_person.exclude", + "loom.knowledge_crawl_exclusions": "crawl_person.exclude", + } assert not canonical & (census | step_only | no_content) for table, step in erasure_census.CANONICAL_OWNER_TABLES.items(): assert table.startswith("loom.") and step.endswith(".erase"), (table, step) diff --git a/tests/unit/test_google_workspace.py b/tests/unit/test_google_workspace.py new file mode 100644 index 0000000000..bc7ffdb69b --- /dev/null +++ b/tests/unit/test_google_workspace.py @@ -0,0 +1,610 @@ +"""Delegated JWTs and bounded Directory HTTP; no registration or provider is needed.""" + +from __future__ import annotations + +import asyncio +import copy +import json +import time +from contextlib import asynccontextmanager +from typing import Any +from urllib.parse import parse_qs + +import httpx +import jwt +import pytest +from cryptography.hazmat.primitives import serialization +from cryptography.hazmat.primitives.asymmetric import rsa + +from server.clients import gdrive +from server.clients import google_workspace as google +from server.clients._vendor import CredentialRefused, RateLimited, TooLarge, VendorError +from server.knowledge.app import crawl_authority +from server.settings import settings + +pytestmark = pytest.mark.asyncio + +_SUBJECT = "directory-admin@example.com" +_ACCESS = "private-delegated-access" + + +@pytest.fixture(scope="module") +def account() -> tuple[gdrive.ServiceAccount, Any]: + private = rsa.generate_private_key(public_exponent=65537, key_size=2048) + pem = private.private_bytes( + serialization.Encoding.PEM, + serialization.PrivateFormat.PKCS8, + serialization.NoEncryption(), + ).decode() + credential = json.dumps( + { + "type": "service_account", + "client_email": "service@example.com", + "private_key": pem, + "token_uri": "https://untrusted.example/token", + } + ) + return gdrive.parse_service_account(credential), private.public_key() + + +@asynccontextmanager +async def _http(monkeypatch: pytest.MonkeyPatch, handler: Any): + calls: list[httpx.Request] = [] + + async def record(request: httpx.Request) -> httpx.Response: + calls.append(request) + return handler(request) + + async def pace(delay: float) -> None: + assert delay == gdrive.PAGE_PACE_S + + async with httpx.AsyncClient( + transport=httpx.MockTransport(record), follow_redirects=True + ) as client: + monkeypatch.setattr(google, "get_client", lambda: client) + monkeypatch.setattr(google, "_pace", pace) + monkeypatch.setattr(gdrive, "get_client", lambda: client) + yield calls + + +def _token(scope: gdrive.ReadScope, *, deadline: float | None = None) -> google.DelegatedToken: + return google.DelegatedToken(_ACCESS, _SUBJECT, scope, deadline or time.monotonic() + 30) + + +def _person(**changes: Any) -> dict[str, Any]: + return { + "id": "123456", + "primaryEmail": "person@example.com", + "aliases": ["alias@example.com"], + "customerId": "customer-one", + "suspended": False, + "archived": False, + **changes, + } + + +class _Body(httpx.AsyncByteStream): + def __init__(self, *chunks: bytes) -> None: + self.chunks = chunks + self.reads = 0 + self.closed = False + + async def __aiter__(self): + for chunk in self.chunks: + self.reads += 1 + yield chunk + + async def aclose(self) -> None: + self.closed = True + + +class _WaitingBody(_Body): + def __init__(self) -> None: + super().__init__() + self.entered = asyncio.Event() + + async def __aiter__(self): + self.entered.set() + await asyncio.Event().wait() + yield b"" + + +@pytest.mark.parametrize("scope", list(gdrive.ReadScope)) +async def test_grant_signs_exact_delegated_subject_scope_and_fixed_endpoint( + monkeypatch: pytest.MonkeyPatch, account: Any, scope: gdrive.ReadScope +) -> None: + service, public = account + + def answer(request: httpx.Request) -> httpx.Response: + assert str(request.url) == "https://oauth2.googleapis.com/token" + form = parse_qs(request.content.decode()) + assert set(form) == {"grant_type", "assertion"} + assert form["grant_type"] == ["urn:ietf:params:oauth:grant-type:jwt-bearer"] + claims = jwt.decode( + form["assertion"][0], public, algorithms=["RS256"], audience=str(request.url) + ) + assert set(claims) == {"iss", "sub", "scope", "aud", "iat", "exp"} + assert (claims["iss"], claims["sub"], claims["scope"]) == ( + service.client_email, + _SUBJECT, + scope.value, + ) + assert claims["exp"] - claims["iat"] == 600 + return httpx.Response( + 200, + json={ + "access_token": _ACCESS, + "token_type": "Bearer", + "expires_in": 3600, + "scope": scope.value, + }, + ) + + clock = time.monotonic() + + def advancing() -> float: + nonlocal clock + clock += 0.001 + return clock + + monkeypatch.setattr(google, "_now", advancing) + deadline = clock + 30 + async with _http(monkeypatch, answer) as calls: + token = await google.delegated_token( + service, subject=_SUBJECT, scope=scope, deadline=deadline, max_bytes=4096 + ) + assert len(calls) == 1 and token.deadline == deadline + assert (token.subject, token.scope) == (_SUBJECT, scope) + assert _ACCESS not in repr(token) and _SUBJECT not in repr(token) + with pytest.raises(TypeError): + str(token) + with pytest.raises(TypeError): + copy.copy(token) + + +async def test_shared_grant_keeps_its_original_payload_and_response_contract( + monkeypatch: pytest.MonkeyPatch, account: Any +) -> None: + service, public = account + + def answer(request: httpx.Request) -> httpx.Response: + form = parse_qs(request.content.decode()) + claims = jwt.decode( + form["assertion"][0], + public, + algorithms=["RS256"], + audience="https://oauth2.googleapis.com/token", + ) + assert "sub" not in claims and claims["scope"] == gdrive.ReadScope.DRIVE.value + return httpx.Response(200, json={"access_token": "ordinary-token"}) + + async with _http(monkeypatch, answer) as calls: + assert await gdrive.access_token(service) == "ordinary-token" + assert len(calls) == 1 and calls[0].url.host == "oauth2.googleapis.com" + + +@pytest.mark.parametrize( + ("subject", "scope"), + [ + (None, gdrive.ReadScope.DRIVE), + ("not-an-email", gdrive.ReadScope.DRIVE), + ("person@example.com\n", gdrive.ReadScope.DIRECTORY_USERS), + (_SUBJECT, "https://www.googleapis.com/auth/drive"), + (_SUBJECT, "https://www.googleapis.com/auth/admin.directory.group.member"), + ], +) +async def test_subject_and_scope_refusals_make_no_http_request( + monkeypatch: pytest.MonkeyPatch, account: Any, subject: Any, scope: Any +) -> None: + async with _http(monkeypatch, lambda request: pytest.fail("no request permitted")) as calls: + with pytest.raises(CredentialRefused): + await google.delegated_token( + account[0], + subject=subject, + scope=scope, + deadline=time.monotonic() + 30, + max_bytes=4096, + ) + assert calls == [] + + +@pytest.mark.parametrize("status", [302, 400, 403, 429]) +async def test_grant_refusals_do_not_read_echoed_secrets_follow_or_retry( + monkeypatch: pytest.MonkeyPatch, account: Any, status: int +) -> None: + body = _Body(b"private-provider-body") + async with _http( + monkeypatch, + lambda request: httpx.Response( + status, headers={"Location": "https://untrusted.example/private"}, stream=body + ), + ) as calls: + with pytest.raises(VendorError) as caught: + await google.delegated_token( + account[0], + subject=_SUBJECT, + scope=gdrive.ReadScope.DRIVE, + deadline=time.monotonic() + 30, + max_bytes=4096, + ) + assert caught.value.status == status and body.closed and body.reads == 0 + assert len(calls) == 1 + assert all(secret not in str(caught.value) for secret in ("private", _SUBJECT, "untrusted")) + + +@pytest.mark.parametrize( + ("error", "expected"), + [ + ({"errors": [{"reason": "userRateLimitExceeded"}]}, RateLimited), + ({"errors": [{"reason": "quotaExceeded"}]}, RateLimited), + ({"status": "RESOURCE_EXHAUSTED"}, RateLimited), + ({"errors": [{"reason": "forbidden"}]}, CredentialRefused), + ], +) +async def test_directory_forbidden_distinguishes_quota_from_credentials_without_error_echo( + monkeypatch: pytest.MonkeyPatch, error: dict[str, Any], expected: type[VendorError] +) -> None: + body = _Body(json.dumps({"error": {**error, "message": "private-provider-body"}}).encode()) + async with _http(monkeypatch, lambda request: httpx.Response(403, stream=body)) as calls: + with pytest.raises(expected) as caught: + await google.user( + _token(gdrive.ReadScope.DIRECTORY_USERS), + "123456", + customer_id="customer-one", + max_bytes=4096, + ) + assert type(caught.value) is expected and caught.value.status == 403 + assert len(calls) == 1 and body.reads == 1 and body.closed + assert all(value not in str(caught.value) for value in ("private", _ACCESS, _SUBJECT)) + + +@pytest.mark.parametrize("raw", [b"not-json", b'{"error":{"errors":12}}', b'{"error":{}}']) +async def test_ambiguous_directory_forbidden_does_not_claim_credential_revocation( + monkeypatch: pytest.MonkeyPatch, raw: bytes +) -> None: + body = _Body(raw) + async with _http(monkeypatch, lambda request: httpx.Response(403, stream=body)) as calls: + with pytest.raises(VendorError) as caught: + await google.user( + _token(gdrive.ReadScope.DIRECTORY_USERS), + "123456", + customer_id="customer-one", + max_bytes=4096, + ) + assert type(caught.value) is VendorError and caught.value.status == 403 + assert len(calls) == 1 and body.reads == 1 and body.closed + assert raw.decode() not in str(caught.value) + + +async def test_directory_error_body_bound_stops_reading_and_closes_stream( + monkeypatch: pytest.MonkeyPatch, +) -> None: + body = _Body(b"x" * 8, b"x" * 9, b"private-must-not-be-read") + async with _http(monkeypatch, lambda request: httpx.Response(403, stream=body)) as calls: + with pytest.raises(TooLarge) as caught: + await google.user( + _token(gdrive.ReadScope.DIRECTORY_USERS), + "123456", + customer_id="customer-one", + max_bytes=16, + ) + assert caught.value.status == 403 and "private" not in str(caught.value) + assert len(calls) == 1 and body.reads == 2 and body.closed + + +@pytest.mark.parametrize( + "change", + [ + {"token_type": None}, + {"expires_in": True}, + {"access_token": "private\naccess"}, + {"scope": "https://www.googleapis.com/auth/drive"}, + ], +) +async def test_bad_token_metadata_never_returns_a_checkout( + monkeypatch: pytest.MonkeyPatch, account: Any, change: dict[str, Any] +) -> None: + answer = {"access_token": _ACCESS, "token_type": "Bearer", "expires_in": 3600, **change} + async with _http(monkeypatch, lambda request: httpx.Response(200, json=answer)) as calls: + with pytest.raises(VendorError): + await google.delegated_token( + account[0], + subject=_SUBJECT, + scope=gdrive.ReadScope.DRIVE, + deadline=time.monotonic() + 30, + max_bytes=4096, + ) + assert len(calls) == 1 + + +async def test_user_and_directory_identity_require_explicit_active_customer_evidence( + monkeypatch: pytest.MonkeyPatch, +) -> None: + token = _token(gdrive.ReadScope.DIRECTORY_USERS) + + def answer(request: httpx.Request) -> httpx.Response: + person = ( + _person() + if request.url.path.endswith("/123456") + else _person(id="directory-admin-id", primaryEmail=_SUBJECT, aliases=[]) + ) + return httpx.Response(200, json=person) + + async with _http(monkeypatch, answer) as calls: + principal = await google.directory_identity(token, max_bytes=4096) + person = await google.user( + token, "123456", customer_id=principal.customer_id, max_bytes=4096 + ) + assert person.id == "123456" and person.email == "person@example.com" + assert person.aliases == ("alias@example.com",) + assert len(calls) == 2 + assert [r.url.path for r in calls] == [ + "/admin/directory/v1/users/" + _SUBJECT, + "/admin/directory/v1/users/123456", + ] + assert all(r.url.host == "admin.googleapis.com" for r in calls) + assert all(r.headers["Authorization"] == "Bearer " + _ACCESS for r in calls) + assert all("deletionTime" in r.url.params["fields"] for r in calls) + + +@pytest.mark.parametrize( + "change", + [{"suspended": True}, {"archived": True}, {"deletionTime": "deleted"}, {"suspended": None}], +) +async def test_user_refuses_inactive_or_unproven_state( + monkeypatch: pytest.MonkeyPatch, change: dict[str, Any] +) -> None: + async with _http(monkeypatch, lambda request: httpx.Response(200, json=_person(**change))): + with pytest.raises(CredentialRefused): + await google.user( + _token(gdrive.ReadScope.DIRECTORY_USERS), + "123456", + customer_id="customer-one", + max_bytes=4096, + ) + + +async def test_external_customer_is_not_an_admitted_employee( + monkeypatch: pytest.MonkeyPatch, +) -> None: + async with _http( + monkeypatch, + lambda request: httpx.Response(200, json=_person(customerId="external-customer")), + ) as calls: + with pytest.raises(CredentialRefused): + await google.user( + _token(gdrive.ReadScope.DIRECTORY_USERS), + "123456", + customer_id="customer-one", + max_bytes=4096, + ) + assert len(calls) == 1 + + +async def test_user_response_must_match_the_requested_id_or_verified_address( + monkeypatch: pytest.MonkeyPatch, +) -> None: + async with _http(monkeypatch, lambda request: httpx.Response(200, json=_person())) as calls: + with pytest.raises(VendorError, match="identity mismatch"): + await google.user( + _token(gdrive.ReadScope.DIRECTORY_USERS), + "different-id", + customer_id="customer-one", + max_bytes=4096, + ) + assert len(calls) == 1 + + +async def test_members_pages_preserve_types_without_inferring_employee_status_or_expanding_customer( + monkeypatch: pytest.MonkeyPatch, +) -> None: + def page(request: httpx.Request) -> httpx.Response: + assert request.url.host == "admin.googleapis.com" + assert request.url.path == "/admin/directory/v1/groups/group@example.com/members" + assert request.url.params["maxResults"] == "200" + assert request.url.params["includeDerivedMembership"] == "false" + if "pageToken" not in request.url.params: + return httpx.Response( + 200, + json={ + "members": [{"id": "external", "type": "USER", "email": "other@external.com"}], + "nextPageToken": "next-page", + }, + ) + assert request.url.params["pageToken"] == "next-page" + return httpx.Response( + 200, + json={ + "members": [ + {"id": "nested", "type": "GROUP", "email": "nested@example.com"}, + {"id": "customer-one", "type": "CUSTOMER", "status": "UNDOCUMENTED"}, + ] + }, + ) + + async with _http(monkeypatch, page) as calls: + members = await google.group_members( + _token(gdrive.ReadScope.DIRECTORY_MEMBERS), + "group@example.com", + max_bytes=4096, + max_pages=2, + ) + assert [(m.kind, m.id, m.email) for m in members] == [ + ("USER", "external", "other@external.com"), + ("GROUP", "nested", "nested@example.com"), + ("CUSTOMER", "customer-one", None), + ] + assert len(calls) == 2 and all("/groups/" in r.url.path for r in calls) + + +@pytest.mark.parametrize("failure", ["repeated-token", "repeated-member", "page-bound", "bad-type"]) +async def test_incomplete_membership_never_returns_a_partial_list( + monkeypatch: pytest.MonkeyPatch, failure: str +) -> None: + pages = 0 + + def answer(request: httpx.Request) -> httpx.Response: + nonlocal pages + pages += 1 + kind = "UNKNOWN" if failure == "bad-type" else "USER" + identifier = str(pages) if failure != "repeated-member" else "same" + return httpx.Response( + 200, json={"members": [{"type": kind, "id": identifier}], "nextPageToken": "same-page"} + ) + + async with _http(monkeypatch, answer) as calls: + with pytest.raises(VendorError): + await google.group_members( + _token(gdrive.ReadScope.DIRECTORY_MEMBERS), + "group@example.com", + max_bytes=4096, + max_pages=1 if failure == "page-bound" else 3, + ) + assert len(calls) == (1 if failure in {"page-bound", "bad-type"} else 2) + + +async def test_membership_over_the_crawl_bound_is_not_truncated( + monkeypatch: pytest.MonkeyPatch, +) -> None: + pages = 0 + + def answer(request: httpx.Request) -> httpx.Response: + nonlocal pages + pages += 1 + return httpx.Response( + 200, + json={ + "members": [{"type": "USER", "id": f"{pages}-{n}"} for n in range(200)], + "nextPageToken": str(pages), + }, + ) + + async with _http(monkeypatch, answer) as calls: + with pytest.raises(TooLarge, match="crawl bound"): + await google.group_members( + _token(gdrive.ReadScope.DIRECTORY_MEMBERS), + "group@example.com", + max_bytes=32768, + max_pages=11, + ) + assert len(calls) == 11 + + +@pytest.mark.parametrize("configured", [False, True]) +async def test_response_cap_stops_reading_and_closes_stream( + monkeypatch: pytest.MonkeyPatch, + configured: bool, +) -> None: + cap = 16 + if configured: + monkeypatch.setattr(settings, "knowledge_crawl_directory_max_response_bytes", 1024) + cap = crawl_authority.configured_bounds().directory.body_bytes + body = _Body(b"x" * (cap // 2), b"x" * (cap // 2 + 1), b"must-not-be-read") + async with _http(monkeypatch, lambda request: httpx.Response(200, stream=body)) as calls: + with pytest.raises(TooLarge): + await google.user( + _token(gdrive.ReadScope.DIRECTORY_USERS), + "123456", + customer_id="customer-one", + max_bytes=cap, + ) + assert len(calls) == 1 and body.reads == 2 and body.closed + + +async def test_elapsed_pacing_refuses_the_next_page_before_http( + monkeypatch: pytest.MonkeyPatch, +) -> None: + deadline = time.monotonic() + 30 + waits = [] + + async def spent(delay: float) -> None: + waits.append(delay) + monkeypatch.setattr(google, "_now", lambda: deadline + 1) + + async with _http( + monkeypatch, lambda request: httpx.Response(200, json={"nextPageToken": "next"}) + ) as calls: + monkeypatch.setattr(google, "_pace", spent) + with pytest.raises(TimeoutError): + await google.group_members( + _token(gdrive.ReadScope.DIRECTORY_MEMBERS, deadline=deadline), + "group@example.com", + max_bytes=4096, + max_pages=2, + ) + assert len(calls) == 1 and waits == [gdrive.PAGE_PACE_S] + + +async def test_invalid_bounds_scope_and_path_make_no_http_call( + monkeypatch: pytest.MonkeyPatch, +) -> None: + async with _http(monkeypatch, lambda request: pytest.fail("no request permitted")) as calls: + with pytest.raises(ValueError): + await google.group_members( + _token(gdrive.ReadScope.DIRECTORY_MEMBERS), "group", max_bytes=4096, max_pages=0 + ) + with pytest.raises(ValueError): + await google.user( + _token(gdrive.ReadScope.DIRECTORY_USERS), + "123456", + customer_id="customer-one", + max_bytes=True, + ) + with pytest.raises(ValueError): + await google.user( + _token(gdrive.ReadScope.DIRECTORY_USERS), + "../users/other", + customer_id="customer-one", + max_bytes=4096, + ) + with pytest.raises(CredentialRefused): + await google.user( + _token(gdrive.ReadScope.DRIVE), "123456", customer_id="customer-one", max_bytes=4096 + ) + with pytest.raises(TimeoutError): + await google.user( + _token(gdrive.ReadScope.DIRECTORY_USERS, deadline=float("nan")), + "123456", + customer_id="customer-one", + max_bytes=4096, + ) + assert calls == [] + + +@pytest.mark.parametrize("status", [200, 403]) +async def test_cancellation_closes_the_owned_stream_and_propagates( + monkeypatch: pytest.MonkeyPatch, + status: int, +) -> None: + body = _WaitingBody() + async with _http(monkeypatch, lambda request: httpx.Response(status, stream=body)) as calls: + task = asyncio.create_task( + google.user( + _token(gdrive.ReadScope.DIRECTORY_USERS), + "123456", + customer_id="customer-one", + max_bytes=4096, + ) + ) + await body.entered.wait() + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + assert len(calls) == 1 and body.closed + + +@pytest.mark.parametrize("status", [200, 403]) +async def test_absolute_deadline_covers_stream_body_and_closes_it( + monkeypatch: pytest.MonkeyPatch, + status: int, +) -> None: + body = _WaitingBody() + async with _http(monkeypatch, lambda request: httpx.Response(status, stream=body)) as calls: + with pytest.raises(TimeoutError): + await google.user( + _token(gdrive.ReadScope.DIRECTORY_USERS, deadline=time.monotonic() + 0.1), + "123456", + customer_id="customer-one", + max_bytes=4096, + ) + assert len(calls) == 1 and body.closed diff --git a/tests/unit/test_restore_reconcile.py b/tests/unit/test_restore_reconcile.py index b5a3f1acc6..8cdada9312 100644 --- a/tests/unit/test_restore_reconcile.py +++ b/tests/unit/test_restore_reconcile.py @@ -51,14 +51,15 @@ async def no_lost(after: datetime) -> list[Any]: monkeypatch.setattr(knowledge_reconcile, "held_sources", no_sources) monkeypatch.setattr(knowledge_reconcile, "dataset_sources_lost", no_lost) - # These cases isolate memory and erasure replay participants. Real connection - # cleanup and its publication ordering are covered in test_connected_account_restore. - from server.knowledge.app import connected_restore + # These cases isolate memory and erasure replay participants; their restored + # store contains no connected accounts or crawl registries. + from server.knowledge.app import connected_restore, crawl_restore - async def no_connections(*a: Any, **k: Any) -> None: + async def no_private_sources(*a: Any, **k: Any) -> None: pass - monkeypatch.setattr(connected_restore, "restore_all", no_connections) + monkeypatch.setattr(connected_restore, "restore_all", no_private_sources) + monkeypatch.setattr(crawl_restore, "restore_all", no_private_sources) @pytest.fixture(autouse=True) diff --git a/tests/unit/test_rmt_write_paths.py b/tests/unit/test_rmt_write_paths.py index 0868159da3..a441d0eb65 100644 --- a/tests/unit/test_rmt_write_paths.py +++ b/tests/unit/test_rmt_write_paths.py @@ -43,10 +43,9 @@ "query_history_sources": ("builder", "test_repository_column_sets_match_schema.py"), "knowledge_mirrors": ("builder", "test_knowledge_mirrors.py"), "knowledge_source_listings": ("builder", "test_knowledge_mirrors.py"), - "knowledge_group_members": ( - "insert_once", - "read-side schema only; no production writer inserts or rewrites membership evidence", - ), + "knowledge_group_members": ("builder", "test_crawl_storage_contracts.py"), + "knowledge_crawl_exclusion_keys": ("builder", "test_crawl_storage_contracts.py"), + "knowledge_crawl_exclusions": ("builder", "test_crawl_storage_contracts.py"), "machine_identities": ("builder", "test_machines.py"), "org_model_calls": ("builder", "test_model_calls_switch.py"), "extraction_policies": ("builder", "test_extraction_policy.py"), diff --git a/tests/unit/test_schema_baseline.py b/tests/unit/test_schema_baseline.py index 021e30af66..1eb07df53e 100644 --- a/tests/unit/test_schema_baseline.py +++ b/tests/unit/test_schema_baseline.py @@ -169,6 +169,7 @@ def test_the_journal_files() -> None: "111_connected_accounts.sql", "112_code_repo_git_proxy_identity.sql", "113_oauth_verified_email.sql", + "114_workspace_crawl.sql", ] diff --git a/tests/unit/test_scim.py b/tests/unit/test_scim.py index 97f1a7e0c0..1a9d6787a7 100644 --- a/tests/unit/test_scim.py +++ b/tests/unit/test_scim.py @@ -23,6 +23,7 @@ grant_claims, login, oauth, + person_erasure, revocation, scim, scim_routes, @@ -37,7 +38,7 @@ user_name_and_email, ) from server.core import ch -from server.core.errors import GrantBusyError +from server.core.errors import BackendUnavailableError, GrantBusyError from server.identity.app import me_service_accounts, scim_groups from server.observability import audit from server.settings import settings @@ -157,7 +158,7 @@ def test_active_takes_booleans_and_their_strings() -> None: class _Table: """loom.scim_users, loom.namespaces as `_subjects` reads it, and loom.revoked_tokens as the revocation module reads and writes it, - with an empty connected-account inventory.""" + with verified namespace contacts and empty connected-account/crawl inventories.""" def __init__( self, @@ -260,6 +261,38 @@ async def query_one(self, sql: str, params: dict[str, Any] | None = None) -> Any return row async def query_all(self, sql: str, params: dict[str, Any]) -> list[dict[str, Any]]: + if "SELECT owner_email, contact_email" in sql: + assert set(params) == {"org", "sub"} + assert "FROM loom.namespaces FINAL" in sql and "LIMIT 8" in sql + assert "ifNull(active, true) = true" in sql and "contact_email != ''" in sql + return [ + {"owner_email": subject, "contact_email": email} + for email, subjects in self.accounts.items() + for subject in subjects + if params["org"] in self.reached_in.get(subject, ["acme"]) + and params["sub"].lower() in {subject.lower(), email.lower()} + ] + if "SELECT n.namespace, n.owner_email, n.contact_email, n.password_hash," in sql: + assert set(params) == {"org", "sub", "owner"} + assert "FROM loom.namespaces FINAL" in sql and "FROM loom.sso_links FINAL" in sql + assert "active = 1 AND contact_email != ''" in sql and "LIMIT 129" in sql + return [ + { + "namespace": f"verified-{subject}", + "owner_email": subject, + "contact_email": email, + "password_hash": "verified-password-hash", + "hosted_domain": "", + "issuer": "", + } + for email, subjects in self.accounts.items() + for subject in subjects + if params["org"] in self.reached_in.get(subject, ["acme"]) + and ( + subject.lower() == params["owner"].lower() + or email.lower() == params["sub"].lower() + ) + ] if sql == ( "SELECT org, subject, kind, connection_id, account_id, workspace_id, " "account_email_enc, credential_enc, scopes, active, refresh_id, connected_at, " @@ -320,6 +353,51 @@ async def insert(self, table: str, rows: list[dict[str, Any]]) -> None: self.rows.extend(dict(r) for r in rows) +class _PersonCleanup: + """An empty crawl inventory with real capture and durable pending validation.""" + + def __init__(self, world: _World) -> None: + self.world = world + self.prepared: list[person_erasure.Capture] = [] + self.cleaned: list[person_erasure.Capture] = [] + self.finalized: list[person_erasure.Capture] = [] + + async def prepare( + self, pending: person_erasure.Pending + ) -> tuple[person_erasure.Workspace, ...]: + assert self.world.holding > 0 + await pending.validate(before_delete=True) + assert not pending.captured.prepared and not pending.captured.workspaces + self.prepared.append(pending.captured) + return () + + async def cleanup(self, pending: person_erasure.Pending) -> None: + assert self.world.holding == 0 + await pending.validate() + assert pending.captured.prepared and not pending.captured.workspaces + self.cleaned.append(pending.captured) + + async def finalize(self, pending: person_erasure.Pending) -> None: + assert self.world.holding > 0 and pending.captured in self.cleaned + await pending.validate() + for row in self.world.table.rows: + if row["id"] != pending.captured.id: + continue + if row["crawl_erasure_enc"]: + historical = person_erasure.load(row["id"], row["crawl_erasure_enc"]) + assert all( + any( + (old.org, old.subject) == (new.org, new.subject) + and set(old.emails) <= set(new.emails) + and set(old.spellings) <= set(new.spellings) + for new in pending.captured.people + ) + for old in historical.people + ) + row.update(crawl_erasure_enc="", crawl_erasure_required=True, crawl_erasure_done=True) + self.finalized.append(pending.captured) + + class _World: """The tables, the real revocation writes and reads over them, and the audit log.""" @@ -328,12 +406,13 @@ def __init__(self, monkeypatch: pytest.MonkeyPatch) -> None: self.elsewhere: dict[str, list[str]] = {} # subject -> other orgs it is in self.reached_in: dict[str, list[str]] = {} # subject -> orgs its email reaches it in self.table = _Table(self.accounts, self.elsewhere, self.reached_in) + self.person_cleanup = _PersonCleanup(self) self.audits: list[tuple[str, dict[str, Any]]] = [] self.revoked_at_audit: list[dict[str, Any]] = [] # SCIM's revocations as each audit lands self.claim_limit: int | None = None # names held at once before a claim runs out self.claims: dict[tuple[str, str], asyncio.Lock] = {} # one holder a name, as a lease self.leases: list[str] = [] - self.holding = 0 # SCIM leases held now + self._holding: dict[asyncio.Task[Any] | None, int] = {} # per-caller SCIM leases self.waits: list[float] = [] # what the real `wait_out` sleeps async def stale_snapshot(subject: str) -> bool: @@ -380,6 +459,10 @@ async def no_refresh() -> None: monkeypatch.setattr(settings, "scim_token", _TOKEN) monkeypatch.setattr(scim, "get_client", lambda: self.table) monkeypatch.setattr(ch, "get_client", lambda: self.table) + monkeypatch.setattr(person_erasure, "get_client", lambda: self.table) + monkeypatch.setattr(person_erasure, "_preparer", self.person_cleanup.prepare) + monkeypatch.setattr(person_erasure, "_cleanup", self.person_cleanup.cleanup) + monkeypatch.setattr(person_erasure, "_finalizer", self.person_cleanup.finalize) monkeypatch.setattr(scim, "_held", held) monkeypatch.setattr(grant_claims, "service_account_change", name_claim) monkeypatch.setattr(revocation, "asyncio", SimpleNamespace(sleep=no_wait)) @@ -387,7 +470,20 @@ async def no_refresh() -> None: monkeypatch.setattr(revocation, "refresh", no_refresh) monkeypatch.setattr(audit, "emit", emit) + @property + def holding(self) -> int: + return self._holding.get(asyncio.current_task(), 0) + + @holding.setter + def holding(self, value: int) -> None: + task = asyncio.current_task() + if value: + self._holding[task] = value + else: + self._holding.pop(task, None) + def person(self, email: str, *subjects: str) -> None: + """Seed verified namespace contacts, separately from grants in other orgs.""" self.accounts[email] = list(subjects) @property @@ -686,7 +782,29 @@ async def test_every_row_written_carries_every_column(world) -> None: await scim.create( "lee@example.com", "lee@example.com", active=True, emails=[] ) # a reactivation - assert [sorted(r) for r in world.table.rows] == [sorted(column_names("scim_users"))] * 4 + # Creation, deactivation, canonical capture, prepared capture, deletion, reactivation. + assert [sorted(r) for r in world.table.rows] == [sorted(column_names("scim_users"))] * 6 + assert [(r["active"], r["deleted"]) for r in world.table.rows] == [ + (1, 0), + (0, 0), + (0, 0), + (0, 0), + (0, 1), + (1, 0), + ] + assert all(r["id"] == uid and r["email"] == "lee@example.com" for r in world.table.rows) + [captured] = world.person_cleanup.prepared + assert captured.id == uid and not captured.prepared and not captured.workspaces + assert [(p.org, p.subject, p.emails) for p in captured.people] == [ + ("acme", "lee", ("lee@example.com",)) + ] + [cleaned] = world.person_cleanup.cleaned + assert cleaned.people == captured.people and cleaned.prepared and not cleaned.workspaces + assert world.person_cleanup.finalized == [cleaned] + assert all( + not r["crawl_erasure_enc"] and r["crawl_erasure_required"] and r["crawl_erasure_done"] + for r in world.table.rows + ) columns = sorted(column_names("revoked_tokens")) assert [sorted(r) for r in world.table.revocations] == [columns] * 2 @@ -828,6 +946,97 @@ async def test_a_token_the_person_revoked_still_takes_its_children(world) -> Non assert world.table.live_tokens() == [] +@pytest.mark.parametrize("recreate", [False, True]) +async def test_delete_retry_without_durable_identity_proof_stays_inactive_and_pending( + world, monkeypatch, recreate: bool +) -> None: + world.person("lee@example.com", "lee") + uid = await _create("lee@example.com") + query = world.table.query_all + failed = False + + async def read(sql: str, params): + nonlocal failed + if "SELECT DISTINCT owner_email, org" in sql and not failed: + failed = True + raise RuntimeError("identity backend unavailable before capture") + return await query(sql, params) + + monkeypatch.setattr(world.table, "query_all", read) + with pytest.raises(RuntimeError, match="identity backend unavailable"): + await scim.delete(uid) + row = world.table.newest(uid) + assert row is not None and row["deleted"] and not row["active"] + assert row["crawl_erasure_required"] and not row["crawl_erasure_done"] + assert not row["crawl_erasure_enc"] and not world.person_cleanup.finalized + with pytest.raises(BackendUnavailableError): + await scim.delete(uid) + assert world.table.newest(uid) == row + assert await world.keeps_out("lee") + assert not world.person_cleanup.finalized + if recreate: + with pytest.raises(BackendUnavailableError): + await scim.create("lee@example.com", "lee@example.com", active=False, emails=[]) + current = world.table.newest(uid) + assert current is not None and not current["deleted"] and not current["active"] + assert current["crawl_erasure_required"] and not current["crawl_erasure_done"] + assert not current["crawl_erasure_enc"] and not world.person_cleanup.finalized + assert await world.keeps_out("lee") + + +async def test_an_older_cleanup_attempt_refuses_after_another_request_finishes_raw_erasure( + world, monkeypatch +) -> None: + email = _PEOPLE["a"] + world.person(email, *_REACHES["a"]) + world.person(_PEOPLE["b"], *_REACHES["b"]) + uid = await _create(email) + run = _Run(world) + run.together = True + entered, release = asyncio.Event(), asyncio.Event() + original = world.person_cleanup.cleanup + first = True + + async def cleanup(pending: person_erasure.Pending) -> None: + nonlocal first + if first: + first = False + entered.set() + await release.wait() + await original(pending) + + monkeypatch.setattr(person_erasure, "_cleanup", cleanup) + task = asyncio.create_task(run.step(("delete", "a", True), ())) + signal = asyncio.create_task(entered.wait()) + try: + done, _ = await asyncio.wait({task, signal}, return_when=asyncio.FIRST_COMPLETED) + if task in done: + await task + pytest.fail("the DELETE did not reach its external cleanup boundary") + assert entered.is_set() + await scim.create(email, email, active=False, emails=[]) + release.set() + assert await task is True # A stale proof refuses; the request is not a clean success. + assert not run.pending_identity and "a" in run.dirty + row = world.table.newest(uid) + assert row is not None and not row["active"] and not row["deleted"] + assert row["crawl_erasure_done"] and not row["crawl_erasure_enc"] + assert any(c.id == uid for c in world.person_cleanup.finalized) + assert all( + not r["crawl_erasure_enc"] and r["crawl_erasure_done"] + for r in world.table.rows + if r["id"] == uid + ) + assert await world.keeps_out(_REACHES["a"][0]) + await run.check() + finally: + release.set() + signal.cancel() + if not task.done(): + task.cancel() + await asyncio.gather(signal, task, return_exceptions=True) + + async def test_many_tokens_are_revoked_in_one_request_a_few_names_held_at_once(world) -> None: world.person("lee@example.com", "lee") world.table.tokens += [_saved_token(f"bot-{i:04}", "lee") for i in range(2000)] @@ -998,8 +1207,12 @@ async def test_a_reactivation_waits_once_its_leases_are_let_go(world, monkeypatc @asynccontextmanager async def held(name: str) -> AsyncIterator[None]: events.append(f"held {name.split(':')[0]}") - yield - events.append(f"let go {name.split(':')[0]}") + world.holding += 1 + try: + yield + finally: + world.holding -= 1 + events.append(f"let go {name.split(':')[0]}") async def wait(cutoff: datetime) -> None: events.append("waited") @@ -2867,6 +3080,7 @@ def __init__(self, world: _World) -> None: self.together = False # requests at once: no faults, which they would share self.race: random.Random | None = None # whether a sign-in waits on a row write self.refused_late = 0 # sign-ins refused at the second check + self.pending_identity = False # a failed capture needs verified historical repair self.operator: set[str] = set() # subjects an operator revoked outright self.sessions: list[tuple[str, str, int]] = [] # (person, subject, iat) signed in for s in sorted({s for subjects in _REACHES.values() for s in subjects}): @@ -2964,7 +3178,38 @@ async def step(self, op: tuple[str, str, bool], faults: _Faults) -> bool: started = self.now() try: settled = await self._send(*op) + except BackendUnavailableError: + row = self.world.table.newest(scim.user_id(_PEOPLE[op[1]])) + assert row is not None and row["crawl_erasure_required"] + encoded = row["crawl_erasure_enc"] + if row["crawl_erasure_done"]: + # Another request may finish the exact raw cleanup while this + # request's retained Pending proof correctly becomes unusable. + assert self.together and not encoded + assert any(c.id == row["id"] for c in self.world.person_cleanup.finalized) + assert all( + not old["crawl_erasure_enc"] and old["crawl_erasure_done"] + for old in self.world.table.rows + if old["id"] == row["id"] + ) + else: + # A permitted recreate resets deleted, but cannot repair absent + # historical identity. Deactivation still keeps the person out. + assert not row["active"] and not row["crawl_erasure_done"] + if not self.together and op[0] == "delete": + assert row["deleted"] + assert not encoded or not person_erasure.load(row["id"], encoded).prepared + self.pending_identity = True + self.dirty[op[1]] = self.now() + return True except (RuntimeError, scim.Busy, login.SignInUnavailable): + if op[1] in _PEOPLE: + row = self.world.table.newest(scim.user_id(_PEOPLE[op[1]])) + if row and row["deleted"] and not row["crawl_erasure_done"]: + encoded = row["crawl_erasure_enc"] + self.pending_identity |= ( + not encoded or not person_erasure.load(row["id"], encoded).prepared + ) if "row-landed" in self.fired[:-1]: # and the read-back failed after it self.unaudited[op[1]] += 1 if op[0] != "signin": # a sign-in is no SCIM request @@ -3096,13 +3341,14 @@ def _fresh_world() -> Iterator[_World]: async def _run( units: list[_Unit], fired: list[tuple[bool, bool]] | None = None -) -> tuple[list[Any], list[tuple[bool, bool]]]: - """The state after each unit, and whether it failed and made a JIT account. +) -> tuple[list[Any], list[tuple[bool, bool]], int | None]: + """States, request failures/JIT, and the first state requiring verified identity repair. With `fired` (a run before), a unit that failed and was retried is sent once, clean.""" with _fresh_world() as world: run = _Run(world) states, fires = [], [] + pending_from = None for i, (op, faults, retried) in enumerate(units): if fired is not None and fired[i][0] and retried: if fired[i][1]: @@ -3115,7 +3361,9 @@ async def _run( await run.step(op, ()) await run.check() states.append(run.state()) - return states, fires + if run.pending_identity and pending_from is None: + pending_from = i + return states, fires, pending_from async def test_every_sequence_keeps_the_invariants_and_a_retry_converges() -> None: @@ -3129,11 +3377,18 @@ async def test_every_sequence_keeps_the_invariants_and_a_retry_converges() -> No faults = tuple(rng.sample(_FAULTS, rng.choice((1, 1, 2)))) if rng.random() < 0.5 else () units.append((rng.choice(_OPS), faults, bool(faults) and rng.random() < 0.7)) try: - states, fired = await _run(units) - if any(failed for failed, _ in fired): # (d) a retry ends where a clean request does - clean, _ = await _run(units, fired) - assert states == clean - runs += 1 + states, fired, pending_from = await _run(units) + if any(failed for failed, _ in fired): # (d) proven retries match a clean request + clean, _, clean_pending_from = await _run(units, fired) + uncertain = [n for n in (pending_from, clean_pending_from) if n is not None] + if uncertain: + # All schedules and invariants still run. Only the prefix before + # unavailable historical proof can promise clean retry equality. + end = min(uncertain) + assert states[:end] == clean[:end] + else: + assert states == clean + runs += 1 except AssertionError as e: raise AssertionError(f"{units}: {e}") from e assert runs > 1500 # failures fired and were compared, not just drawn @@ -3154,9 +3409,11 @@ async def held(name: str) -> AsyncIterator[None]: await lock.acquire() except TimeoutError: raise scim.Busy from None + world.holding += 1 try: yield finally: + world.holding -= 1 lock.release() async def turns() -> None: diff --git a/tests/unit/test_secrets_at_rest.py b/tests/unit/test_secrets_at_rest.py index 253f64da5e..d441e79af7 100644 --- a/tests/unit/test_secrets_at_rest.py +++ b/tests/unit/test_secrets_at_rest.py @@ -49,6 +49,20 @@ def _no_analysis_lease(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr(service, "lease", lambda ch=None: contextlib.nullcontext()) +@pytest.fixture(autouse=True) +def _no_crawl_registries(monkeypatch: pytest.MonkeyPatch) -> None: + """These sealed-registry fakes contain no crawl approvals, keys or person capsules.""" + from server.auth import person_erasure + from server.knowledge.app import crawl_reseal + + async def empty(*, dry_run: bool, ch: object) -> tuple[int, list[str]]: + return 0, [] + + monkeypatch.setattr(crawl_reseal, "approvals", empty) + monkeypatch.setattr(crawl_reseal, "keys", empty) + monkeypatch.setattr(person_erasure, "reseal_registry", empty) + + def test_purposes_do_not_open_each_other_and_empty_stays_empty( monkeypatch: pytest.MonkeyPatch, ) -> None: @@ -233,8 +247,9 @@ def no_global_client(): pytest.fail("reseal must use the supplied ClickHouse client") @asynccontextmanager - async def source_lease(name: str, *, ttl_s: float, wait_s: float, ch=None): + async def source_lease(name: str, *, ttl_s: float, wait_s: float, ch=None, settle_writes: bool): assert ch is client and name == "knowledge-source:o:docs" + assert settle_writes is True assert ttl_s == 60.0 and wait_s == 5.0 calls.append("lease") yield True diff --git a/tests/unit/test_source_reseal.py b/tests/unit/test_source_reseal.py new file mode 100644 index 0000000000..b2b7c5e294 --- /dev/null +++ b/tests/unit/test_source_reseal.py @@ -0,0 +1,279 @@ +"""Root rotation cannot copy metadata back after canonical-person cleanup.""" + +from __future__ import annotations + +import copy +from contextlib import asynccontextmanager +from typing import Any + +import pytest + +from server.cli import secrets as cli +from server.core import leases +from server.core.secrets_at_rest import seal, unseal +from server.erasure.store import crawl_controls +from server.knowledge.app import sources as source_ops +from server.settings import settings +from tests.unit.test_crawl_person_controls import _EMAILS, _World + +pytestmark = pytest.mark.asyncio +_TABLE = "loom.knowledge_sources" +_SOURCE = "00000000-0000-4000-8000-000000000001" + + +class _Rows(_World): + def __init__(self, monkeypatch: pytest.MonkeyPatch, *, race: bool, lost_reply: bool = False): + super().__init__() + self.race = race + self.lost_reply = lost_reply + self.cleaned = False + self.repairs = 0 + self.held: tuple[str, str] | None = None + self.insert_scopes: list[tuple[str, str] | None] = [] + self.ignore_write = False + self.cipher_writes: list[tuple[str, str] | None] = [] + self.expire_after_read = False + self.expired = False + monkeypatch.setattr(leases, "holding", self.holding) + + @asynccontextmanager + async def holding(self, key: str, *, ttl_s: float, wait_s: float, ch: Any, settle_writes: bool): + assert ch is self and (ttl_s, wait_s) == (60.0, 5.0) and settle_writes + prefix, org, namespace = key.split(":") + assert prefix == "knowledge-source" and self.held is None + self.held = org, namespace + try: + yield True + finally: + self.held = None + if self.expired: + raise leases.ClaimExpired("source lease expired", operation=key) + + def latest(self) -> list[dict[str, Any]]: + rows: dict[tuple[str, str, str], dict[str, Any]] = {} + for row in self.rows: + key = row["org"], row["namespace"], row["name"] + if key not in rows or row["version"] > rows[key]["version"]: + rows[key] = row + return list(rows.values()) + + async def cleanup_once(self) -> None: + async with source_ops.sources_lease("org", "namespace", ch=self): + await crawl_controls.controls( + "org", + "namespace", + _SOURCE, + "deleted", + _EMAILS, + spellings=("deleted", "gone@example.com"), + ch=self, + ) + + async def cleanup(self) -> None: + if self.cleaned or not self.race: + return + try: + await self.cleanup_once() + except RuntimeError as error: + assert str(error) == "lost cleanup reply" and not self.lost_reply + self.repairs += 1 + await self.cleanup_once() + self.cleaned = True # The real mutation polling and residue checks returned. + + async def query_all(self, sql: str, params: dict[str, Any] | None = None): + compact = " ".join(sql.split()) + if compact.startswith("SELECT config_id FROM loom.provider_configuration FINAL"): + assert params == {"s": list(cli._REBUILDING)} + return [] + if compact.startswith("SELECT DISTINCT org, namespace FROM loom.knowledge_sources FINAL"): + assert params is None and self.held is None + assert compact.endswith("ORDER BY org, namespace") + column = "credential_enc" if "credential_enc != ''" in compact else "secret_enc" + keys = sorted({(r["org"], r["namespace"]) for r in self.latest() if r[column]}) + if not self.expire_after_read: + await self.cleanup() + return [{"org": org, "namespace": ns} for org, ns in keys] + if compact.startswith("SELECT * FROM loom.knowledge_sources FINAL"): + column = "credential_enc" if "credential_enc != ''" in compact else "secret_enc" + rows = [r for r in self.latest() if r[column]] + if params is not None: + assert set(params) == {"org", "ns"} and self.held == (params["org"], params["ns"]) + assert "org = {org:String} AND namespace = {ns:String}" in compact + rows = [r for r in rows if (r["org"], r["namespace"]) == self.held] + snapshot = copy.deepcopy(rows) + if self.expire_after_read and params is not None: + # The store grants cleanup a new lease after the old holder expires. + # Its saved read may still resume; a late write must carry no metadata. + previous = self.held + self.held = None + self.expire_after_read = False + await self.cleanup() + self.held = previous + self.expired = True + else: + await self.cleanup() + return snapshot + assert params is not None + return await super().query_all(sql, params=params) + + async def query_one(self, sql: str, params: dict[str, Any] | None = None): + if "FROM loom.signing_keys FINAL" in sql: + assert params is None + return None + assert params is not None + if "SELECT count() AS n FROM loom.knowledge_sources" in sql and "old" in params: + assert "FINAL" not in sql and set(params) == {"org", "ns", "name", "id", "old", "new"} + column = "credential_enc" if "credential_enc = {old:String}" in sql else "secret_enc" + return {"n": len(self.cipher_rows(params, column))} + return await super().query_one(sql, params=params) + + def cipher_rows(self, params: dict[str, Any], column: str): + return [ + r + for r in self.rows + if self.scope(r, params) and r["name"] == params["name"] and r[column] == params["old"] + ] + + async def execute(self, sql: str, *, params: dict[str, Any], settings=None): + assert self.held == (params["org"], params["ns"]) + if "old" in params: + column = "credential_enc" if "UPDATE credential_enc" in sql else "secret_enc" + assert sql.startswith("ALTER TABLE loom.knowledge_sources UPDATE " + column) + assert settings == {"mutations_sync": 2} + assert "name = {name:String}" in sql and "source_id = {id:String}" in sql + self.cipher_writes.append(self.held) + if not self.ignore_write: + for row in self.cipher_rows(params, column): + row[column] = params["new"] + return + await super().execute(sql, params=params) + if self.lost_reply: + self.lost_reply = False + raise RuntimeError("lost cleanup reply") + + async def insert(self, table: str, rows: list[dict[str, Any]], *, settings: dict[str, int]): + assert table == _TABLE and settings == {"async_insert": 0} + self.insert_scopes.append(self.held) + if not self.ignore_write: + self.rows.extend(copy.deepcopy(rows)) + + +def _rotate(monkeypatch: pytest.MonkeyPatch, rows: _Rows, column: str = "secret_enc") -> str: + purpose = ( + "knowledge_sources.secret" if column == "secret_enc" else "knowledge_sources.credential" + ) + monkeypatch.setattr(settings, "jwt_secret", "old-root-secret") + monkeypatch.setattr(settings, "jwt_secret_previous", "") + old = seal(purpose, "retained-service-key") + for row in rows.rows: + row.update(source_id=_SOURCE, registrant_subject="deleted", created_by="gone@example.com") + row[column] = old + monkeypatch.setattr(settings, "jwt_secret", "new-root-secret") + monkeypatch.setattr(settings, "jwt_secret_previous", "old-root-secret") + monkeypatch.setattr(cli, "SEALED", ((_TABLE, column, (), purpose, ""),)) + return purpose + + +@pytest.mark.parametrize("lost_reply", [False, True]) +async def test_command_reseals_only_fresh_rows_after_actual_person_cleanup_and_repair( + monkeypatch: pytest.MonkeyPatch, + lost_reply: bool, +) -> None: + rows = _Rows(monkeypatch, race=True, lost_reply=lost_reply) + purpose = _rotate(monkeypatch, rows) + foreign = copy.deepcopy(rows.rows[-1]) + foreign.update(namespace="other", name="other-source", source_id="other-source") + foreign["secret_enc"] = seal(purpose, "other-service-key") + rows.rows.append(foreign) + foreign_before = copy.deepcopy(foreign) + counts, lost, checked = await cli.reseal_all(rows, dry_run=False) + assert rows.cleaned and rows.repairs == int(lost_reply) + assert all( + not r["active"] + and not r["approval_enc"] + and not r["crawl_generation"] + and not r["registrant_subject"] + and not r["created_by"] + and "gone@example.com" not in r["options_json"] + for r in rows.rows + if r["namespace"] == "namespace" + ) + assert counts == {_TABLE + ".secret_enc": 1} and lost == {} and not checked + assert rows.cipher_writes == [("org", "namespace")] and not rows.insert_scopes + assert [r for r in rows.rows if r["namespace"] == "other"] == [foreign_before] + monkeypatch.setattr(settings, "jwt_secret_previous", "") + [current] = [r for r in rows.latest() if r["namespace"] == "namespace"] + assert unseal(purpose, current["secret_enc"]) == "retained-service-key" + + +@pytest.mark.parametrize("column", ["credential_enc", "secret_enc"]) +async def test_shared_source_ciphertexts_keep_every_other_field_and_version_contract( + monkeypatch: pytest.MonkeyPatch, + column: str, +) -> None: + rows = _Rows(monkeypatch, race=False) + purpose = _rotate(monkeypatch, rows, column) + for row in rows.rows: + row.update( + credential_mode="shared", + kind="gdrive", + source_kind="", + options_json="{}", + registrant_subject="", + created_by="registrant", + approval_enc="", + crawl_generation="", + crawl_credential_hash="", + ) + before = copy.deepcopy(rows.rows[-1]) + counts, lost, _checked = await cli.reseal_all(rows, dry_run=False) + [current] = rows.latest() + assert {k: v for k, v in current.items() if k != column} == { + k: v for k, v in before.items() if k != column + } + assert current["version"] == before["version"] + assert rows.cipher_writes == [("org", "namespace")] and not rows.insert_scopes + assert counts == {_TABLE + "." + column: 1} and lost == {} + monkeypatch.setattr(settings, "jwt_secret_previous", "") + assert unseal(purpose, current[column]) == "retained-service-key" + + +async def test_source_reseal_dry_run_never_writes_a_version(monkeypatch): + rows = _Rows(monkeypatch, race=False) + _rotate(monkeypatch, rows) + before = copy.deepcopy(rows.rows) + counts, lost, _checked = await cli.reseal_all(rows, dry_run=True) + assert counts == {_TABLE + ".secret_enc": 1} and lost == {} + assert rows.rows == before and not rows.insert_scopes and not rows.cipher_writes + + +async def test_source_reseal_refuses_a_success_reply_without_visible_replacement(monkeypatch): + rows = _Rows(monkeypatch, race=False) + _rotate(monkeypatch, rows) + rows.ignore_write = True + before = copy.deepcopy(rows.rows) + with pytest.raises(RuntimeError, match="source_reseal_residue"): + await cli.reseal_all(rows, dry_run=False) + assert rows.rows == before and rows.cipher_writes == [("org", "namespace")] + assert not rows.insert_scopes + + +async def test_expired_source_holder_cannot_restore_erased_metadata_from_its_saved_read( + monkeypatch, +): + rows = _Rows(monkeypatch, race=True) + purpose = _rotate(monkeypatch, rows) + rows.expire_after_read = True + with pytest.raises(leases.ClaimExpired, match="source lease expired"): + await cli.reseal_all(rows, dry_run=False) + assert rows.cleaned and rows.expired and not rows.insert_scopes + assert all( + not r["active"] + and not r["registrant_subject"] + and not r["created_by"] + and not r["approval_enc"] + and "gone@example.com" not in r["options_json"] + for r in rows.rows + ) + monkeypatch.setattr(settings, "jwt_secret_previous", "") + assert all(unseal(purpose, r["secret_enc"]) == "retained-service-key" for r in rows.rows) diff --git a/tests/unit/test_storage_layout_rows.py b/tests/unit/test_storage_layout_rows.py index 9303cd6bf3..94c96dd8cc 100644 --- a/tests/unit/test_storage_layout_rows.py +++ b/tests/unit/test_storage_layout_rows.py @@ -357,6 +357,8 @@ def _erasure_cell(table: str) -> set[str]: parts.add(f"step `{step}`") if qualified in census.CANONICAL_OWNER_TABLES: parts.add(f"canonical-owner census, `{census.CANONICAL_OWNER_TABLES[qualified]}`") + if qualified in census.CANONICAL_PERSON_TABLES: + parts.add(f"canonical-person census, `{census.CANONICAL_PERSON_TABLES[qualified]}`") if qualified in census.NO_MEMORY_CONTENT: parts.add("no content") return parts diff --git a/tests/unit/test_token_mint.py b/tests/unit/test_token_mint.py index d23dd3cc06..b6fb3b32cc 100644 --- a/tests/unit/test_token_mint.py +++ b/tests/unit/test_token_mint.py @@ -493,8 +493,11 @@ async def test_the_identity_saves_to_reads_is_the_one_the_token_verifies_as( delegated = dataclasses.replace(delegated, saves_to="a") verified = verify_token(delegated.mint("sa:cursor-ab12cd")) built = delegated.as_auth("sa:cursor-ab12cd") - # Set at verification only: the token's id and its iat/exp. - minted_only = {"token_id", "issued_at", "expires_at"} + # Verification attaches both token metadata and the checked signing-key identity. + minted_only = {"token_id", "issued_at", "expires_at", "signing_authority"} + assert built.signing_authority is None + assert isinstance(verified.signing_authority, jwt_mod.SigningAuthority) + assert verified.signing_authority.current() for field in dataclasses.fields(AuthInfo): if field.name not in minted_only: assert getattr(built, field.name) == getattr(verified, field.name), field.name