diff --git a/packages/catalog/test-subset.json b/packages/catalog/test-subset.json index 2457bbf5c6d..2cb23f652d8 100644 --- a/packages/catalog/test-subset.json +++ b/packages/catalog/test-subset.json @@ -22,6 +22,7 @@ "catalog-test-subset-test", "realm-policy-activation-test", "realm-policy-anonymous-read-test", + "realm-policy-anonymous-search-test", "realm-policy-archived-seal-test", "realm-policy-batch-test", "realm-policy-capabilities-test", diff --git a/packages/realm-server/handlers/handle-search.ts b/packages/realm-server/handlers/handle-search.ts index cf448367cbc..2b28e7f952e 100644 --- a/packages/realm-server/handlers/handle-search.ts +++ b/packages/realm-server/handlers/handle-search.ts @@ -63,6 +63,7 @@ import { } from '../search-type-watermarks.ts'; import type { CodeRef, + AnonymousCaller, DBAdapter, Realm, VirtualNetwork, @@ -137,7 +138,7 @@ export default function handleSearch(opts: { let linkShapePolicy = opts.linkShapePolicy ?? LinkShapePolicy.pinned('full'); let liveSearchCache = opts.liveSearchCache ?? new LiveSearchCache(); return async function (ctxt: Koa.Context) { - let { realmList, grantCandidates, principal } = + let { realmList, grantCandidates, principal, anonymousCallers } = getMultiRealmAuthorization(ctxt); let payload = getSearchRequestPayload(ctxt); // Every realm the request names, in the order it names them, which is the @@ -222,8 +223,9 @@ export default function handleSearch(opts: { // only once no realm the caller reads will mount. One the caller // reaches only through its policy reads the declaration on that realm's // own authority, the way the policy gate reads it. No policy admits a - // request that authenticated nobody, nor a realm-authority principal, - // so for either there are none of the latter. + // named query to a request that authenticated nobody, nor to a + // realm-authority principal, so for either there are none of the + // latter. let policyRealms = async () => principal?.kind === 'user' ? await realmsThatMayNamePolicy(grantCandidates) @@ -296,18 +298,39 @@ export default function handleSearch(opts: { // policy is asked about it. let readable = new Set(realmList); let candidates = new Set(grantCandidates); + // A request that authenticated nobody is counted by each realm that + // admitted it, against the address the request came from. + // It is counted only for a search the realm can run, so a request whose + // query won't parse is refused before any realm counts it. + let anonymousRequest = anonymousCallers + ? await fetchRequestFromContext(ctxt) + : undefined; + if (anonymousRequest) { + try { + parseSearchEntryQueryFromPayload(payload); + } catch (e) { + if (e instanceof SearchRequestError) { + await respondToInvalidSearch(ctxt, e); + return; + } + throw e; + } + } let access: RealmAccess = unresolved ? { readable: new Set(), scoped: new Map(), failed: unresolved } : await withSearchConnectionTenant(ctxt, named, () => policyAccess( readable, consultPolicies - ? named.filter((realm) => candidates.has(realm)) + ? [...new Set(named.filter((realm) => candidates.has(realm)))] : [], invocation, principal, realmsThatMayNamePolicy, mountRealms, + anonymousCallers && anonymousRequest + ? { callers: anonymousCallers, request: anonymousRequest } + : undefined, ), ); // The realms this search does work for: those it reads rows from. Each @@ -343,6 +366,13 @@ export default function handleSearch(opts: { // is a realm mounted when its `realm.json` names no policy: with no policy // there is nothing to ask, and it contributes nothing, as a realm whose // policy grants nothing does. + // + // A request that authenticated nobody is searched in each realm that + // admitted it (`anonymous`), and each counts the search against the + // caller's budget there once its grants have scoped it. A realm that turns + // the caller away, its budget spent or its count unreadable, is failed: its + // rows are withheld and the result is marked incomplete, while the other + // realms still answer. async function policyAccess( readable: Set, grantCandidates: string[], @@ -350,6 +380,7 @@ export default function handleSearch(opts: { principal: SearchPrincipal | undefined, realmsThatMayNamePolicy: (urls: string[]) => Promise, mountRealms: MountRealms, + anonymous?: { callers: Map; request: Request }, ): Promise { let access: RealmAccess = { readable, @@ -372,15 +403,33 @@ export default function handleSearch(opts: { access.failed.add(url); return; } + let caller = anonymous?.callers.get(url); + if (anonymous && !caller) { + return; + } + if (caller?.turnedAway) { + access.failed.add(url); + return; + } try { let scope = await policyQueryScope(realm.operationCore, { ...invocation, principal, transport: 'federated-search', }); - if (scope.kind === 'scoped') { - access.scoped.set(url, scope.filters); + if (scope.kind !== 'scoped') { + return; + } + if ( + anonymous && + caller && + (await realm.countAnonymousSearch(anonymous.request, caller)) + .kind !== 'counted' + ) { + access.failed.add(url); + return; } + access.scoped.set(url, scope.filters); } catch (e) { // Not a policy that could not be judged but a principal no policy // may be asked about, which only broken wiring gets here with. It @@ -526,14 +575,7 @@ export default function handleSearch(opts: { : await parseRequest(); } catch (e) { if (e instanceof SearchRequestError) { - // `invalid-query` / `invalid-render` are client request-shape errors - // → the JSON:API search-error body; anything else (bad method / JSON) - // → a plain bad request, the same split as the other search handlers. - if (e.code === 'invalid-query' || e.code === 'invalid-render') { - await setContextResponse(ctxt, buildSearchErrorResponse(e.message)); - } else { - await sendResponseForBadRequest(ctxt, e.message); - } + await respondToInvalidSearch(ctxt, e); return; } throw e; @@ -1155,3 +1197,18 @@ async function respondWithJobScopedSearchCache( ); emitTelemetry(); } + +// The answer to a search whose request is malformed. `invalid-query` and +// `invalid-render` are client request-shape errors, answered with the JSON:API +// search-error body; anything else (bad method / JSON) is a plain bad request, +// the same split as the other search handlers. +async function respondToInvalidSearch( + ctxt: Koa.Context, + e: SearchRequestError, +): Promise { + if (e.code === 'invalid-query' || e.code === 'invalid-render') { + await setContextResponse(ctxt, buildSearchErrorResponse(e.message)); + } else { + await sendResponseForBadRequest(ctxt, e.message); + } +} diff --git a/packages/realm-server/middleware/multi-realm-authorization.ts b/packages/realm-server/middleware/multi-realm-authorization.ts index d55c9eb1d2a..fd502a1c1a0 100644 --- a/packages/realm-server/middleware/multi-realm-authorization.ts +++ b/packages/realm-server/middleware/multi-realm-authorization.ts @@ -1,11 +1,18 @@ import type Koa from 'koa'; -import type { DBAdapter, Realm } from '@cardstack/runtime-common'; +import type { + AnonymousCaller, + DBAdapter, + Realm, + VirtualNetwork, +} from '@cardstack/runtime-common'; import { archivedRealmURLs, ensureTrailingSlash, fetchRealmPermissions, fetchUserPermissions, isSessionRevoked, + logger, + MAX_REALMS_PER_SEARCH_REQUEST, param, parseRealmsFromPayload, parseSearchRequestPayload, @@ -19,12 +26,16 @@ import { AuthenticationErrorMessages, } from '@cardstack/runtime-common/router'; import { + isNamedQueryPayload, + searchInvocation, searchPrincipal, type SearchPrincipal, } from '@cardstack/runtime-common/card-operations'; import type { MatrixClient } from '@cardstack/runtime-common/matrix-client'; import RealmPermissionChecker from '@cardstack/runtime-common/realm-permission-checker'; import type { RealmRegistryReconciler } from '../lib/realm-registry-reconciler.ts'; +import { mayNameRealmPolicy } from '../lib/realm-policy-pointer.ts'; +import { resolveRealmsForFederatedRequest } from '../lib/realm-routing.ts'; import { retrieveTokenClaim, type RealmServerTokenClaim, @@ -59,12 +70,19 @@ export type MultiRealmAuthorizationState = { grantCandidates: string[]; // Who the request's token was verified for: the user it names, or a // realm-authority principal — a session a realm renders its own cards - // under. Absent on a request that carried no token, which reached here only - // because every realm it names is publicly readable. + // under. For a request that carried no token, an anonymous principal where + // a realm it names admitted it through its policy (see `anonymousCallers`), + // and absent where every realm it names is publicly readable. principal?: SearchPrincipal; + // For a request that carried no token, the caller each realm it may search + // only through its policy admitted (see `Realm.admitAnonymousSearch`). Each + // counts the search against that caller's budget. These are the request's + // only grant candidates. + anonymousCallers?: Map; }; const MULTI_REALM_AUTH_STATE = 'multiRealmAuthorization'; +const log = logger('realm-server:multi-realm-authorization'); const SEARCH_REQUEST_PAYLOAD_STATE = 'searchRequestPayload'; export function multiRealmAuthorization( @@ -74,12 +92,16 @@ export function multiRealmAuthorization( realmSecretSeed, realms, reconciler, + realmsRootPath, + virtualNetwork, }: { dbAdapter: DBAdapter; matrixClient: MatrixClient; realmSecretSeed: string; realms: Realm[]; reconciler: RealmRegistryReconciler; + realmsRootPath: string; + virtualNetwork: VirtualNetwork; }, { unreadableRealms = 'refuse', @@ -161,6 +183,7 @@ export function multiRealmAuthorization( let readableRealms = new Set(); let principal: SearchPrincipal | undefined; + let anonymousCallers: Map | undefined; let authorization = ctxt.req.headers['authorization']; if (!authorization) { let publicPermissions = await fetchUserPermissions(dbAdapter, { @@ -174,18 +197,38 @@ export function multiRealmAuthorization( // Authentication comes before authorization, and a request that named // nobody is told to say who it is rather than answered for nobody. A - // policy grants by who is asking, so there is no grant for this request - // to be judged by either — which is why this refuses where an - // authenticated caller without permission does not. + // policy grants by who is asking, so such a request is judged only by + // the grants that opt in to callers who aren't signed in. On an endpoint + // that carries the realms a caller can't read, each such realm whose + // policy opens `query` to these callers, and that doesn't refuse the + // caller's address, is searched through its policy, and the rest + // contribute no rows, as they would for a signed-in caller. A named + // query is granted by its own name, which no grant opens to these + // callers. A request no realm admits is refused, which is why this + // refuses where an authenticated caller without permission does not. let realmsRequiringAuth = realmList.filter( (realmURL) => !readableRealms.has(realmURL), ); if (realmsRequiringAuth.length > 0) { - await sendResponseForUnauthorizedRequest( - ctxt, - `Authorization required for realms: ${realmsRequiringAuth.join(', ')}`, - ); - return; + let payload = (ctxt.state as Record)[ + SEARCH_REQUEST_PAYLOAD_STATE + ]; + anonymousCallers = + unreadableRealms === 'carry' && opensToAnonymousSearch(payload) + ? await admitAnonymousSearch( + await notArchived(dbAdapter, realmsRequiringAuth), + request, + { reconciler, realmsRootPath, virtualNetwork }, + ) + : new Map(); + if (anonymousCallers.size === 0) { + await sendResponseForUnauthorizedRequest( + ctxt, + `Authorization required for realms: ${realmsRequiringAuth.join(', ')}`, + ); + return; + } + principal = { kind: 'anonymous' }; } } else { let token: RealmServerTokenClaim & { iat: number; exp: number }; @@ -301,7 +344,9 @@ export function multiRealmAuthorization( if (principal?.kind !== 'realm-authority') { let archived = await archivedRealmURLs(dbAdapter, unreadable); grantCandidates = unreadable.filter( - (realmURL) => !archived.has(realmURL), + (realmURL) => + !archived.has(realmURL) && + (!anonymousCallers || anonymousCallers.has(realmURL)), ); } @@ -309,12 +354,84 @@ export function multiRealmAuthorization( realmList: readable, grantCandidates, ...(principal === undefined ? {} : { principal }), + ...(anonymousCallers ? { anonymousCallers } : {}), } satisfies MultiRealmAuthorizationState; await next(); }; } +// Whether a search could be opened to a request that authenticated nobody at +// all, judged before any realm is asked: an ad-hoc search, since a named query +// is granted by its own name, which no grant opens to such callers, that names +// a type some grant could be on. +function opensToAnonymousSearch(payload: unknown): boolean { + return ( + !isNamedQueryPayload(payload) && + (searchInvocation(payload as Parameters[0])?.types + .length ?? 0) > 0 + ); +} + +// The realms among `urls` that are not archived. An archived realm is +// sealed and serves no rows to anyone, so it is never asked to admit anyone. +async function notArchived( + dbAdapter: DBAdapter, + urls: string[], +): Promise { + let archived = await archivedRealmURLs(dbAdapter, urls); + return urls.filter((url) => !archived.has(url)); +} + +// The realms among `urls` that admit a request that authenticated nobody to a +// search, each with the caller it admitted. A request naming more realms than +// a search may fan out to admits none, so the realms a caller who isn't signed +// in can have the server mount and ask are bounded as a client's search is. A +// realm whose `realm.json` names no policy is not mounted to ask: with no +// policy there is nothing that could admit the caller. A realm that won't +// mount, or can't answer, admits nobody, and the others still answer. +async function admitAnonymousSearch( + urls: string[], + request: Request, + locate: { + reconciler: RealmRegistryReconciler; + realmsRootPath: string; + virtualNetwork: VirtualNetwork; + }, +): Promise> { + let admitted = new Map(); + urls = [...new Set(urls)]; + if (urls.length > MAX_REALMS_PER_SEARCH_REQUEST) { + return admitted; + } + let named = await Promise.all( + urls.map((url) => mayNameRealmPolicy(url, locate)), + ); + let asked = urls.filter((_url, index) => named[index]); + let mounted = await resolveRealmsForFederatedRequest( + locate.reconciler, + asked, + ); + await Promise.all( + mounted.map(async (realm, index) => { + let caller: AnonymousCaller | undefined; + try { + caller = await realm?.admitAnonymousSearch(request); + } catch (e) { + log.warn( + `${asked[index]} could not say whether it admits a search from a caller who isn't signed in, so it admits none`, + e, + ); + return; + } + if (caller) { + admitted.set(asked[index], caller); + } + }), + ); + return admitted; +} + // Whether a realm lets a user read it, judged as the realm judges a session // that carries only `read`. An archived realm is sealed, and lets no one read // it. diff --git a/packages/realm-server/tests/realm-policy-anonymous-search-test.ts b/packages/realm-server/tests/realm-policy-anonymous-search-test.ts new file mode 100644 index 00000000000..9a984db324a --- /dev/null +++ b/packages/realm-server/tests/realm-policy-anonymous-search-test.ts @@ -0,0 +1,545 @@ +import QUnit from 'qunit'; +const { module, test } = QUnit; +import supertest from 'supertest'; +import type { Test, SuperTest, Response } from 'supertest'; +import { basename, join } from 'path'; +import { dirSync } from 'tmp'; +import { + archiveRealm, + MAX_REALMS_PER_SEARCH_REQUEST, + parseAddressRanges, + rri, + SupportedMimeType, +} from '@cardstack/runtime-common'; +import type { + QueuePublisher, + QueueRunner, + Realm, + RealmPermissions, +} from '@cardstack/runtime-common'; +import { + setAnonymousRequestSink, + type AnonymousRequestEvent, +} from '@cardstack/runtime-common/card-operations/telemetry'; +import type { PgAdapter } from '@cardstack/postgres'; +import { resetCatalogRealms } from '../handlers/handle-fetch-catalog-realms.ts'; +import type { RealmHttpServer as Server } from '../server.ts'; +import { + clearOfRateLimitWindowEdge, + closeServer, + createJWT, + createVirtualNetwork, + matrixURL, + realmConfigCardJSON, + runTestRealmServerWithRealms, + setupDB, + setupTestDatabaseTemplate, +} from './helpers/index.ts'; +import { setupCatalogTestSubset } from './helpers/catalog-test-subset.ts'; + +// Two realms nobody may read without signing in, governed by policy cards in +// an Org realm. The Newsroom's policy opens searches over its published +// articles to callers who aren't signed in. The Gazette's opens the same. The +// Library's policy opens nothing to such callers. +const NEWSROOM = 'http://127.0.0.1:4444/newsroom/'; +const GAZETTE = 'http://127.0.0.1:4444/gazette/'; +const LIBRARY = 'http://127.0.0.1:4444/library/'; +// A realm anyone may read, which a search reads outright. +const WIRE = 'http://127.0.0.1:4444/wire/'; +const ORG = 'http://127.0.0.1:4444/org/'; +const NEWSROOM_POLICY = `${ORG}policies/newsroom`; +const GAZETTE_POLICY = `${ORG}policies/gazette`; +const LIBRARY_POLICY = `${ORG}policies/library`; +const EDITOR = '@editor:localhost'; +const ORG_ADMIN = '@org-admin:localhost'; + +// Callers' addresses come from the ranges reserved for documentation. +const VISITOR = '192.0.2.10'; +const BLOCKED_VISITOR = '198.51.100.7'; +const OUR_SERVICES = '203.0.113.5'; + +const REALM_POLICY = { + module: rri('@cardstack/catalog/realm-policy/realm-policy'), + name: 'RealmPolicy', +}; + +const ARTICLE_MODULE = ` + import { contains, containsMany, field, CardDef } from "@cardstack/base/card-api"; + import StringField from "@cardstack/base/string"; + export class Article extends CardDef { + @field headline = contains(StringField); + @field status = contains(StringField); + @field authorIds = containsMany(StringField); + } +`; + +function articleType(realm: string) { + return { module: `${realm}article`, name: 'Article' }; +} + +function article(headline: string, status: string, authorIds: string[] = []) { + return JSON.stringify({ + data: { + type: 'card', + attributes: { headline, status, authorIds }, + meta: { adoptsFrom: { module: '../article', name: 'Article' } }, + }, + }); +} + +// The grants every governed realm's policy carries for signed-in callers, +// plus, where `anonymous`, the ones that open searches to callers who aren't +// signed in: the published articles, and a grant that reads the caller, which +// never admits one. +function policyCard(realm: string, anonymous: boolean) { + let targetType = articleType(realm); + let grants: Record[] = [ + { operation: 'query', where: '.status == "published"' }, + ]; + if (anonymous) { + grants.push( + { + operation: 'query', + anonymous: true, + where: '.status == "published"', + }, + { + operation: 'query', + anonymous: true, + where: '.authorIds | any(. == actor())', + }, + ); + } + return JSON.stringify({ + data: { + type: 'card', + attributes: { rules: [{ targetType, grants }] }, + meta: { adoptsFrom: REALM_POLICY }, + }, + }); +} + +function governedRealm(url: string, name: string, policy: string) { + return { + realmURL: new URL(url), + fileSystem: { + 'realm.json': realmConfigCardJSON({ name, policy }), + 'article.gts': ARTICLE_MODULE, + 'articles/published.json': article(`${name} published`, 'published'), + 'articles/draft.json': article(`${name} draft`, 'draft', [EDITOR]), + }, + // The org admin reads the realms the org's policies govern, which is + // what lets them ask a policy what it decides there. + permissions: { + [EDITOR]: ['read', 'write', 'realm-owner'], + [ORG_ADMIN]: ['read'], + } satisfies RealmPermissions as RealmPermissions, + }; +} + +module(basename(import.meta.filename), function (hooks) { + let newsroom: Realm; + let gazette: Realm; + let library: Realm; + let wire: Realm; + let org: Realm; + let db: PgAdapter; + let request: SuperTest; + let server: Server; + let records: AnonymousRequestEvent[]; + + setupCatalogTestSubset(hooks); + + async function start({ + dbAdapter, + publisher, + runner, + }: { + dbAdapter: PgAdapter; + publisher: QueuePublisher; + runner: QueueRunner; + }) { + let result = await runTestRealmServerWithRealms({ + virtualNetwork: createVirtualNetwork(), + realmsRootPath: join(dirSync().name, 'realm_server_1'), + clientAddress: { + trustedProxyHops: 1, + infraAddresses: parseAddressRanges([OUR_SERVICES]).ranges, + }, + realms: [ + governedRealm(NEWSROOM, 'Newsroom', NEWSROOM_POLICY), + governedRealm(GAZETTE, 'Gazette', GAZETTE_POLICY), + governedRealm(LIBRARY, 'Library', LIBRARY_POLICY), + { + realmURL: new URL(WIRE), + fileSystem: { + 'realm.json': realmConfigCardJSON({ name: 'Wire' }), + 'article.gts': ARTICLE_MODULE, + 'articles/published.json': article('Wire published', 'published'), + }, + permissions: { '*': ['read'], [EDITOR]: ['read', 'write'] }, + }, + { + realmURL: new URL(ORG), + fileSystem: { + 'realm.json': realmConfigCardJSON({ name: 'Org' }), + 'policies/newsroom.json': policyCard(NEWSROOM, true), + 'policies/gazette.json': policyCard(GAZETTE, true), + 'policies/library.json': policyCard(LIBRARY, false), + }, + permissions: { [ORG_ADMIN]: ['read', 'write', 'realm-owner'] }, + }, + ], + dbAdapter, + publisher, + runner, + matrixURL, + }); + server = result.testRealmHttpServer; + request = supertest(server); + let realm = (url: string) => + result.realms.find((candidate) => candidate.url === url)!; + newsroom = realm(NEWSROOM); + gazette = realm(GAZETTE); + library = realm(LIBRARY); + wire = realm(WIRE); + org = realm(ORG); + } + + async function stop() { + for (let realm of [newsroom, gazette, library, wire, org]) { + realm.__testOnlyClearCaches(); + realm.unsubscribe(); + } + await closeServer(server); + resetCatalogRealms(); + } + + let templateDatabase = setupTestDatabaseTemplate(hooks, { + key: import.meta.filename, + build: async (args) => { + await start(args); + return stop; + }, + }); + + setupDB(hooks, { + templateDatabase, + beforeEach: async (dbAdapter, publisher, runner) => { + db = dbAdapter; + await start({ dbAdapter, publisher, runner }); + records = []; + setAnonymousRequestSink((record) => records.push(record)); + }, + afterEach: async () => { + setAnonymousRequestSink(undefined); + await stop(); + }, + }); + + function articles(realm: string) { + return { filter: { 'item.on': articleType(realm) } }; + } + + // A search of one realm's own `_search`, from `from`. + function realmSearch(realm: string, from: string = VISITOR) { + return request + .post(`${new URL(realm).pathname}_search`) + .set('Accept', SupportedMimeType.CardJson) + .set('Content-Type', 'application/json') + .set('X-HTTP-Method-Override', 'QUERY') + .set('X-Forwarded-For', from) + .send(articles(realm)); + } + + // A search across `realms` on the realm server, of each one's articles. + function federatedSearch( + realms: string[], + from: string = VISITOR, + extra: Record = {}, + ) { + return request + .post('/_federated-search') + .set('Accept', SupportedMimeType.CardJson) + .set('Content-Type', 'application/json') + .set('X-HTTP-Method-Override', 'QUERY') + .set('X-Forwarded-For', from) + .send({ + filter: { + any: [...new Set(realms)].map((realm) => ({ + 'item.on': articleType(realm), + })), + }, + realms, + ...extra, + }); + } + + function ids(response: Response): string[] { + if (response.status !== 200) { + throw new Error(`search answered ${response.status}: ${response.text}`); + } + return (response.body as { data: { id: string }[] }).data + .map(({ id }) => id) + .sort(); + } + + async function setLimit(realm: Realm, requests: number) { + let name = realm === newsroom ? 'Newsroom' : 'Gazette'; + let policy = realm === newsroom ? NEWSROOM_POLICY : GAZETTE_POLICY; + await realm.write( + 'realm.json', + realmConfigCardJSON({ + name, + policy, + anonymousRateLimit: { requests, windowSeconds: 600 }, + }), + ); + await realm.indexing(); + await clearOfRateLimitWindowEdge({ windowSeconds: 600 }); + } + + function unauthenticated(response: Response, label: string, assert: Assert) { + assert.strictEqual(response.status, 401, `${label}: 401`); + } + + test("a realm's own search answers a caller who is not signed in with the rows its anonymous grants admit", async function (assert) { + let response = await realmSearch(NEWSROOM); + assert.deepEqual( + ids(response), + [`${NEWSROOM}articles/published`], + 'the published article, and not the draft its author is named on', + ); + let [record] = records.filter((r) => r.route.endsWith('/_search')); + assert.strictEqual(record?.outcome, 'admitted', 'counted'); + assert.strictEqual(record?.operation, 'query'); + assert.strictEqual(record?.clientIP, VISITOR); + }); + + test('a realm whose policy opens no search to such callers tells them to authenticate', async function (assert) { + unauthenticated(await realmSearch(LIBRARY), 'library', assert); + }); + + test("a realm's own search is counted against the address, and refused once it is spent", async function (assert) { + await setLimit(newsroom, 1); + assert.strictEqual((await realmSearch(NEWSROOM)).status, 200); + let over = await realmSearch(NEWSROOM); + assert.strictEqual(over.status, 429, 'the second search is refused'); + assert.strictEqual(over.body.errors[0].code, 'rate-limited'); + }); + + test('a blocked address is told to authenticate', async function (assert) { + await newsroom.write( + 'realm.json', + realmConfigCardJSON({ + name: 'Newsroom', + policy: NEWSROOM_POLICY, + anonymousBlocklist: [`${BLOCKED_VISITOR}/32`], + }), + ); + await newsroom.indexing(); + unauthenticated( + await realmSearch(NEWSROOM, BLOCKED_VISITOR), + 'a blocked address', + assert, + ); + unauthenticated( + await federatedSearch([NEWSROOM], BLOCKED_VISITOR), + 'a blocked address, across realms', + assert, + ); + }); + + test('a search across realms answers such a caller with what each realm admits, alongside a realm anyone reads', async function (assert) { + let response = await federatedSearch([NEWSROOM, GAZETTE, WIRE]); + assert.deepEqual(ids(response), [ + `${GAZETTE}articles/published`, + `${NEWSROOM}articles/published`, + `${WIRE}articles/published`, + ]); + assert.notOk(response.body.meta?.incomplete, 'nothing failed'); + let counted = records + .filter((r) => r.route.endsWith('/_federated-search')) + .map((r) => [r.realmURL, r.outcome]) + .sort(); + assert.deepEqual( + counted, + [ + [GAZETTE, 'admitted'], + [NEWSROOM, 'admitted'], + ], + 'each realm searched through its policy counted it, against its own budget', + ); + }); + + test('a realm that admits nothing contributes no rows to a search another realm admits', async function (assert) { + let response = await federatedSearch([NEWSROOM, LIBRARY]); + assert.deepEqual(ids(response), [`${NEWSROOM}articles/published`]); + assert.notOk(response.body.meta?.incomplete); + }); + + test('a search naming more realms than a search may fan out to is told to authenticate, and no realm is asked', async function (assert) { + let realms = [NEWSROOM, GAZETTE, LIBRARY]; + assert.true( + realms.length > MAX_REALMS_PER_SEARCH_REQUEST, + 'precondition: more private realms than a search may fan out to', + ); + unauthenticated(await federatedSearch(realms), 'too many realms', assert); + assert.deepEqual(records, [], 'no realm was asked'); + }); + + test('a realm named twice is counted once', async function (assert) { + let response = await federatedSearch([NEWSROOM, NEWSROOM]); + assert.strictEqual(response.status, 200, response.text); + assert.strictEqual( + records.filter((r) => r.outcome === 'admitted').length, + 1, + ); + }); + + test('an archived realm admits nobody to a search', async function (assert) { + await archiveRealm(db, new URL(NEWSROOM)); + unauthenticated( + await federatedSearch([NEWSROOM]), + 'only an archived realm', + assert, + ); + let response = await federatedSearch([NEWSROOM, GAZETTE]); + assert.deepEqual(ids(response), [`${GAZETTE}articles/published`]); + assert.deepEqual( + records.map((r) => r.realmURL), + [GAZETTE], + 'the archived realm was never asked', + ); + }); + + test('a malformed search is refused before any realm counts it', async function (assert) { + await setLimit(newsroom, 1); + let response = await federatedSearch([NEWSROOM], VISITOR, { + page: { size: 'many' }, + }); + assert.strictEqual(response.status, 400, response.text); + assert.strictEqual( + (await realmSearch(NEWSROOM)).status, + 200, + 'the budget is untouched', + ); + }); + + test('an endpoint that refuses realms a caller cannot read still tells such a caller to authenticate', async function (assert) { + let response = await request + .post('/_federated-info') + .set('Accept', SupportedMimeType.JSONAPI) + .set('Content-Type', 'application/json') + .set('X-HTTP-Method-Override', 'QUERY') + .set('X-Forwarded-For', VISITOR) + .send({ realms: [NEWSROOM] }); + unauthenticated(response, '_federated-info', assert); + assert.deepEqual(records, [], 'no realm was asked'); + }); + + test("a named query on a realm's own search tells such a caller to authenticate", async function (assert) { + let response = await request + .post(`${new URL(NEWSROOM).pathname}_search`) + .set('Accept', SupportedMimeType.CardJson) + .set('Content-Type', 'application/json') + .set('X-HTTP-Method-Override', 'QUERY') + .set('X-Forwarded-For', VISITOR) + .send({ operation: 'listMine', on: articleType(NEWSROOM) }); + unauthenticated(response, 'a named query', assert); + }); + + test('a search across realms none of which admits such a caller tells them to authenticate', async function (assert) { + unauthenticated(await federatedSearch([LIBRARY]), 'library', assert); + }); + + test('a realm whose budget is spent is left out of a search across realms, which says it is incomplete', async function (assert) { + await setLimit(newsroom, 1); + assert.strictEqual((await realmSearch(NEWSROOM)).status, 200); + let response = await federatedSearch([NEWSROOM, GAZETTE]); + assert.deepEqual( + ids(response), + [`${GAZETTE}articles/published`], + 'the other realm still answers', + ); + assert.true(response.body.meta?.incomplete, 'marked incomplete'); + let only = await federatedSearch([NEWSROOM]); + assert.deepEqual(ids(only), [], 'alone, it answers no rows'); + assert.true(only.body.meta?.incomplete); + }); + + test('a named query is never opened to such a caller across realms', async function (assert) { + let response = await request + .post('/_federated-search') + .set('Accept', SupportedMimeType.CardJson) + .set('Content-Type', 'application/json') + .set('X-HTTP-Method-Override', 'QUERY') + .set('X-Forwarded-For', VISITOR) + .send({ + operation: 'listMine', + on: articleType(NEWSROOM), + realms: [NEWSROOM], + }); + unauthenticated(response, 'a named query', assert); + }); + + test('a capability check answers such a caller about a search as the search would', async function (assert) { + let ask = (realm: string) => + request + .post(`${new URL(realm).pathname}_capabilities`) + .set('Accept', SupportedMimeType.JSON) + .set('Content-Type', SupportedMimeType.JSON) + .set('X-Forwarded-For', VISITOR) + .send({ + checks: [{ target: articleType(realm), operation: 'query' }], + }); + let response = await ask(NEWSROOM); + assert.strictEqual(response.status, 200, response.text); + assert.true(response.body.checks[0].allowed, 'the newsroom opens it'); + }); + + test('explain judges such a caller’s search by the grants that opt in to them', async function (assert) { + let explain = async (policy: string, realm: string) => { + let response = await request + .post(`${new URL(ORG).pathname}_operations`) + .set('X-HTTP-Method-Override', 'QUERY') + .set('Accept', SupportedMimeType.BoxelOperations) + .set('Content-Type', SupportedMimeType.BoxelOperations) + .set( + 'Authorization', + `Bearer ${createJWT(org, ORG_ADMIN, ['read', 'write', 'realm-owner'])}`, + ) + .send( + JSON.stringify({ + 'boxel:operations': [ + { + op: 'invoke', + 'boxel:name': 'explain', + href: policy, + data: { + actor: '', + target: realm, + operation: 'query', + search: { filter: { 'item.on': articleType(realm) } }, + }, + }, + ], + }), + ); + if (response.status !== 200) { + throw new Error( + `explain answered ${response.status}: ${response.text}`, + ); + } + return response.body['atomic:results'][0]; + }; + let newsroomAnswer = await explain(NEWSROOM_POLICY, NEWSROOM); + assert.strictEqual(newsroomAnswer.reason, 'granted', 'the newsroom'); + let libraryAnswer = await explain(LIBRARY_POLICY, LIBRARY); + assert.strictEqual( + libraryAnswer.reason, + 'actor-required', + 'a policy that opens nothing tells such a caller to authenticate', + ); + }); +}); diff --git a/packages/runtime-common/anonymous-admission.ts b/packages/runtime-common/anonymous-admission.ts index 07328e40ae1..621457af3dd 100644 --- a/packages/runtime-common/anonymous-admission.ts +++ b/packages/runtime-common/anonymous-admission.ts @@ -73,16 +73,28 @@ export interface AnonymousCaller { // What the caller is answered with in place of running the request, when // the realm can't count it: its address has used up the realm's limit, or // the count couldn't be read. Settled at admission, before anything runs. - turnedAway?: - | { kind: 'rate-limited'; retryAfterSeconds: number } - | { kind: 'unavailable' }; + turnedAway?: Exclude; } +// What came of counting an anonymous caller's invocation against its +// address's budget: counted, refused for want of budget, or not counted +// because the count couldn't be read or updated. +export type AnonymousCount = + | { kind: 'counted' } + | { kind: 'rate-limited'; retryAfterSeconds: number } + | { kind: 'unavailable' }; + // The card+json read: what an anonymous grant on `read` opens. export const ANONYMOUS_CARD_READ: AnonymousDispatch = { operations: ['read'], charged: true, }; +// A search of the realm: what a `query` grant opens. It is counted once its +// grants have scoped it, whatever rows it finds. +export const ANONYMOUS_SEARCH: AnonymousDispatch = { + operations: ['query'], + charged: true, +}; // A data file's or a card's stored bytes: what a `readSource` grant opens. export const ANONYMOUS_BYTES_READ: AnonymousDispatch = { operations: ['readSource'], @@ -258,9 +270,31 @@ export class AnonymousAdmission { if (!caller || cost < 1) { return undefined; } + let counted = await this.chargeCaller(request, caller, cost); + switch (counted.kind) { + case 'counted': + return undefined; + case 'rate-limited': + return this.#rateLimitedResponse( + requestContext, + counted.retryAfterSeconds, + ); + case 'unavailable': + return this.#countUnavailableResponse(requestContext); + } + } + + // Counts `cost` invocations by `caller` against its address's budget in + // this realm, and records what came of it. A caller of ours is recorded and + // never counted. + async chargeCaller( + request: Request, + caller: AnonymousCaller, + cost: number, + ): Promise { if (caller.infra) { this.#record(request, caller, { outcome: 'infra' }); - return undefined; + return { kind: 'counted' }; } let { limit, limitFrom } = await this.#env.access(); let outcome: AnonymousRateOutcome; @@ -273,7 +307,7 @@ export class AnonymousAdmission { }); } catch (e: unknown) { this.#recordCountFailure(request, caller, e); - return this.#countUnavailableResponse(requestContext); + return { kind: 'unavailable' }; } let recordedLimit = { ...limit, from: limitFrom }; let costDetail = cost > 1 ? { cost } : {}; @@ -284,7 +318,7 @@ export class AnonymousAdmission { count: outcome.count, ...costDetail, }); - return undefined; + return { kind: 'counted' }; } this.#record(request, caller, { outcome: 'rate-limited', @@ -292,7 +326,10 @@ export class AnonymousAdmission { retryAfterSeconds: outcome.retryAfterSeconds, ...costDetail, }); - return this.#rateLimitedResponse(requestContext, outcome.retryAfterSeconds); + return { + kind: 'rate-limited', + retryAfterSeconds: outcome.retryAfterSeconds, + }; } // Whether the caller's address has budget left for one more invocation, diff --git a/packages/runtime-common/card-operations/explain.ts b/packages/runtime-common/card-operations/explain.ts index 279155a5046..8b56f8c54c1 100644 --- a/packages/runtime-common/card-operations/explain.ts +++ b/packages/runtime-common/card-operations/explain.ts @@ -473,7 +473,13 @@ async function explainSearch( if (acl.read) { return { ...base, decision: 'allowed', reason: 'acl' }; } - if (actor.kind !== 'user') { + // A caller who isn't signed in is judged, as the search judges one, by the + // grants that opt in to such callers, where the policy opens the operation + // to them, and told to authenticate everywhere else. + if ( + actor.kind !== 'user' && + !(await opensToAnonymous(core, invocation.operation)) + ) { return refused(base, 'actor-required', { status: 401, code: 'actor-required', @@ -495,7 +501,10 @@ async function explainSearch( scope = await policyQueryScope(core, { operation: invocation.operation, types: invocation.types, - principal: { kind: 'user', user: actor.actor }, + principal: + actor.kind === 'user' + ? { kind: 'user', user: actor.actor } + : { kind: 'anonymous' }, transport: 'explain', hypothetical: true, }); diff --git a/packages/runtime-common/card-operations/policy-query.ts b/packages/runtime-common/card-operations/policy-query.ts index 54ef5b5557f..43b83c590c7 100644 --- a/packages/runtime-common/card-operations/policy-query.ts +++ b/packages/runtime-common/card-operations/policy-query.ts @@ -111,11 +111,14 @@ export type PolicyQueryScope = const DENIED: PolicyQueryScope = { kind: 'denied' }; // Who a search runs for. A realm-authority principal is a session a realm -// renders its own cards under (`TokenClaims.realmAuthority`). Any other is the -// user its session names, including a render a user asked for. +// renders its own cards under (`TokenClaims.realmAuthority`). An anonymous +// principal is a request that authenticated nobody, which the realm admitted +// because its policy opens `query` to such callers. Any other is the user its +// session names, including a render a user asked for. export type SearchPrincipal = | { kind: 'user'; user: string } - | { kind: 'realm-authority'; user: string }; + | { kind: 'realm-authority'; user: string } + | { kind: 'anonymous' }; // The principal a request authenticated as, or none for a request that // authenticated nobody. @@ -156,10 +159,10 @@ export class RealmAuthorityPolicyScopeError extends Error { // gate's is: a policy card commonly lives in a realm the caller cannot read. // // `principal` is required rather than optional. A policy grants by who is -// asking, so a request that authenticated nobody has no grant to be judged -// by, whatever the policy holds — the rule the realm already applies before -// it hands a refused request to the gate at all. A realm-authority principal -// is not someone asking either, and is refused with +// asking, so a request that authenticated nobody is judged only as an +// anonymous principal, which the realm makes of one only where its policy +// opens `query` to such callers, and only by the grants that opt in to them. +// A realm-authority principal is not someone asking, and is refused with // `RealmAuthorityPolicyScopeError` before any type is judged or the policy is // read. // @@ -197,7 +200,7 @@ export async function policyQueryScope( emitPolicySearchScope({ kind: 'policy-search-scope', realmURL: core.realmURL, - actor: principal.user, + actor: principal.kind === 'user' ? principal.user : null, operation, types: types.map(typeLabel).join(','), transport: invocation.transport ?? 'search', @@ -211,7 +214,13 @@ export async function policyQueryScope( }; let scope: PolicyQueryScope; try { - scope = await scopeFor(core, operation, types, principal.user, contributed); + scope = await scopeFor( + core, + operation, + types, + principal.kind === 'user' ? principal.user : undefined, + contributed, + ); } catch (e: unknown) { record('failed'); throw e; @@ -246,7 +255,7 @@ async function scopeFor( core: OperationCore, operation: string, types: readonly CodeRef[], - actor: string, + actor: string | undefined, contributed: MatchedGrant[], ): Promise { let distinct = [ @@ -311,8 +320,10 @@ function excluding(filter: Filter, excluded: Filter[]): Filter { } // What this realm's policy contributes to a search `principal` sends, where -// the request may have authenticated nobody. Only a user is granted anything. -// A request that authenticated nobody has no grant to be judged by. A +// the request may have authenticated nobody. A user is judged by every grant, +// and an anonymous principal by the grants that opt in to callers who aren't +// signed in. A request that authenticated nobody and that the realm didn't +// admit as an anonymous principal has no grant to be judged by. A // realm-authority principal is a render, and what a render produces is served // to every viewer, so it reads what the ACL grants it and the policy is never // asked about it. @@ -324,7 +335,7 @@ export async function principalQueryScope( } & SearchScopeRecording, principal: SearchPrincipal | undefined, ): Promise { - return principal?.kind === 'user' + return principal?.kind === 'user' || principal?.kind === 'anonymous' ? await policyQueryScope(core, { ...invocation, principal }) : DENIED; } @@ -334,7 +345,7 @@ async function typeScope( core: OperationCore, operation: string, on: CodeRef, - actor: string, + actor: string | undefined, contributed: MatchedGrant[], ): Promise { let policy = await core.policy?.compiledPolicy(); @@ -405,7 +416,10 @@ function ownDeclaration( // Every matching grant's filter, with the caller filled in, in the grammar the // engine runs, each comparison in it kept from judging a card whose type reads -// the compared path differently from the rule's type. +// the compared path differently from the rule's type. For a caller with no +// actor, one who isn't signed in, only a grant that opts in to such callers +// matches, and never one whose filter reads the caller, which has no one to +// stand for. // // A compiled filter stands the caller as the `{ $ref: 'actor' }` marker a // declared query uses, so filling one in is the substitution a named query @@ -418,7 +432,7 @@ async function grantFilters( policy: CompiledRealmPolicy, types: string[], operation: string, - actor: string, + actor: string | undefined, core: OperationCore, ): Promise<{ filter: Filter; matched: MatchedGrant }[]> { let matched = await matchingGrants(policy, types, operation, core.policy!); @@ -428,9 +442,12 @@ async function grantFilters( if (!grant.filter) { continue; } + if (actor === undefined && (!grant.anonymous || grant.where?.readsActor)) { + continue; + } let bound = lowerQueryOperation( { base: 'query', query: { filter: grant.filter } }, - { actor }, + actor === undefined ? {} : { actor }, ); if (!bound.filter) { throw new Error( diff --git a/packages/runtime-common/index.ts b/packages/runtime-common/index.ts index cbf7bc85f48..83b51892491 100644 --- a/packages/runtime-common/index.ts +++ b/packages/runtime-common/index.ts @@ -1887,6 +1887,7 @@ export * from './tasks/index.ts'; export * from './worker.ts'; export * from './stream.ts'; export * from './realm.ts'; +export type { AnonymousCaller, AnonymousCount } from './anonymous-admission.ts'; export * from './realm-index-updater.ts'; export * from './fetcher.ts'; export * from './test-waiters.ts'; diff --git a/packages/runtime-common/realm.ts b/packages/runtime-common/realm.ts index 2e58e4f08cb..dd7f2db6ced 100644 --- a/packages/runtime-common/realm.ts +++ b/packages/runtime-common/realm.ts @@ -9,10 +9,12 @@ import { ANONYMOUS_BYTES_READ, ANONYMOUS_CAPABILITY_CHECK, ANONYMOUS_CARD_READ, + ANONYMOUS_SEARCH, ANONYMOUS_STYLESHEET, AnonymousAdmission, servedAnonymous, type AnonymousCaller, + type AnonymousCount, type AnonymousDispatch, } from './anonymous-admission.ts'; import { @@ -2929,13 +2931,19 @@ export class Realm { '/_search', SupportedMimeType.CardJson, this.searchEntriesResponse.bind(this), - APPLIES_ARCHIVED_SEAL, + { + ...APPLIES_ARCHIVED_SEAL, + anonymous: ANONYMOUS_SEARCH, + }, ) .query( '/_search', SupportedMimeType.CardJson, this.searchEntriesResponse.bind(this), - APPLIES_ARCHIVED_SEAL, + { + ...APPLIES_ARCHIVED_SEAL, + anonymous: ANONYMOUS_SEARCH, + }, ) // Answered on the realm ACL alone, as `_lint` and `_mtimes` are. .get( @@ -7195,6 +7203,9 @@ export class Realm { #searchPrincipal( requestContext: RequestContext, ): SearchPrincipal | undefined { + if (requestContext.anonymousCaller) { + return { kind: 'anonymous' }; + } return searchPrincipal( requestContext.authenticatedUser, requestContext.realmAuthority, @@ -8078,6 +8089,34 @@ export class Realm { return true; } + // Whether this realm admits a request that authenticated nobody to a search + // that spans several realms, which the realm server answers rather than this + // realm's own routes. The same question this realm's own `_search` asks: its + // policy opens `query` to such callers, and it doesn't refuse the caller's + // address. The caller that comes back is the one its grants are composed + // for and its budget is counted under (see `countAnonymousSearch`); one + // whose address has no budget left comes back marked as turned away. + async admitAnonymousSearch( + request: Request, + ): Promise { + return await this.#anonymous.callerFor(request, ANONYMOUS_SEARCH); + } + + // Counts one search of this realm by a caller `admitAnonymousSearch` + // admitted, once the realm's grants have scoped it, against the caller's + // budget here. A search that spans several realms is counted by each one it + // reads rows from, and a realm that turns the caller away contributes no + // rows to it. + async countAnonymousSearch( + request: Request, + caller: AnonymousCaller, + ): Promise { + if (caller.turnedAway) { + return caller.turnedAway; + } + return await this.#anonymous.chargeCaller(request, caller, 1); + } + // Whether the realm's policy is the one to judge a caller the ACL refused // with `refusal`: the realm names a policy, and the refusal was of a verified // user's permission rather than of a request that authenticated nobody. @@ -14334,6 +14373,12 @@ export class Realm { let declaredLinks: LinkStrategy | undefined; let unshareableFormats: PrerenderedHtmlFormat[] = []; if (isNamedQueryPayload(payload)) { + // A named query is granted by its own name, which no grant opens to a + // caller who isn't signed in, so one is told to authenticate before the + // declaration is read. + if (requestContext.anonymousCaller) { + return this.#authenticationRequired(requestContext); + } // A named query searches this realm and no other, so this realm is the // whole of the scope it may resolve to. try { @@ -14424,6 +14469,9 @@ export class Realm { return noRows(); } if (policyScope) { + // A caller admitted without a session is counted for a search its + // grants scoped, whatever rows it finds. + servedAnonymous(requestContext); // Composed before the page is applied below, so the page the engine // fills is a page of rows the policy admits rather than a page of the // caller's rows with some removed.