diff --git a/packages/realm-server/handlers/handle-search.ts b/packages/realm-server/handlers/handle-search.ts index e259bdf9400..d87e7a78275 100644 --- a/packages/realm-server/handlers/handle-search.ts +++ b/packages/realm-server/handlers/handle-search.ts @@ -154,6 +154,11 @@ export default function handleSearch(opts: { // assembly pass entirely: the host re-resolves every result from its raw // card+source file, so the transitive `included[]` expansion is // throwaway work in this path. Same gating as `cacheOnlyDefinitions`. + // + // This is also what keeps the assembled-resource budget off a render's + // search: the pass the budget bounds does not run at all here, so there is + // nothing to exempt. The budget's exemption is carried by the routes that + // do run the pass during a render — the card+html entry leg. let omitIncluded = cacheOnlyDefinitions; let jobPriority = sanitizeJobPriorityHeader( ctxt.get(PRERENDER_JOB_PRIORITY_HEADER), diff --git a/packages/realm-server/tests/card-endpoints-test.ts b/packages/realm-server/tests/card-endpoints-test.ts index e30bb1f9f14..5814ddd78fd 100644 --- a/packages/realm-server/tests/card-endpoints-test.ts +++ b/packages/realm-server/tests/card-endpoints-test.ts @@ -862,8 +862,8 @@ module(basename(import.meta.filename), function () { let etag = response.get('etag') ?? ''; assert.ok(etag, 'response carries an ETag'); assert.true( - /^"\d+(?:-[0-9a-f]+)?:card-rri"$/.test(etag), - `ETag matches "(-)?:card-rri" pattern (got ${etag})`, + /^"\d+(?:-[0-9a-f]+)?:card-rri-lb\d+"$/.test(etag), + `ETag matches "(-)?:card-rri-lb" pattern (got ${etag})`, ); assert.strictEqual( response.get('cache-control'), @@ -3733,7 +3733,7 @@ module(basename(import.meta.filename), function () { 'old ETag no longer matches → fresh 200', ); assert.true( - /^"\d+(?:-[0-9a-f]+)?:card-rri"$/.test( + /^"\d+(?:-[0-9a-f]+)?:card-rri-lb\d+"$/.test( staleResponse.get('etag') ?? '', ), `GET reports a validator for the read shape (got ${staleResponse.get('etag')})`, @@ -3818,7 +3818,10 @@ module(basename(import.meta.filename), function () { // caller from treating the echo as the card's read representation. assert.strictEqual( patchResponse.get('etag'), - (initialEtag ?? '').replace(/:card-rri"$/, ':card-rri-write-echo"'), + (initialEtag ?? '').replace( + /:card-rri-lb\d+"$/, + ':card-rri-write-echo"', + ), 'no-op PATCH validates the unchanged state under the write-echo shape', ); assert.notStrictEqual( diff --git a/packages/realm-server/tests/card-html-endpoints-test.ts b/packages/realm-server/tests/card-html-endpoints-test.ts index 4060a7b9a8f..58e971a8e24 100644 --- a/packages/realm-server/tests/card-html-endpoints-test.ts +++ b/packages/realm-server/tests/card-html-endpoints-test.ts @@ -270,13 +270,14 @@ module(basename(import.meta.filename), function () { ), 'the card item rides in included', ); - // An item carries `meta.realmInfo` (which can change without a reindex), - // so an item-bearing response folds the realm-info hash in as a third - // segment on top of the `:none` composite. + // An item carries `meta.realmInfo` (which can change without a reindex) + // and is assembled under the link budget (whose value decides what the + // item's closure holds), so an item-bearing response folds both in on top + // of the `:none` composite. A pure-html response carries neither. let etag = response.get('etag') ?? ''; assert.true( - /^"\d+:none:[^:"]+"$/.test(etag), - `an item response has no rendering channel + a realm-info segment, got ${etag}`, + /^"\d+:none:[^:"]+:lb\d+"$/.test(etag), + `an item response has no rendering channel + realm-info and budget segments, got ${etag}`, ); }); @@ -320,10 +321,11 @@ module(basename(import.meta.filename), function () { assert.strictEqual(data.type, 'entry'); // A file renders natively; whichever branch it resolves, the ETag pairs // the two channels (a rendering → `:`, else `:none`), plus - // a realm-info segment when it falls back to its item. + // realm-info and budget segments when it falls back to its item — both + // ride on an item and neither on a pure rendering. let etag = response.get('etag') ?? ''; assert.true( - /^"\d+:(\d+|none)(:[^:"]+)?"$/.test(etag), + /^"\d+:(\d+|none)(:[^:"]+:lb\d+)?"$/.test(etag), `the file entry carries a composite ETag, got ${etag}`, ); }); diff --git a/packages/realm-server/tests/link-assembly-budget-test.ts b/packages/realm-server/tests/link-assembly-budget-test.ts new file mode 100644 index 00000000000..d66249c969e --- /dev/null +++ b/packages/realm-server/tests/link-assembly-budget-test.ts @@ -0,0 +1,716 @@ +import QUnit from 'qunit'; +const { module, test } = QUnit; +import type { Test, SuperTest } from 'supertest'; +import { basename } from 'path'; +import { + rri, + SupportedMimeType, + setSearchBoundsForTests, + resetSearchBoundsForTests, +} from '@cardstack/runtime-common'; +import type { + DBAdapter, + LooseSingleCardDocument, + Realm, +} from '@cardstack/runtime-common'; +import { + setupPermissionedRealmCached, + testRealmURLFor, +} from './helpers/index.ts'; + +// How much of a card's link graph one response may assemble. The walk that +// builds `included[]` terminates on its own once it has visited everything +// reachable, so what bounds its cost is not how far it travels but how much it +// brings back — a card with dozens of relationships is dozens of resources one +// hop out, and hundreds two hops out, at any depth limit. +// +// The fixture is a fan: one consumer linking to TARGET_COUNT targets, each of +// which links on to its own child. So the full closure is 2 × TARGET_COUNT and +// one hop is TARGET_COUNT, which lets a budget set between them clip mid-walk +// rather than at a layer boundary. + +const realmURL = testRealmURLFor('test/'); +const TARGET_COUNT = 8; +const FULL_CLOSURE = TARGET_COUNT * 2; + +let testDbAdapter: DBAdapter; + +function buildFileSystem(): Record { + let fs: Record = {}; + + fs['target.gts'] = ` + import { contains, field, linksTo, CardDef } from "@cardstack/base/card-api"; + import StringField from "@cardstack/base/string"; + + export class Target extends CardDef { + @field name = contains(StringField); + @field tag = contains(StringField); + @field child = linksTo(() => Target); + } + `; + + // A card whose members are found by running a query rather than by following + // a stored link. Its targets are written into `included[]` by the same walk, + // so the budget reaches them too — and running out part-way through one field + // is the case where a partially-carried field has to stay coherent. + fs['query-consumer.gts'] = ` + import { contains, field, linksToMany, CardDef } from "@cardstack/base/card-api"; + import StringField from "@cardstack/base/string"; + import { Target } from "./target"; + + export class QueryConsumer extends CardDef { + @field name = contains(StringField); + @field matches = linksToMany(() => Target, { + query: { + filter: { eq: { tag: 'query-match' } }, + page: { size: 50, number: 0 }, + }, + }); + } + `; + + let consumerFields = Array.from( + { length: TARGET_COUNT }, + (_, i) => ` @field link${i} = linksTo(() => Target);`, + ).join('\n'); + fs['consumer.gts'] = ` + import { contains, field, linksTo, CardDef } from "@cardstack/base/card-api"; + import StringField from "@cardstack/base/string"; + import { Target } from "./target"; + + export class Consumer extends CardDef { + @field name = contains(StringField); +${consumerFields} + } + `; + + for (let i = 0; i < TARGET_COUNT; i++) { + fs[`child-${i}.json`] = { + data: { + attributes: { name: `Child ${i}` }, + meta: { adoptsFrom: { module: rri('./target'), name: 'Target' } }, + }, + } as LooseSingleCardDocument; + fs[`target-${i}.json`] = { + data: { + attributes: { name: `Target ${i}`, tag: 'query-match' }, + relationships: { child: { links: { self: `./child-${i}` } } }, + meta: { adoptsFrom: { module: rri('./target'), name: 'Target' } }, + }, + } as LooseSingleCardDocument; + } + + let relationships: Record = {}; + for (let i = 0; i < TARGET_COUNT; i++) { + relationships[`link${i}`] = { links: { self: `./target-${i}` } }; + } + fs['probe.gts'] = ` + import { contains, field, linksTo, CardDef } from "@cardstack/base/card-api"; + import StringField from "@cardstack/base/string"; + import { Target } from "./target"; + + export class ProbeConsumer extends CardDef { + @field name = contains(StringField); +${consumerFields} + } + `; + + fs['consumer-1.json'] = { + data: { + attributes: { name: 'C1' }, + relationships, + meta: { adoptsFrom: { module: rri('./consumer'), name: 'Consumer' } }, + }, + } as LooseSingleCardDocument; + + // A second consumer over the same targets. Its closure overlaps the first's + // entirely, which is what makes it the fixture for "a card reached twice is + // paid for once". + fs['consumer-2.json'] = { + data: { + attributes: { name: 'C2' }, + relationships, + meta: { adoptsFrom: { module: rri('./consumer'), name: 'Consumer' } }, + }, + } as LooseSingleCardDocument; + + // A second fan root over the same targets, read by the SQL probe alone. The + // realm's card+json response cache is keyed on the validator, and the budget + // is part of that validator — so a probe sharing a root with another test + // would be answered from that test's cache entry and observe no SQL at all. + // + // Its own type, not another `Consumer`: the search tests below count the rows + // a `Consumer` query returns, and a third instance would move those counts + // while saying nothing about the bound. + fs['probe-consumer.json'] = { + data: { + attributes: { name: 'PROBE' }, + relationships, + meta: { + adoptsFrom: { module: rri('./probe'), name: 'ProbeConsumer' }, + }, + }, + } as LooseSingleCardDocument; + + fs['query-consumer-1.json'] = { + data: { + attributes: { name: 'QC1' }, + meta: { + adoptsFrom: { module: rri('./query-consumer'), name: 'QueryConsumer' }, + }, + }, + } as LooseSingleCardDocument; + + return fs; +} + +// A card+html body is not served as JSON, so supertest leaves it unparsed. +function entryBody(response: { text: string }): { + included?: { id?: string }[]; + meta?: { linkClosureTruncated?: boolean }; +} { + return JSON.parse(response.text); +} + +// The side-loaded link resources, which on this fixture are every included +// resource whose id names a target or a child. +function linkResourceIds(included: { id?: string }[] | undefined): string[] { + return (included ?? []) + .map((r) => r.id ?? '') + .filter((id) => /\/(target|child)-\d+$/.test(id)); +} + +module(basename(import.meta.filename), function () { + module('the assembled-resource budget', function (hooks) { + let request: SuperTest; + let realmHref: string; + let searchPath: string; + let realm: Realm; + + function onRealmSetup(args: { + testRealm: Realm; + request: SuperTest; + dbAdapter: DBAdapter; + }) { + request = args.request; + realm = args.testRealm; + testDbAdapter = args.dbAdapter; + realmHref = new URL(args.testRealm.url).href; + searchPath = `${new URL(args.testRealm.url).pathname.replace(/\/$/, '')}/_search`; + } + + setupPermissionedRealmCached(hooks, { + mode: 'before', + realmURL, + permissions: { '*': ['read'] }, + fileSystem: buildFileSystem(), + onRealmSetup, + }); + + hooks.afterEach(function () { + resetSearchBoundsForTests(); + }); + + function cardPath(name: string) { + return `${new URL(realmHref).pathname}${name}`; + } + + test('a closure that fits arrives whole and the document claims nothing', async function (assert) { + setSearchBoundsForTests({ maxAssembledLinkResources: FULL_CLOSURE }); + let response = await request + .get(cardPath('consumer-1')) + .set('Accept', SupportedMimeType.CardJson); + + assert.strictEqual(response.status, 200, `HTTP 200: ${response.text}`); + assert.strictEqual( + linkResourceIds(response.body.included).length, + FULL_CLOSURE, + 'both hops of the closure are carried', + ); + assert.notOk( + response.body.meta?.linkClosureTruncated, + 'a whole closure is not reported truncated', + ); + }); + + test('a closure past the budget is clipped to it, and the document says so', async function (assert) { + let budget = TARGET_COUNT + 3; + setSearchBoundsForTests({ maxAssembledLinkResources: budget }); + let response = await request + .get(cardPath('consumer-1')) + .set('Accept', SupportedMimeType.CardJson); + + assert.strictEqual(response.status, 200, `HTTP 200: ${response.text}`); + let ids = linkResourceIds(response.body.included); + assert.strictEqual( + ids.length, + budget, + `included carries exactly the ${budget} resources the budget allowed`, + ); + assert.true( + response.body.meta?.linkClosureTruncated, + 'the document reports the closure it carries is partial', + ); + // The clip lands mid-walk: the first hop fits and the second does not, so + // a caller receives every target and only some of their children. The + // budget is spent per resource, so where it runs out is where it stops — + // it does not round to a whole layer in either direction. + assert.strictEqual( + ids.filter((id) => id.includes('/target-')).length, + TARGET_COUNT, + 'the first hop is complete', + ); + assert.strictEqual( + ids.filter((id) => id.includes('/child-')).length, + 3, + 'the second hop carries only what was left of the budget', + ); + }); + + test('a clipped link still names its target, so a consumer can fetch it', async function (assert) { + setSearchBoundsForTests({ maxAssembledLinkResources: 2 }); + let response = await request + .get(cardPath('target-0')) + .set('Accept', SupportedMimeType.CardJson); + assert.strictEqual(response.status, 200, `HTTP 200: ${response.text}`); + assert.strictEqual( + linkResourceIds(response.body.included).length, + 1, + 'the one link this card has fits', + ); + + // Now with no room at all: the relationship must survive as a named but + // uncarried target — the shape a consumer reads as "not loaded yet" — + // rather than being dropped, which would silently lose the edge. + setSearchBoundsForTests({ maxAssembledLinkResources: 1 }); + let clipped = await request + .get(cardPath('consumer-1')) + .set('Accept', SupportedMimeType.CardJson); + assert.strictEqual(clipped.status, 200, `HTTP 200: ${clipped.text}`); + assert.strictEqual( + linkResourceIds(clipped.body.included).length, + 1, + 'one resource fits', + ); + let relationships = clipped.body.data.relationships as Record< + string, + { links?: { self?: string } } + >; + for (let i = 0; i < TARGET_COUNT; i++) { + assert.ok( + relationships[`link${i}`]?.links?.self, + `link${i} still names its target`, + ); + } + }); + + test('the budget bounds what is read, not only what is returned', async function (assert) { + // The assertion the response body cannot make. A walk that read the whole + // closure and then returned the first few resources would produce a + // byte-identical body to one that never read the rest, so only the emitted + // SQL says which happened — and reading less is the point of a bound whose + // justification is event-loop cost. + // + // Counted as distinct resources rather than as bind slots: the batched + // lookup matches on either column (`i.url IN (…) OR i.file_alias IN (…)`), + // so every resource it asks for contributes more than one bind. + // + // Reads `probe-consumer`, and at budgets no other test uses, because the + // response cache is keyed on a validator the budget is folded into: a + // (root, budget) pair some earlier test already read would be served from + // its entry, and the probe would see no SQL whether or not the bound + // works. + async function resourcesRead(budget: number): Promise { + setSearchBoundsForTests({ maxAssembledLinkResources: budget }); + let originalExecute = testDbAdapter.execute.bind(testDbAdapter); + let dbExecute = testDbAdapter as { + execute: typeof testDbAdapter.execute; + }; + let asked = new Set(); + try { + dbExecute.execute = async (sql, opts) => { + let normalized = sql.replace(/\s+/g, ' '); + if ( + /FROM boxel_index\b/.test(normalized) && + /\bi\.url\s+IN\s*\(/.test(normalized) + ) { + for (let bound of opts?.bind ?? []) { + if ( + typeof bound === 'string' && + /\/(target|child)-\d+/.test(bound) + ) { + asked.add(bound.replace(/\.json$/, '')); + } + } + } + return originalExecute(sql, opts); + }; + let response = await request + .get(cardPath('probe-consumer')) + .set('Accept', SupportedMimeType.CardJson); + assert.strictEqual( + response.status, + 200, + `HTTP 200: ${response.text}`, + ); + } finally { + dbExecute.execute = originalExecute; + } + return asked.size; + } + + // The positive control. Without it the bounded arm's low count would be + // equally consistent with a counter that never matched a query at all. + assert.strictEqual( + await resourcesRead(FULL_CLOSURE + 1), + FULL_CLOSURE, + 'a budget that fits reads every resource the graph holds', + ); + assert.strictEqual( + await resourcesRead(6), + 6, + `and a budget of 6 reads six, rather than reading all ${FULL_CLOSURE} and returning six`, + ); + }); + + test('a card reached by two paths spends one slot, not two', async function (assert) { + // Both consumers link to all TARGET_COUNT targets. Charging per edge + // rather than per resource would spend 2 × TARGET_COUNT on the first hop + // and clip a search that comfortably fits. + setSearchBoundsForTests({ maxAssembledLinkResources: FULL_CLOSURE }); + let response = await request + .post(searchPath) + .set('Accept', SupportedMimeType.CardJson) + .set('X-HTTP-Method-Override', 'QUERY') + .send({ + filter: { + 'item.on': { module: `${realmHref}consumer`, name: 'Consumer' }, + }, + fields: { entry: ['item'] }, + }); + + assert.strictEqual(response.status, 200, `HTTP 200: ${response.text}`); + let ids = linkResourceIds(response.body.included); + assert.strictEqual( + ids.length, + FULL_CLOSURE, + 'two consumers over one shared closure assemble it once', + ); + assert.strictEqual( + new Set(ids).size, + FULL_CLOSURE, + 'and carry no resource twice', + ); + assert.notOk( + response.body.meta?.linkClosureTruncated, + 'so the search is not reported truncated', + ); + }); + + test('a search reports a clipped closure on the document, not on a row', async function (assert) { + // One assembly serves the whole page, so no single row is the one that + // ran out — the report belongs where the page is described. + setSearchBoundsForTests({ maxAssembledLinkResources: 4 }); + let response = await request + .post(searchPath) + .set('Accept', SupportedMimeType.CardJson) + .set('X-HTTP-Method-Override', 'QUERY') + .send({ + filter: { + 'item.on': { module: `${realmHref}consumer`, name: 'Consumer' }, + }, + fields: { entry: ['item'] }, + }); + + assert.strictEqual(response.status, 200, `HTTP 200: ${response.text}`); + assert.strictEqual( + linkResourceIds(response.body.included).length, + 4, + 'the page assembles the budget and no more', + ); + assert.true( + response.body.meta.linkClosureTruncated, + 'the document reports the page carries a partial closure', + ); + assert.strictEqual( + response.body.data.length, + 2, + 'both matched rows are still returned — the bound clips links, not results', + ); + }); + + test('a card+html item leg is bounded and reports it', async function (assert) { + setSearchBoundsForTests({ maxAssembledLinkResources: 3 }); + let response = await request + .get(`${cardPath('consumer-1')}?fields=item`) + .set('Accept', SupportedMimeType.CardHtml); + + assert.strictEqual(response.status, 200, `HTTP 200: ${response.text}`); + let body = entryBody(response); + assert.strictEqual( + linkResourceIds(body.included).length, + 3, + 'the item leg assembles the budget and no more', + ); + assert.true( + body.meta?.linkClosureTruncated, + 'and the document reports it', + ); + }); + + test('a render reads its whole closure — the budget is for live traffic', async function (assert) { + // What a prerender assembles is rendered into HTML that is cached and + // served long after this request, and the cached copy carries no way to + // say it was short. So the exemption is not an optimization: a clipped + // render would be a wrong answer with no way to notice. + setSearchBoundsForTests({ maxAssembledLinkResources: 2 }); + + let live = await request + .get(cardPath('consumer-1')) + .set('Accept', SupportedMimeType.CardJson); + assert.strictEqual( + linkResourceIds(live.body.included).length, + 2, + 'the live read is bounded', + ); + + let duringPrerender = await request + .get(cardPath('consumer-1')) + .set('Accept', SupportedMimeType.CardJson) + .set('x-boxel-during-prerender', '1'); + assert.strictEqual( + duringPrerender.status, + 200, + `HTTP 200: ${duringPrerender.text}`, + ); + assert.strictEqual( + linkResourceIds(duringPrerender.body.included).length, + FULL_CLOSURE, + 'the render carries the whole closure at the same budget', + ); + assert.notOk( + duringPrerender.body.meta?.linkClosureTruncated, + 'and reports nothing, having been clipped by nothing', + ); + }); + + test('retuning the budget rotates the validator', async function (assert) { + // Changing the ceiling changes which cards come back clipped and what a + // clipped one contains, while `indexed_at` and the realm-info hash stand + // still. Without the budget in the validator, every client holding one + // would be 304'd to the shape it cached across the change. + setSearchBoundsForTests({ maxAssembledLinkResources: FULL_CLOSURE }); + let whole = await request + .get(cardPath('consumer-1')) + .set('Accept', SupportedMimeType.CardJson); + let wholeEtag = whole.headers['etag']; + assert.ok(wholeEtag, 'the card+json read emits a validator'); + + setSearchBoundsForTests({ maxAssembledLinkResources: 2 }); + let clipped = await request + .get(cardPath('consumer-1')) + .set('Accept', SupportedMimeType.CardJson); + assert.notStrictEqual( + clipped.headers['etag'], + wholeEtag, + 'a different budget is a different validator', + ); + + // And the validator minted under the first budget does not match, so a + // client holding it is sent the shape the second budget produces rather + // than being told the copy it has is fresh. + let conditional = await request + .get(cardPath('consumer-1')) + .set('Accept', SupportedMimeType.CardJson) + .set('If-None-Match', wholeEtag); + assert.strictEqual( + conditional.status, + 200, + 'the stale validator is not honoured', + ); + assert.strictEqual( + linkResourceIds(conditional.body.included).length, + 2, + 'and the client receives the shape the new budget produces', + ); + }); + + test('a query-backed field clipped part-way through stays coherent', async function (assert) { + // The members of a query-backed field are written as `matches.N` + // relationships by the same pass that assembles the closure, so the budget + // can run out in the middle of one. What must survive is the answer a + // consumer cannot cheaply recompute: which cards the field names. The + // cards themselves it can fetch. + setSearchBoundsForTests({ maxAssembledLinkResources: 3 }); + let response = await request + .get(cardPath('query-consumer-1')) + .set('Accept', SupportedMimeType.CardJson); + + assert.strictEqual(response.status, 200, `HTTP 200: ${response.text}`); + let relationships = response.body.data.relationships as Record< + string, + { + links?: { self?: string; search?: string }; + data?: { id: string }[] | { id: string }; + } + >; + let umbrella = relationships.matches; + assert.strictEqual( + Array.isArray(umbrella?.data) ? umbrella.data.length : -1, + TARGET_COUNT, + 'the field still names every match, including the ones not carried', + ); + assert.ok(umbrella?.links?.search, 'and still carries its query link'); + + let carried = linkResourceIds(response.body.included); + assert.strictEqual(carried.length, 3, 'only the budget is carried'); + assert.true( + response.body.meta?.linkClosureTruncated, + 'and the document reports the rest is missing', + ); + + // Every member the response did not carry is still reachable: its own + // `matches.N` entry names it, which is what sends a consumer to fetch it + // rather than reading the field as short. + // + // A query-backed member is answered in the pass that assembles the + // closure, so it goes out carrying `data` as well as `links.self` — + // including the ones clipped here, whose `data` therefore names a + // resource `included[]` does not hold. That is the shape both `LinksTo` + // and `LinksToMany` deserialize to a not-loaded value, so it is pinned + // here rather than left to read either way. + let named = new Set(); + let carryingData = 0; + for (let [key, rel] of Object.entries(relationships)) { + if (!key.startsWith('matches.')) { + continue; + } + if (rel.links?.self) { + named.add(rel.links.self.replace(/^\.\//, '')); + } + if (rel.data && !Array.isArray(rel.data)) { + carryingData++; + } + } + assert.strictEqual( + named.size, + TARGET_COUNT, + 'every member names its target, carried or not', + ); + assert.strictEqual( + carryingData, + TARGET_COUNT, + 'and every member carries the identity the query answered, clipped or not', + ); + assert.strictEqual( + new Set(carried).size, + carried.length, + 'included[] carries no resource twice', + ); + assert.true( + carried.length < TARGET_COUNT, + 'and holds only some of the members — so some data names an absent resource', + ); + }); + + test('a bounded read and an exempt one never share a validator', async function (assert) { + // The two shapes are different bodies at the same index generation, so a + // shared validator would let a conditional request be answered with the + // other one's. That direction matters most: an exempt read is a render's, + // and a truncated closure reused by one is baked into cached HTML, which + // is the outcome the exemption exists to prevent. + setSearchBoundsForTests({ maxAssembledLinkResources: 2 }); + + let live = await request + .get(cardPath('consumer-1')) + .set('Accept', SupportedMimeType.CardJson); + let exempt = await request + .get(cardPath('consumer-1')) + .set('Accept', SupportedMimeType.CardJson) + .set('x-boxel-during-prerender', '1'); + + assert.ok(live.headers['etag'], 'the bounded read emits a validator'); + assert.ok(exempt.headers['etag'], 'the exempt read emits one too'); + assert.notStrictEqual( + live.headers['etag'], + exempt.headers['etag'], + 'and they differ, because the bodies do', + ); + assert.strictEqual( + linkResourceIds(live.body.included).length, + 2, + 'the bounded body is clipped', + ); + assert.strictEqual( + linkResourceIds(exempt.body.included).length, + FULL_CLOSURE, + 'and the exempt body is whole', + ); + + // The exempt shape is answered with its own body rather than 304'd to the + // clipped one it would otherwise have matched. + let conditional = await request + .get(cardPath('consumer-1')) + .set('Accept', SupportedMimeType.CardJson) + .set('x-boxel-during-prerender', '1') + .set('If-None-Match', live.headers['etag']); + assert.strictEqual( + conditional.status, + 200, + 'the bounded validator does not satisfy an exempt request', + ); + assert.strictEqual( + linkResourceIds(conditional.body.included).length, + FULL_CLOSURE, + 'which receives the whole closure', + ); + }); + + test('retuning the budget leaves an exempt validator alone', async function (assert) { + // The ceiling does not bind an exempt assembly, so its body cannot vary + // with the ceiling — and a validator that moved anyway would make every + // render revalidate for nothing on an operator's retune. + setSearchBoundsForTests({ maxAssembledLinkResources: 2 }); + let before = await request + .get(cardPath('consumer-2')) + .set('Accept', SupportedMimeType.CardJson) + .set('x-boxel-during-prerender', '1'); + + setSearchBoundsForTests({ maxAssembledLinkResources: 7 }); + let after = await request + .get(cardPath('consumer-2')) + .set('Accept', SupportedMimeType.CardJson) + .set('x-boxel-during-prerender', '1'); + + assert.ok(before.headers['etag'], 'the exempt read emits a validator'); + assert.strictEqual( + after.headers['etag'], + before.headers['etag'], + 'and it is unchanged across a retune', + ); + }); + + test('the realm serves the bound without a caller opting in', async function (assert) { + // The engine holds every assembly to the budget and takes an exemption + // rather than an opt-in, so a route added later is bounded by default. + // Asserted through the engine directly, since a future route would not + // be reachable through the ones above. + setSearchBoundsForTests({ maxAssembledLinkResources: 5 }); + let result = await realm.realmIndexQueryEngine.cardDocument( + new URL(`${realmHref}consumer-1`), + { loadLinks: true }, + ); + let doc = result?.type === 'doc' ? result.doc : undefined; + assert.ok(doc, 'the card assembled'); + assert.strictEqual( + linkResourceIds(doc?.included).length, + 5, + 'an opts object that says nothing about the budget is still bounded', + ); + assert.true( + doc?.meta?.linkClosureTruncated, + 'and the assembly reports the clip', + ); + }); + }); +}); diff --git a/packages/runtime-common/card-operations/dispatch.ts b/packages/runtime-common/card-operations/dispatch.ts index 1529ab483a5..b3a09432636 100644 --- a/packages/runtime-common/card-operations/dispatch.ts +++ b/packages/runtime-common/card-operations/dispatch.ts @@ -166,6 +166,7 @@ export interface OperationIndexQueryEngine { loadLinks?: boolean; skipQueryBackedExpansion?: boolean; resolveLinksOnly?: boolean; + skipLinkAssemblyBudget?: boolean; }, ): Promise; instance( @@ -192,6 +193,13 @@ export interface RunOperationOptions { // caller emits has to fold it in, since it distinguishes two documents // assembled from the same index row. resolveLinksOnly?: boolean; + // Exempt this read's link assembly from the assembled-resource budget. Set + // for a read serving a prerender request, whose closure is rendered into HTML + // that outlives the request: a clipped one would be cached, and the cached + // copy carries no way to say it was clipped. Derived from the same signal as + // `skipQueryBackedExpansion`, which is already part of the response cache's + // key — so the two shapes never share a cache entry. + skipLinkAssemblyBudget?: boolean; // Report a stored-bytes read's `version` only where the realm already // recorded one, rather than reading the file to fingerprint it. // diff --git a/packages/runtime-common/card-operations/read.ts b/packages/runtime-common/card-operations/read.ts index 363d54494c8..d022a2a319f 100644 --- a/packages/runtime-common/card-operations/read.ts +++ b/packages/runtime-common/card-operations/read.ts @@ -132,6 +132,7 @@ async function readDocument( loadLinks: true, skipQueryBackedExpansion: opts.skipQueryBackedExpansion ?? false, resolveLinksOnly: opts.resolveLinksOnly ?? false, + skipLinkAssemblyBudget: opts.skipLinkAssemblyBudget ?? false, }); if (result === undefined) { // A path with no instance row may still hold bytes: a file asked for as diff --git a/packages/runtime-common/constants.ts b/packages/runtime-common/constants.ts index 96c34d465f6..b9bacac372f 100644 --- a/packages/runtime-common/constants.ts +++ b/packages/runtime-common/constants.ts @@ -154,8 +154,6 @@ export const EXTRA_TOKENS_PRICING: Record = { 80000: 100, // in USD }; -export const maxLinkDepth = 5; - export const DEFAULT_PERMISSIONS = Object.freeze([ 'read', 'write', diff --git a/packages/runtime-common/document-types.ts b/packages/runtime-common/document-types.ts index 04a95a9e74b..27c3623fef3 100644 --- a/packages/runtime-common/document-types.ts +++ b/packages/runtime-common/document-types.ts @@ -15,9 +15,27 @@ import { isEntryResource, } from './resource-types.ts'; +// What a response says about its own `included[]` when the assembled-resource +// budget stopped the link walk short. It is the document's answer to a question +// `included[]` cannot answer for itself: a short one is what a small graph and a +// clipped large one both look like. The ceiling that applied is deliberately +// absent — it is an operator's number, recorded in the realm-server log, while +// the fact is what a consumer could act on. `meta.incomplete` does the same job +// for a result set whose row count came up short. +// +// No consumer branches on this yet. A clipped link is already self-describing +// to the client, which deserializes it to a not-loaded value and fetches the +// card when something reads the field, so nothing has to consult the document +// to behave correctly. It is here so that a caller assembling a total, or a +// person reading a response, can tell a short `included[]` from a small graph. +export interface DocumentClosureMeta { + linkClosureTruncated?: boolean; +} + export interface SingleCardDocument { data: CardResource; included?: (FileMetaResource | CardResource)[]; + meta?: DocumentClosureMeta; } export interface CardCollectionDocument { data: CardResource[]; @@ -40,12 +58,13 @@ export type EntryIncludedResource = export interface EntryCollectionDocument { data: EntryResource[]; included?: EntryIncludedResource[]; - meta: QueryResultsMeta & { - // The applied (bound or defaulted) htmlQuery, echoed once at the document - // level — it cannot vary across entries, so it is never repeated per - // entry. Present whenever the fieldset puts the html branch in play. - htmlQuery?: HtmlQuery; - }; + meta: QueryResultsMeta & + DocumentClosureMeta & { + // The applied (bound or defaulted) htmlQuery, echoed once at the document + // level — it cannot vary across entries, so it is never repeated per + // entry. Present whenever the fieldset puts the html branch in play. + htmlQuery?: HtmlQuery; + }; } // The single-instance entry response (the card+html / file-meta+html GET): one @@ -56,6 +75,7 @@ export interface EntryCollectionDocument { export interface EntrySingleDocument { data: EntryResource; included?: EntryIncludedResource[]; + meta?: DocumentClosureMeta; } // The public-API name for the raw entry wire format a programmatic diff --git a/packages/runtime-common/realm-index-query-engine.ts b/packages/runtime-common/realm-index-query-engine.ts index 1500fcc2e35..0d657ddc3d2 100644 --- a/packages/runtime-common/realm-index-query-engine.ts +++ b/packages/runtime-common/realm-index-query-engine.ts @@ -6,7 +6,6 @@ import { baseRealmRRI, inferContentType, unixTime, - maxLinkDepth, maybeURL, IndexQueryEngine, MATCH_RELEVANCE_SORT_KEY, @@ -97,7 +96,10 @@ import { buildQuerySearchURL, getValueForResourcePath, } from './query-field-utils.ts'; -import { applyServerSearchPageBound } from './search-bounds.ts'; +import { + applyServerSearchPageBound, + assembledLinkResourceBudget, +} from './search-bounds.ts'; import { screenshotsMetaFromManifest, type ScreenshotManifest, @@ -178,6 +180,20 @@ type Options = { // promptly instead of running every layer to completion. Threaded like // `timings`; absent for everything except a bounded live search. signal?: AbortSignal; + // Exempts this assembly from the assembled-resource budget + // (SERVER_MAX_ASSEMBLED_LINK_RESOURCES). Set by the realm-server on its own + // during-prerender traffic, which is deliberately unbounded: a render reads + // the closure it is given, so a truncated one would be baked into prerendered + // HTML and served from the cache long after the request that produced it. + // Every other caller is bounded, including one added later that does not know + // this option exists — which is the point of expressing the exemption rather + // than the bound. + skipLinkAssemblyBudget?: boolean; + // Fires when the assembled-resource budget stopped the walk short of a + // closure. Reported from inside the walk because the caller cannot infer it: + // a short `included[]` is what a small graph and a truncated large one both + // look like. + onLinkClosureTruncated?: () => void; // Fires once per query-backed field this pass applies, whatever the // outcome. The caller cannot read this off the assembled document: // `applyQueryResults` writes the `links.search` marker only when the query @@ -689,6 +705,9 @@ export class RealmIndexQueryEngine { if (collection.included && collection.included.length > 0) { doc.included = collection.included; } + if (collection.meta.linkClosureTruncated) { + doc.meta = { linkClosureTruncated: true }; + } return doc; } @@ -720,15 +739,28 @@ export class RealmIndexQueryEngine { if (fullItemRoots.length > 0 && opts?.loadLinks && !opts?.omitIncluded) { let omit = itemResources.map((r) => r.id).filter(Boolean) as string[]; + // One assembly serves the whole page, so the budget is spent across the + // page's rows jointly and the report belongs on the document rather than + // on any one row: no row is individually the one that ran out. + let truncated = false; let runLoadLinks = () => this.loadLinks( { realmURL: this.realmURL, rootResources: fullItemRoots, omit }, - opts, + { + ...opts, + onLinkClosureTruncated: () => { + truncated = true; + opts?.onLinkClosureTruncated?.(); + }, + }, ); let linked = opts?.timings ? await opts.timings.time('loadLinks', runLoadLinks) : await runLoadLinks(); included.push(...linked); + if (truncated) { + doc.meta.linkClosureTruncated = true; + } } if (included.length > 0) { @@ -822,6 +854,10 @@ export class RealmIndexQueryEngine { } let queryBacked = false; if (opts?.loadLinks) { + // Stamped here rather than by each caller, so every route that answers + // with an assembled card document reports a clipped closure without + // having to know the budget exists. + let truncated = false; let included = await this.loadLinks( { realmURL: this.realmURL, @@ -834,11 +870,18 @@ export class RealmIndexQueryEngine { queryBacked = true; opts.onQueryFieldApplied?.(); }, + onLinkClosureTruncated: () => { + truncated = true; + opts.onLinkClosureTruncated?.(); + }, }, ); if (included.length > 0) { doc.included = included; } + if (truncated) { + doc.meta = { ...doc.meta, linkClosureTruncated: true }; + } } relativizeDocument(doc, this.realmURL, this.#realm.virtualNetwork); await this.attachRealmInfo(doc); @@ -1649,14 +1692,16 @@ export class RealmIndexQueryEngine { return new Map(entries); } - // TODO The caller should provide a list of fields to be included via JSONAPI - // request. currently we just use the maxLinkDepth to control how deep to load - // links. - // // Level-order BFS: each layer issues at most one batched DB query for // in-realm cards and one for in-realm file-meta resources, alongside // Promise.all-fanout cross-realm fetches, all running concurrently // regardless of how many siblings reference links at that depth. + // + // The walk is bounded by how many resources it assembles rather than by how + // far it travels. It terminates on its own once every reachable resource has + // been visited, so what needs bounding is the width it brings back. The budget + // is spent at classification time, before a target's URL joins the layer's + // batched read, so an over-budget graph is neither fetched nor assembled. private async loadLinks( { realmURL, @@ -1703,9 +1748,39 @@ export class RealmIndexQueryEngine { } } + // The assembled-resource budget, and the bookkeeping that spends it. + // `committed` counts the resources this pass has undertaken to side-load; + // `decided` remembers every target it has already ruled on, so a card + // reached down three separate paths costs one slot rather than three. + // + // Seeded with the roots, the caller's `omit` list and anything already in + // `included`, none of which this pass side-loads: a relationship pointing + // back at one of them still has its `data` rewritten below, and charging it + // would spend the budget on resources the response was always going to + // carry anyway. Keyed on the resolved link URL, which is the one spelling + // step 2 always computes; a seed recorded only under some other equivalent + // form costs one slot it needn't have, which is the harmless direction. + let budget = opts?.skipLinkAssemblyBudget + ? Infinity + : assembledLinkResourceBudget(); + let committed = 0; + let truncated = false; + let decided = new Set(omitSet); + for (let existing of includedIds) { + for (let form of allIdForms(existing)) { + decided.add(form); + } + } + for (let resource of rootResources) { + if (resource.id != null) { + for (let form of allIdForms(resource.id)) { + decided.add(form); + } + } + } + type LayerItem = { resource: LooseCardResource | FileMetaResource; - stack: string[]; applyLinkFields: boolean; // Roots are returned in doc.data; everything else gets cloned and // pushed onto included[] *after* its relationships are rewritten in @@ -1726,7 +1801,6 @@ export class RealmIndexQueryEngine { } layer.push({ resource, - stack: [], applyLinkFields: !!opts?.linkFields, isRoot: true, }); @@ -1957,6 +2031,29 @@ export class RealmIndexQueryEngine { resource.id ? vn.toURL(resource.id) : realmURL, ); + // Spend the budget here — before the target joins this layer's + // batched read — so an over-budget closure costs neither the read nor + // the assembly. A target already ruled on falls through: it has been + // paid for, and its relationship still needs the rewrite below. + if (!decided.has(linkURL.href)) { + if (committed >= budget) { + // Out of budget. The relationship is left exactly as it stands: + // a stored link carries `links.self` and no `data`, while a + // query-backed member carries both, because step 1 has already + // answered it. Either way the target is named and absent from + // `included[]`, which both `LinksTo` and `LinksToMany` + // deserialize to a not-loaded value whose getter fetches the card + // on demand — so the two shapes differ on the wire and not in + // what a consumer does with them. The document says a closure was + // clipped as well, see `linkClosureTruncated`, so a short + // `included[]` is distinguishable from a small graph. + truncated = true; + continue; + } + committed++; + decided.add(linkURL.href); + } + let relationshipType = relationship.data?.type as | typeof CardResourceType | typeof FileMetaResourceType @@ -2152,22 +2249,12 @@ export class RealmIndexQueryEngine { linkResource = crossRealmMap.get(entry.linkURL.href); } - let descendStack = - entry.item.resource.id != null - ? [entry.item.resource.id, ...entry.item.stack] - : entry.item.stack; - - // TODO stop using maxLinkDepth. we should save the JSON-API doc - // in the index based on keeping track of the rendered fields - // and invalidate the index as consumed cards change. - // - // Gate uses the CURRENT item's stack length (ancestors only), - // matching the original recursive `stack.length <= maxLinkDepth` - // check which ran before pushing the current resource onto the - // stack for the recursive call. Using descendStack.length here - // would cut traversal off one level early. + // Every entry that reaches step 4 has already been charged to the + // budget (or found to need no charge) in step 2, so there is nothing + // left to gate on here: a resource this walk paid for is a resource it + // carries. let foundLinks = false; - if (linkResource && entry.item.stack.length <= maxLinkDepth) { + if (linkResource) { let alreadyVisited = linkResource.id != null && visited.has(linkResource.id); if (!alreadyVisited) { @@ -2176,14 +2263,13 @@ export class RealmIndexQueryEngine { } // Schedule expansion at the next layer. linkFields applies // only at the root layer; nested relationships are fully - // loaded up to maxLinkDepth. The clone+push to included[] + // loaded within the budget. The clone+push to included[] // happens at the END of that layer's processing — once the // resource's relationships have been rewritten — so the // clone captures the mutations rather than the pre-rewrite // state from pristine_doc. nextLayer.push({ resource: linkResource, - stack: descendStack, applyLinkFields: false, isRoot: false, }); @@ -2278,8 +2364,18 @@ export class RealmIndexQueryEngine { layer = nextLayer; } + if (truncated) { + // Logged at warn, once per assembly: the budget is sized not to engage on + // healthy content, so a line here is the signal that some graph outgrew + // it — and the only place the ceiling that applied is recorded, since the + // document carries the fact of the truncation rather than the number. + this.#log.warn( + `[loadLinks ${invocationId}] assembled-resource budget of ${budget} reached for realm=${realmURL.href} roots=${rootResources.length}; included[] carries ${included.length} of a larger closure and the response reports it truncated`, + ); + opts?.onLinkClosureTruncated?.(); + } this.#log.debug( - `[loadLinks ${invocationId}] complete layers=${layerIndex} included=${included.length} visited=${visited.size}`, + `[loadLinks ${invocationId}] complete layers=${layerIndex} included=${included.length} visited=${visited.size} committed=${committed}`, ); return included; } diff --git a/packages/runtime-common/realm.ts b/packages/runtime-common/realm.ts index e5248904a98..1d5e034e26a 100644 --- a/packages/runtime-common/realm.ts +++ b/packages/runtime-common/realm.ts @@ -19,6 +19,7 @@ import type { SearchOpts } from './search-utils.ts'; import { buildSearchErrorBody, SearchRequestError } from './search-utils.ts'; import { applyServerSearchPageBound, + assembledLinkResourceBudget, isItemLegSearch, runWithSearchTimeBudget, SearchBoundError, @@ -618,6 +619,19 @@ const SOURCE_ETAG_VARIANT = 'source'; // relationship ids) in canonical prefix (RRI) form for mapped realms. const CARD_JSON_ETAG_VARIANT = 'card-rri'; +// The variant the card+json validator carries, with the assembled-resource +// budget folded in. The budget decides which cards come back with a clipped +// closure and what a clipped one contains, and it is settable per server, so +// changing it changes bodies while `indexed_at`, the realm-info hash and the +// screenshots fingerprint all stand still. That is the case the constant above +// exists for, except that the change arrives by configuration rather than by +// revision — so the value belongs in the validator rather than in the memory of +// whoever edits it. One number for the process, so it fragments no cache: every +// response at a given build and setting shares it. +function cardJsonEtagVariant(): string { + return `${CARD_JSON_ETAG_VARIANT}-lb${assembledLinkResourceBudget()}`; +} + // Postgres NOTIFY channel for cross-instance invalidation of #sourceCache / // #transpiledModuleCache entries on file writes. Two payload shapes: // @@ -895,14 +909,14 @@ function buildEtag( // fields expanded, because nothing reads either off a write response. // // These three take different validators because a client holding one must not -// be 304'd to another's body. They are not the only way a body can vary at -// one `indexed_at`: a read from inside a prerender leaves query-backed fields -// unexpanded while still side-loading static links, and that body shares -// `full`'s validator deliberately — it is never served to a client that -// caches, and the response cache separates it by folding -// `skipQueryBackedExpansion` into its own key rather than into the ETag. -// Adding a member here is the wrong move for a variation the validator does -// not have to carry. +// be 304'd to another's body. The `full` shape splits once more below, on +// whether the assembled-resource budget bounded the closure it carries, since +// a bounded body and an exempt one differ at one `indexed_at`. Query-backed +// expansion gets no member here: a read from inside a prerender leaves those +// fields unexpanded, but that body is never served to a client that caches, +// and the response cache separates it by folding `skipQueryBackedExpansion` +// into its own key rather than into the ETag. Adding a member here is the +// wrong move for a variation the validator does not have to carry. type CardJsonShape = 'full' | 'links-only' | 'write-echo'; function buildCardJsonEtag( @@ -910,6 +924,7 @@ function buildCardJsonEtag( realmInfoHash: string | undefined, screenshotsFingerprint?: string, shape: CardJsonShape = 'full', + unboundedAssembly = false, ): string | undefined { if (indexedAt == null) { return undefined; @@ -924,10 +939,25 @@ function buildCardJsonEtag( // rather than per revision. Without it, a client that cached a narrower // shape would be 304'd to it when it later asks for the full one, and the // shapes would be reachable under one key in the response cache. - let variant = - shape === 'full' - ? CARD_JSON_ETAG_VARIANT - : `${CARD_JSON_ETAG_VARIANT}-${shape}`; + // + // A full read is the only shape that assembles a link closure, so it is the + // only one the assembled-resource budget can move — and it splits again on + // whether that budget applied. An assembly exempt from the budget carries its + // whole closure at any ceiling, so it is named as such rather than by a + // number that does not bind it: naming the exemption leaves an exempt + // response's validator untouched by a retune, which cannot change its body, + // and keeps the two shapes from sharing a validator — which would let a + // conditional request be answered with the other shape's body. That is the + // hazard the exemption exists to prevent, since an exempt read is a render's, + // and a clipped closure reused by one is baked into cached HTML. + let variant: string; + if (shape !== 'full') { + variant = `${CARD_JSON_ETAG_VARIANT}-${shape}`; + } else if (unboundedAssembly) { + variant = `${CARD_JSON_ETAG_VARIANT}-lb-off`; + } else { + variant = cardJsonEtagVariant(); + } return `"${base}:${variant}"`; } @@ -976,6 +1006,7 @@ function buildEntryHtmlEtag( doc: EntrySingleDocument, realmInfoHash: string | undefined, resolveLinksOnly = false, + unboundedAssembly = false, ): string { let indexGeneration = doc.data.meta?.generation ?? 0; let htmlIds = doc.data.relationships.html?.data ?? []; @@ -995,6 +1026,20 @@ function buildEntryHtmlEtag( if (doc.data.relationships.item && resolveLinksOnly) { base = `${base}:links-only`; } + // An item carries a link closure, and the assembled-resource budget decides + // how much of one — so a response bearing an item is a different body at a + // different budget while both generations stand still. This validator has no + // constant component to hang that on the way the card+json one does, so the + // budget is folded in directly. A pure-html response assembles no closure, and + // neither does a links-only item, so neither carries the component. An item + // assembled exempt from the budget carries its whole closure at any ceiling, + // so it is named as such rather than by a number that does not bind it — + // which is also what keeps it from sharing a validator with the bounded shape. + if (doc.data.relationships.item && !resolveLinksOnly) { + base = unboundedAssembly + ? `${base}:lb-off` + : `${base}:lb${assembledLinkResourceBudget()}`; + } return `"${base}"`; } @@ -8590,11 +8635,28 @@ export class Realm { #cardJsonLinkShape(request: Request): { skipQueryBackedExpansion: boolean; resolveLinksOnly: boolean; + skipLinkAssemblyBudget: boolean; } { + let duringPrerender = isDuringPrerenderRequest(request); return { - skipQueryBackedExpansion: isDuringPrerenderRequest(request), + skipQueryBackedExpansion: duringPrerender, resolveLinksOnly: this.#decideLinkShape(request, 'single-row')?.mode === 'links-only', + // The assembled-resource budget bounds live reads. This branch exempts a + // read carrying the during-prerender marker, for the same reason the + // prerendered-HTML leg is exempt from the page and time bounds: what a + // render assembles is baked into cached HTML, so a ceiling that clipped + // it would serve a short closure from cache long after the pressure that + // justified it had passed. + // + // It is defensive rather than load-bearing. No caller attaches that + // marker to a card+json request, so the exemption a render relies on is + // the one on the card+html entry leg, which does receive it and does run + // the assembly pass. The branch stays so it is correct if the marker ever + // arrives here. The validator names the exemption rather than the + // ceiling, so an exempt body and a bounded one never share one — see + // `buildCardJsonEtag`. + skipLinkAssemblyBudget: duringPrerender, }; } @@ -8778,8 +8840,11 @@ export class Realm { let url = this.paths.fileURL(localPath); let start = Date.now(); try { - let { skipQueryBackedExpansion, resolveLinksOnly } = - this.#cardJsonLinkShape(request); + let { + skipQueryBackedExpansion, + resolveLinksOnly, + skipLinkAssemblyBudget, + } = this.#cardJsonLinkShape(request); let result: OperationResult; try { result = await runOperation( @@ -8789,6 +8854,8 @@ export class Realm { name: 'read', ...this.#callerOf(request, requestContext), }, + // No budget flag: a headers-only read assembles no closure, so there + // is nothing for it to bound. { headersOnly: true, skipQueryBackedExpansion, resolveLinksOnly }, ); } catch (e) { @@ -8828,6 +8895,7 @@ export class Realm { this.getCachedRealmInfoHash(), screenshotsEtagFingerprint(result.screenshots), resolveLinksOnly ? 'links-only' : 'full', + skipLinkAssemblyBudget, ); let cacheControl = this.cardJsonCacheControl(requestContext); let lastModified: Record = @@ -8951,8 +9019,11 @@ export class Realm { // has to describe the shape the assembly will produce, and that // validator is what the conditional request and the response cache are // both keyed on. - let { skipQueryBackedExpansion, resolveLinksOnly } = - this.#cardJsonLinkShape(request); + let { + skipQueryBackedExpansion, + resolveLinksOnly, + skipLinkAssemblyBudget, + } = this.#cardJsonLinkShape(request); // The `instance()` peek, which yields this card's validator. It runs // for a client that sent a validator of its own to compare against, @@ -9007,6 +9078,7 @@ export class Realm { realmInfoHash, screenshotsEtagFingerprint(instanceEntry.screenshots), resolveLinksOnly ? 'links-only' : 'full', + skipLinkAssemblyBudget, ); } if ( @@ -9047,6 +9119,7 @@ export class Realm { localPath, skipQueryBackedExpansion, resolveLinksOnly, + skipLinkAssemblyBudget, peekEtag, this.#callerOf(request, requestContext), ); @@ -9120,6 +9193,7 @@ export class Realm { localPath: LocalPath, skipQueryBackedExpansion: boolean, resolveLinksOnly: boolean, + skipLinkAssemblyBudget: boolean, keyEtag: string | undefined, caller: { actor: string; clientRequestId: string }, ): Promise { @@ -9149,7 +9223,7 @@ export class Realm { actor: caller.actor, clientRequestId: caller.clientRequestId, }, - { skipQueryBackedExpansion, resolveLinksOnly }, + { skipQueryBackedExpansion, resolveLinksOnly, skipLinkAssemblyBudget }, ); } catch (e) { if (!isOperationFailure(e)) { @@ -9200,6 +9274,7 @@ export class Realm { this.getCachedRealmInfoHash(), screenshotsEtagFingerprint(headers.screenshots), resolveLinksOnly ? 'links-only' : 'full', + skipLinkAssemblyBudget, ); return { kind: 'document', @@ -9392,7 +9467,9 @@ export class Realm { { htmlQuery, fieldset, kind }, { loadLinks: true, - ...(duringPrerender ? { cacheOnlyDefinitions: true } : {}), + ...(duringPrerender + ? { cacheOnlyDefinitions: true, skipLinkAssemblyBudget: true } + : {}), ...(resolveLinksOnly ? { resolveLinksOnly: true } : {}), }, ); @@ -9407,6 +9484,7 @@ export class Realm { doc, this.getCachedRealmInfoHash(), resolveLinksOnly, + duringPrerender, ); let ifNoneMatch = request.headers.get('if-none-match'); if (ifNoneMatch && ifNoneMatchMatches(ifNoneMatch, etag)) { diff --git a/packages/runtime-common/search-bounds.ts b/packages/runtime-common/search-bounds.ts index 3281c2f2570..d708a411d6d 100644 --- a/packages/runtime-common/search-bounds.ts +++ b/packages/runtime-common/search-bounds.ts @@ -49,6 +49,17 @@ const log = logger('search-bounds'); // decides whether a burst exhausts the heap. Enforced at admission in the // realm-server's request middleware; arrivals above the ceiling wait // briefly for a slot and are then shed with 429 + Retry-After. +// - Assembled link resources (SERVER_MAX_ASSEMBLED_LINK_RESOURCES) — +// server-side only, and the one bound whose polarity is inverted: every +// `loadLinks` assembly is held to it unless a caller opts out, because a +// closure is assembled by more routes than search, and a bound that must be +// remembered per route is a bound a new route forgets. Counted in resources +// rather than in bytes because the resource is what the walk schedules, so +// the count bounds the batched reads as well as the assembly; and the cost +// it stands in for is the event-loop CPU of cloning, rewriting and +// serializing each one, which scales with the count. Counted in resources +// rather than in hops because distance does not track expense: a card +// carrying dozens of relationships is dozens of resources one hop out. // // All bounds are exported consts, overridable via env for ops tuning. // --------------------------------------------------------------------------- @@ -61,11 +72,13 @@ const DEFAULT_SEARCH_TIME_BUDGET_MS = 30_000; const DEFAULT_SEARCH_CONCURRENCY_CAP = 2; const DEFAULT_SERVER_MAX_IN_FLIGHT_SEARCHES = 30; const DEFAULT_SEARCH_ADMISSION_WAIT_MS = 1_000; +const DEFAULT_SERVER_MAX_ASSEMBLED_LINK_RESOURCES = 1_000; const MIN_PAGE_SIZE = 1; const MIN_REALMS = 1; const MIN_TIME_BUDGET_MS = 1_000; const MIN_CONCURRENCY = 1; +const MIN_ASSEMBLED_LINK_RESOURCES = 1; // Clamp an env override to a positive integer, falling back (also clamped) when // the value is missing or non-numeric so a bad env var can't disable a bound. @@ -187,6 +200,39 @@ export const SEARCH_ADMISSION_WAIT_MS = parsePositiveInt( 0, ); +// The most link targets one `loadLinks` assembly may take on, and the whole +// bound on how far a card's transitive link closure is walked. +// +// Counted where a target is classified rather than where it lands, which is +// what lets the ceiling bound the batched reads and not just the assembly. The +// two totals differ by the targets that turn out to have nothing behind them: a +// link whose row is missing or errored, and a cross-realm fetch that fails, +// each spend from the ceiling and add nothing to `included[]`. So a card with +// many broken links can report its closure clipped while carrying fewer +// resources than the ceiling allows — the bound is on work undertaken, which is +// the quantity that costs, and it errs toward doing less of it. +// +// Sized as a safety ceiling rather than as a tuning knob. On realms in use the +// widest card's closure runs to the low hundreds of resources, and a +// hundred-row page of the most connected type unions to about the same, so the +// ceiling sits several times above that. It also lands near the point where one +// assembly would hold the tens of MB of heap SERVER_MAX_IN_FLIGHT_SEARCHES +// assumes per in-flight search, a serialized resource running a little over +// 2 KB. So it is not expected to engage; it exists so that no single card graph +// — authored by a person or by a model, and re-editable at any time — can make +// one request assemble an unbounded document. +// +// Changing this value changes which responses are truncated and what a +// truncated one contains, while none of the other validator inputs move, so it +// is folded into the card+json ETag variant (see `cardJsonEtagVariant` in +// realm.ts). A client holding a validator would otherwise be told the shape it +// cached is still fresh across a retune. +export const SERVER_MAX_ASSEMBLED_LINK_RESOURCES = parsePositiveInt( + env.SERVER_MAX_ASSEMBLED_LINK_RESOURCES, + DEFAULT_SERVER_MAX_ASSEMBLED_LINK_RESOURCES, + MIN_ASSEMBLED_LINK_RESOURCES, +); + // --------------------------------------------------------------------------- // The link-shape policy's tuning (see runtime-common/link-shape-policy.ts). // @@ -306,6 +352,7 @@ let serverMaxPageSize = SERVER_MAX_SEARCH_PAGE_SIZE; let serverAbsoluteMaxPageSize = SERVER_ABSOLUTE_MAX_PAGE_SIZE; let maxRealmsPerRequest = MAX_REALMS_PER_SEARCH_REQUEST; let timeBudgetMs = SEARCH_TIME_BUDGET_MS; +let maxAssembledLinkResources = SERVER_MAX_ASSEMBLED_LINK_RESOURCES; // The (size, max) pairs already reported by `warnOncePerClamp`. In practice a // clamp is driven by an authored page size — a constant in a card definition — @@ -323,6 +370,7 @@ export function setSearchBoundsForTests(overrides: { serverAbsoluteMaxPageSize?: number; maxRealmsPerRequest?: number; timeBudgetMs?: number; + maxAssembledLinkResources?: number; }): void { if (overrides.maxPageSize !== undefined) { maxPageSize = overrides.maxPageSize; @@ -339,6 +387,9 @@ export function setSearchBoundsForTests(overrides: { if (overrides.timeBudgetMs !== undefined) { timeBudgetMs = overrides.timeBudgetMs; } + if (overrides.maxAssembledLinkResources !== undefined) { + maxAssembledLinkResources = overrides.maxAssembledLinkResources; + } } export function resetSearchBoundsForTests(): void { @@ -348,6 +399,15 @@ export function resetSearchBoundsForTests(): void { serverAbsoluteMaxPageSize = SERVER_ABSOLUTE_MAX_PAGE_SIZE; maxRealmsPerRequest = MAX_REALMS_PER_SEARCH_REQUEST; timeBudgetMs = SEARCH_TIME_BUDGET_MS; + maxAssembledLinkResources = SERVER_MAX_ASSEMBLED_LINK_RESOURCES; +} + +// The effective assembled-resource budget. Read through a function rather than +// imported as a const so the test seam above reaches it — a test exercises the +// bound by lowering it to a handful of resources instead of authoring a +// thousand-card graph. +export function assembledLinkResourceBudget(): number { + return maxAssembledLinkResources; } // The item leg (`fields[entry]` includes "item" / "item.") is the live diff --git a/packages/runtime-common/search-entry.ts b/packages/runtime-common/search-entry.ts index ee128346681..5d9fa82469b 100644 --- a/packages/runtime-common/search-entry.ts +++ b/packages/runtime-common/search-entry.ts @@ -1088,6 +1088,14 @@ export function combineSearchEntryResults( if (combined.meta.htmlQuery == null && doc.meta?.htmlQuery != null) { combined.meta.htmlQuery = doc.meta.htmlQuery; } + // Any realm that clipped its closure clips the merged one: the combined + // `included[]` is the union, so a consumer holding it is short by whatever + // that realm withheld. Each realm holds its own ceiling, which is why the + // report is a fact rather than a figure — there is no single number here to + // merge. + if (doc.meta?.linkClosureTruncated) { + combined.meta.linkClosureTruncated = true; + } for (let resource of doc.included ?? []) { if (resource.id) { // NUL-separated so a `(type, id)` pair can't alias another by