From f4a3250d1bff295432891c98274de298929c9c4e Mon Sep 17 00:00:00 2001 From: Hassan Abdel-Rahman Date: Wed, 16 Sep 2026 18:38:03 -0400 Subject: [PATCH 01/13] Carry a batch of named operations over one realm endpoint `POST` and `QUERY {realm}/_operations` read a `boxel:operations` envelope, resolve each entry's name against its target's definition, and answer positionally in `atomic:results`: reads with the document, writes with the identity and version they wrote, deletes with nothing. A read entry answers from the state the batch started from, and the writes commit through the same coordinator the card verbs do, all or nothing. The envelope's parse, its validation and the shapes it answers with live in `card-operations/envelope.ts`, which reads no file and resolves no identifier, so a batch can be read and answered wherever the operation core runs. A file write on an instance whose extension the realm does not register now reaches its executor, which is where a card's stored JSON is told from plain bytes. Co-Authored-By: Claude Opus 5 (1M context) --- .../tests/realm-endpoints/operations-test.ts | 857 ++++++++++++++++++ .../card-operations/bxl-emit.ts | 28 + .../card-operations/dispatch.ts | 60 +- .../card-operations/envelope.ts | 636 +++++++++++++ .../runtime-common/card-operations/index.ts | 17 + .../runtime-common/card-operations/types.ts | 4 + packages/runtime-common/realm.ts | 294 +++++- .../runtime-common/supported-mime-type.ts | 13 + 8 files changed, 1891 insertions(+), 18 deletions(-) create mode 100644 packages/realm-server/tests/realm-endpoints/operations-test.ts create mode 100644 packages/runtime-common/card-operations/envelope.ts diff --git a/packages/realm-server/tests/realm-endpoints/operations-test.ts b/packages/realm-server/tests/realm-endpoints/operations-test.ts new file mode 100644 index 00000000000..ab8825ac10e --- /dev/null +++ b/packages/realm-server/tests/realm-endpoints/operations-test.ts @@ -0,0 +1,857 @@ +import QUnit from 'qunit'; +const { module, test } = QUnit; +import { basename, join } from 'path'; +import fsExtra from 'fs-extra'; +const { existsSync, readFileSync } = fsExtra; +import type { Test, SuperTest } from 'supertest'; +import type { DirResult } from 'tmp'; +import type { PgAdapter } from '@cardstack/postgres'; + +import { rri, SupportedMimeType } from '@cardstack/runtime-common'; +import type { + DBAdapter, + LooseSingleCardDocument, + Realm, +} from '@cardstack/runtime-common'; +import { APP_BOXEL_REALM_EVENT_TYPE } from '@cardstack/runtime-common/matrix-constants'; +import type { + IncrementalIndexEventContent, + RealmEventContent, +} from '@cardstack/base/matrix-event'; +import type { RealmHttpServer as Server } from '../../server.ts'; +import { + createJWT, + setupMatrixRoom, + setupPermissionedRealmCached, + withRealmPath, + type RealmRequest, +} from '../helpers/index.ts'; + +// ============================================================================ +// The `_operations` endpoint. +// +// What is under test is the transport: how a batch is read off the wire, which +// behavior each entry's name resolves to for its target, what comes back +// positionally, and what the realm is left holding. The operation core's own +// behavior — what a transform program does to a document, how a create mints a +// URL — is covered where that behavior lives; the assertions here are about +// what reaching it through this endpoint means, which includes the cases where +// it is never reached at all. +// +// Two realms, because the endpoint's permission is derived from the HTTP +// method and nothing else: one anyone may read and write, where the batches +// run, and one anyone may only read, where a write is refused before a body is +// ever parsed. +// ============================================================================ + +const testRealm = new URL('http://127.0.0.1:4472/test/'); +const testRealmHref = testRealm.href; +const readOnlyRealm = new URL('http://127.0.0.1:4473/test/'); + +const OPERATIONS = SupportedMimeType.BoxelOperations; +const PERSON = { module: rri(`${testRealmHref}person`), name: 'Person' }; +const EXTERNAL_REPORT = { + module: rri(`${testRealmHref}report`), + name: 'ExternalReport', +}; + +function envelope(...operations: unknown[]) { + return JSON.stringify({ 'boxel:operations': operations }); +} + +function invoke( + name: string, + rest: { href?: string; data?: unknown } = {}, +): Record { + return { op: 'invoke', 'boxel:name': name, ...rest }; +} + +function reportFile(): LooseSingleCardDocument { + return { + data: { + type: 'card', + attributes: { + headline: 'Quarterly Review', + status: 'open', + comments: [], + }, + relationships: { owner: { links: { self: './reviewer' } } }, + meta: { adoptsFrom: EXTERNAL_REPORT }, + }, + }; +} + +function makeFileSystem(): Record { + return { + 'person.gts': ` + import { contains, field, linksTo, CardDef, Component } from "@cardstack/base/card-api"; + import StringField from "@cardstack/base/string"; + + export class Person extends CardDef { + @field firstName = contains(StringField); + @field friend = linksTo(() => Person, { searchable: true }); + static isolated = class Isolated extends Component { + + } + static embedded = class Embedded extends Component { + + } + static fitted = class Fitted extends Component { + + } + } + `, + // The declarations reach the endpoint the way an author's do: the module is + // indexed, the prerender host lowers its `@operation`s into the type's + // definition-cache entry, and dispatch reads them back out of it. + 'report.gts': ` + import { contains, containsMany, field, linksTo, CardDef, FieldDef, Component } from "@cardstack/base/card-api"; + import StringField from "@cardstack/base/string"; + import { operation, params, actor, instance } from "@cardstack/base/operations"; + import { Person } from "./person"; + + export class ReportComment extends FieldDef { + @field body = contains(StringField); + @field postedBy = contains(StringField); + } + + export class ExternalReport extends CardDef { + @field headline = contains(StringField); + @field status = contains(StringField); + @field comments = containsMany(ReportComment); + @field owner = linksTo(() => Person, { searchable: true }); + + @operation static escalate = { + base: 'transform', + set: { status: 'escalated' }, + }; + + @operation static addComment = { + base: 'transform', + params: { body: StringField }, + append: { + to: 'comments', + value: { body: params('body'), postedBy: actor() }, + }, + }; + + @operation static openReports = { + base: 'query', + query: { filter: { on: ExternalReport, eq: { status: 'open' } } }, + }; + + static isolated = class Isolated extends Component { + + } + static embedded = class Embedded extends Component { + + } + static fitted = class Fitted extends Component { + + } + } + `, + 'reviewer.json': { + data: { + type: 'card', + attributes: { firstName: 'Reviewer' }, + meta: { adoptsFrom: PERSON }, + }, + }, + // One report per test that changes one, so the tests in this file do not + // have to be ordered against each other. + ...Object.fromEntries( + [ + 'report-named', + 'report-mixed', + 'report-rollback', + 'report-kept', + 'report-anonymous', + 'report-identified', + 'report-deleted', + 'report-relative', + 'report-uncached', + ].map((name) => [`${name}.json`, reportFile()]), + ), + 'notes.md': '# Notes\n', + // A stored file the registered-extension table does not name, so its URL + // classifies as a card and the executor is what tells the two apart. + 'telemetry.log': 'boot\n', + }; +} + +module(basename(import.meta.filename), function (hooks) { + let realm: Realm; + let testDbAdapter: DBAdapter; + let request: RealmRequest; + let serverRequest: SuperTest; + let testRealmHttpServer: Server; + let dir: DirResult; + + setupPermissionedRealmCached(hooks, { + mode: 'before', + realmURL: testRealm, + permissions: { + '*': ['read', 'write'], + '@node-test_realm:localhost': ['read', 'write', 'realm-owner'], + }, + subscribeToRealmEvents: true, + fileSystem: makeFileSystem(), + onRealmSetup(args) { + realm = args.testRealm; + testDbAdapter = args.dbAdapter; + request = withRealmPath(args.request, testRealm); + serverRequest = args.request; + testRealmHttpServer = args.testRealmHttpServer; + dir = args.dir; + }, + }); + + let { getMessagesSince } = setupMatrixRoom(hooks, () => ({ + testRealm: realm, + testRealmHttpServer, + request, + serverRequest, + dir, + dbAdapter: testDbAdapter as PgAdapter, + })); + + function realmFile(localPath: string): string { + return join(dir.name, 'realm_server_1', 'test', localPath); + } + + function storedCard(localPath: string): LooseSingleCardDocument { + return JSON.parse(readFileSync(realmFile(localPath), 'utf8')); + } + + async function indexJobIds(): Promise { + let rows = (await testDbAdapter.execute( + `select id from jobs where job_type = 'incremental-index' + and concurrency_group = $1 order by id`, + { bind: [`indexing:${realm.url}`] }, + )) as { id: number | string }[]; + return rows.map((row) => Number(row.id)); + } + + async function incrementalIndexEventsSince( + since: number, + ): Promise { + let messages = await getMessagesSince(since); + return messages + .filter((message) => message.type === APP_BOXEL_REALM_EVENT_TYPE) + .map((message) => message.content as RealmEventContent) + .filter( + (event): event is IncrementalIndexEventContent => + event.eventName === 'index' && event.indexType === 'incremental', + ); + } + + const TESTER = '@tester:localhost'; + + // The realm lets anyone read and write it, so a batch reaches the endpoint + // with or without credentials — which is the difference the identity module + // is about. Everywhere else the caller is authenticated, since that is the + // ordinary case and it is what gives an operation an actor to read. + function post(body: string) { + return anonymousPost(body).set( + 'Authorization', + `Bearer ${createJWT(realm, TESTER, ['read', 'write'])}`, + ); + } + + function anonymousPost(body: string) { + return request + .post('/_operations') + .set('Accept', OPERATIONS) + .set('Content-Type', OPERATIONS) + .send(body); + } + + // Sent as a `POST` carrying the override header, which is the spelling for + // clients that cannot send a `QUERY` method; the realm reads it back into a + // `QUERY` before it decides which permission the request needs. + function query(body: string) { + return post(body).set('X-HTTP-Method-Override', 'QUERY'); + } + + module('validation', function () { + test('an href outside this realm is refused, naming the entry', async function (assert) { + let response = await post( + envelope( + invoke('escalate', { href: '/report-kept' }), + invoke('escalate', { href: 'http://127.0.0.1:4999/other/report-x' }), + ), + ); + + assert.strictEqual(response.status, 400, 'HTTP 400 status'); + let [error] = response.body.errors; + assert.strictEqual(error.code, 'invalid-params'); + assert.strictEqual( + error.meta.entry, + 1, + 'the error names the entry that carries the foreign href', + ); + assert.true( + error.detail.includes('http://127.0.0.1:4999/other/report-x'), + `detail names the href: ${error.detail}`, + ); + assert.strictEqual( + storedCard('report-kept.json').data.attributes?.status, + 'open', + 'the entry that would have succeeded wrote nothing', + ); + }); + + test('a name the target does not carry is refused', async function (assert) { + let response = await post( + envelope(invoke('unheardOf', { href: '/report-kept' })), + ); + + assert.strictEqual(response.status, 404, 'HTTP 404 status'); + let [error] = response.body.errors; + assert.strictEqual(error.code, 'unknown-operation'); + assert.strictEqual(error.meta.entry, 0); + }); + + test('a body sent without the operations extension is told what it is missing', async function (assert) { + let response = await request + .post('/_operations') + .set('Accept', SupportedMimeType.JSONAPI) + .set('Content-Type', SupportedMimeType.JSONAPI) + .send(envelope(invoke('escalate', { href: '/report-kept' }))); + + assert.strictEqual(response.status, 400, 'HTTP 400 status'); + let [error] = response.body.errors; + assert.strictEqual(error.title, 'Invalid content type'); + assert.true( + error.detail.includes('ext='), + `detail names the extension parameter: ${error.detail}`, + ); + }); + + test('an entry naming a query is sent to the search engine', async function (assert) { + let response = await query( + envelope(invoke('openReports', { href: '/report-kept' })), + ); + + assert.strictEqual(response.status, 400, 'HTTP 400 status'); + let [error] = response.body.errors; + assert.strictEqual(error.code, 'wrong-entry-point'); + assert.strictEqual(error.meta.entry, 0); + assert.true( + error.detail.includes('search engine'), + `detail says where a query runs: ${error.detail}`, + ); + }); + + test('an entry naming readSource is sent to the byte routes', async function (assert) { + let response = await query( + envelope(invoke('readSource', { href: '/notes.md' })), + ); + + assert.strictEqual(response.status, 400, 'HTTP 400 status'); + let [error] = response.body.errors; + assert.strictEqual(error.code, 'wrong-entry-point'); + assert.strictEqual(error.meta.entry, 0); + }); + + test('a QUERY batch carrying a write is refused', async function (assert) { + let response = await query( + envelope( + invoke('read', { href: '/report-kept' }), + invoke('escalate', { href: '/report-kept' }), + ), + ); + + assert.strictEqual(response.status, 400, 'HTTP 400 status'); + let [error] = response.body.errors; + assert.strictEqual(error.code, 'wrong-entry-point'); + assert.strictEqual( + error.meta.entry, + 1, + 'the error names the entry that writes', + ); + assert.strictEqual( + storedCard('report-kept.json').data.attributes?.status, + 'open', + 'nothing was written', + ); + }); + + test('an entry that is not an invocation is refused', async function (assert) { + let response = await post( + envelope({ op: 'parallel', 'boxel:operations': [] }), + ); + + assert.strictEqual(response.status, 400, 'HTTP 400 status'); + let [error] = response.body.errors; + assert.strictEqual(error.meta.entry, 0); + assert.true( + error.detail.includes('invoke'), + `detail names the verb this endpoint carries: ${error.detail}`, + ); + }); + + test('a body that is not an envelope is refused', async function (assert) { + let response = await post(JSON.stringify({ 'atomic:operations': [] })); + + assert.strictEqual(response.status, 400, 'HTTP 400 status'); + let [error] = response.body.errors; + assert.true( + error.detail.includes('boxel:operations'), + `detail names the member it looked for: ${error.detail}`, + ); + }); + + test('two entries claiming one local id are refused', async function (assert) { + let response = await post( + envelope( + invoke('create', { + data: { + lid: 'twin', + type: 'card', + attributes: { firstName: 'Mango' }, + meta: { adoptsFrom: PERSON }, + }, + }), + invoke('create', { + data: { + lid: 'twin', + type: 'card', + attributes: { firstName: 'Van Gogh' }, + meta: { adoptsFrom: PERSON }, + }, + }), + ), + ); + + assert.strictEqual(response.status, 400, 'HTTP 400 status'); + let [error] = response.body.errors; + assert.strictEqual(error.meta.entry, 1); + assert.true( + error.detail.includes('twin'), + `detail names the local id: ${error.detail}`, + ); + }); + + test('a local id no entry creates is refused', async function (assert) { + let response = await post( + envelope( + invoke('create', { + data: { + type: 'card', + attributes: { firstName: 'Mango' }, + relationships: { + friend: { data: { lid: 'nobody', type: 'card' } }, + }, + meta: { adoptsFrom: PERSON }, + }, + }), + ), + ); + + assert.strictEqual(response.status, 400, 'HTTP 400 status'); + let [error] = response.body.errors; + assert.strictEqual(error.meta.entry, 0); + assert.true( + error.detail.includes('nobody'), + `detail names the local id: ${error.detail}`, + ); + }); + }); + + module('invocation', function () { + test('a named transform runs and answers with the identity it wrote', async function (assert) { + let response = await post( + envelope( + invoke('addComment', { + href: '/report-named', + data: { body: 'Reviewed.' }, + }), + ), + ); + + assert.strictEqual(response.status, 200, 'HTTP 200 status'); + let [result] = response.body['atomic:results']; + assert.strictEqual( + result.data.id, + `${testRealmHref}report-named`, + 'the result names the card the entry targeted', + ); + assert.strictEqual( + typeof result.data.meta.version, + 'string', + 'the result carries the version the card now holds', + ); + assert.strictEqual( + result.data.attributes, + undefined, + 'a write answers with an identity rather than a document', + ); + assert.deepEqual( + storedCard('report-named.json').data.attributes?.comments, + [{ body: 'Reviewed.', postedBy: TESTER }], + 'the comment the operation appends is on disk, recording the caller ' + + 'the realm verified as the actor', + ); + }); + + test('a create names the type it mints and echoes the local id', async function (assert) { + let response = await post( + envelope( + invoke('create', { + data: { + lid: 'author', + type: 'card', + attributes: { firstName: 'Mango' }, + meta: { adoptsFrom: PERSON }, + }, + }), + invoke('create', { + data: { + lid: 'sidekick', + type: 'card', + attributes: { firstName: 'Van Gogh' }, + relationships: { + friend: { data: { lid: 'author', type: 'card' } }, + }, + meta: { adoptsFrom: PERSON }, + }, + }), + ), + ); + + assert.strictEqual(response.status, 200, 'HTTP 200 status'); + let [author, sidekick] = response.body['atomic:results']; + assert.deepEqual( + [author.data.lid, sidekick.data.lid], + ['author', 'sidekick'], + 'each create echoes the local id the client named it with', + ); + assert.true( + author.data.id.startsWith(`${testRealmHref}Person/`), + `the minted id is under the type's directory: ${author.data.id}`, + ); + assert.strictEqual( + author.data.type, + 'card', + 'a write answers with a card identity', + ); + let stored = JSON.parse( + readFileSync( + realmFile(`${sidekick.data.id.slice(testRealmHref.length)}.json`), + 'utf8', + ), + ); + assert.strictEqual( + new URL( + stored.data.relationships.friend.links.self, + `${sidekick.data.id}.json`, + ).href, + author.data.id, + 'the second card links to the one the first entry minted', + ); + }); + + test('a delete answers with no state and removes the file', async function (assert) { + let response = await post( + envelope(invoke('delete', { href: '/report-deleted' })), + ); + + assert.strictEqual(response.status, 200, 'HTTP 200 status'); + assert.deepEqual( + response.body['atomic:results'], + [{ data: null }], + 'a delete leaves no state to describe', + ); + assert.false( + existsSync(realmFile('report-deleted.json')), + 'the card is gone from disk', + ); + }); + + test('an href relative to the realm names a card inside it', async function (assert) { + let response = await post( + envelope(invoke('escalate', { href: '/report-relative' })), + ); + + assert.strictEqual(response.status, 200, 'HTTP 200 status'); + assert.strictEqual( + response.body['atomic:results'][0].data.id, + `${testRealmHref}report-relative`, + 'a leading slash names the realm root rather than the origin', + ); + }); + + test('a line is appended to a file whose extension the realm does not register', async function (assert) { + let response = await post( + envelope( + invoke('appendLine', { + href: '/telemetry.log', + data: { line: 'deployed' }, + }), + ), + ); + + assert.strictEqual(response.status, 200, 'HTTP 200 status'); + assert.strictEqual( + readFileSync(realmFile('telemetry.log'), 'utf8'), + 'boot\ndeployed\n', + 'the line is on the end of the file', + ); + }); + + test('a read answers with the document, and a write on a file is refused', async function (assert) { + let read = await query(envelope(invoke('read', { href: '/notes.md' }))); + + assert.strictEqual(read.status, 200, 'HTTP 200 status'); + assert.strictEqual( + read.body['atomic:results'][0].data.type, + 'file-meta', + 'a file reads as its metadata document', + ); + + let write = await post( + envelope(invoke('transform', { href: '/notes.md' })), + ); + assert.strictEqual(write.status, 405, 'HTTP 405 status'); + assert.strictEqual( + write.body.errors[0].code, + 'operation-not-allowed', + 'a file carries no transform', + ); + }); + }); + + module('atomicity', function () { + test('a failing entry leaves the batch unwritten, unindexed and unannounced', async function (assert) { + let jobsBefore = await indexJobIds(); + let since = Date.now(); + + let response = await post( + envelope( + invoke('read', { href: '/report-kept' }), + invoke('escalate', { href: '/report-rollback' }), + invoke('delete', { href: '/does-not-exist' }), + ), + ); + + assert.strictEqual(response.status, 404, 'HTTP 404 status'); + assert.strictEqual( + response.body.errors[0].meta.entry, + 2, + 'the error names the entry the caller sent, not the position it took ' + + 'among the entries that write', + ); + assert.strictEqual( + storedCard('report-rollback.json').data.attributes?.status, + 'open', + 'the entry that could have been carried out wrote nothing', + ); + assert.deepEqual( + await indexJobIds(), + jobsBefore, + 'no index job was enqueued', + ); + assert.deepEqual( + await incrementalIndexEventsSince(since), + [], + 'no index event was broadcast', + ); + }); + + test('a read in a mixed batch answers with the pre-batch document', async function (assert) { + let response = await post( + envelope( + invoke('read', { href: '/report-mixed' }), + invoke('escalate', { href: '/report-mixed' }), + invoke('read', { href: '/notes.md' }), + ), + ); + + assert.strictEqual(response.status, 200, 'HTTP 200 status'); + let [before, write, file] = response.body['atomic:results']; + assert.strictEqual( + before.data.attributes.status, + 'open', + 'the read answers with the state the batch started from', + ); + assert.strictEqual(write.data.id, `${testRealmHref}report-mixed`); + assert.strictEqual( + file.data.type, + 'file-meta', + 'a file read sits alongside card writes', + ); + assert.strictEqual( + storedCard('report-mixed.json').data.attributes?.status, + 'escalated', + 'the write in the same batch landed', + ); + }); + + test('an answer is never cached', async function (assert) { + let written = await post( + envelope(invoke('escalate', { href: '/report-uncached' })), + ); + assert.strictEqual(written.status, 200, 'HTTP 200 status'); + assert.strictEqual(written.get('Cache-Control'), 'no-store'); + assert.strictEqual(written.get('ETag'), undefined); + assert.strictEqual(written.get('Content-Type'), OPERATIONS); + + let read = await query( + envelope(invoke('read', { href: '/report-uncached' })), + ); + assert.strictEqual(read.status, 200, 'HTTP 200 status'); + assert.strictEqual(read.get('Cache-Control'), 'no-store'); + assert.strictEqual(read.get('ETag'), undefined); + }); + }); + + module('identity', function () { + test('an operation that reads the actor refuses a request that authenticated nobody', async function (assert) { + let response = await post( + envelope( + invoke('addComment', { + href: '/report-anonymous', + data: { body: 'Who said this?' }, + }), + ), + ); + + assert.strictEqual(response.status, 401, 'HTTP 401 status'); + let [error] = response.body.errors; + assert.strictEqual(error.code, 'actor-required'); + assert.strictEqual(error.meta.entry, 0); + assert.deepEqual( + storedCard('report-anonymous.json').data.attributes?.comments, + [], + 'nothing was written for a caller with no identity', + ); + }); + + test('an operation that reads no actor is carried out for an anonymous caller', async function (assert) { + let response = await post( + envelope(invoke('escalate', { href: '/report-anonymous' })), + ); + + assert.strictEqual(response.status, 200, 'HTTP 200 status'); + assert.strictEqual( + storedCard('report-anonymous.json').data.attributes?.status, + 'escalated', + 'a realm anyone may write carries out a batch that needs no identity', + ); + }); + + test('the actor an operation reads is the user the realm authenticated', async function (assert) { + let response = await request + .post('/_operations') + .set('Accept', OPERATIONS) + .set('Content-Type', OPERATIONS) + .set( + 'Authorization', + `Bearer ${createJWT(realm, '@tester:localhost', ['read', 'write'])}`, + ) + .send( + envelope( + invoke('addComment', { + href: '/report-identified', + data: { body: 'Reviewed.' }, + }), + ), + ); + + assert.strictEqual(response.status, 200, 'HTTP 200 status'); + assert.deepEqual( + storedCard('report-identified.json').data.attributes?.comments, + [{ body: 'Reviewed.', postedBy: '@tester:localhost' }], + 'the comment records the caller the realm verified', + ); + }); + }); +}); + +module(`${basename(import.meta.filename)} > read-only realm`, function (hooks) { + let realm: Realm; + let request: RealmRequest; + + setupPermissionedRealmCached(hooks, { + mode: 'before', + realmURL: readOnlyRealm, + permissions: { + reader: ['read'], + '@node-test_realm:localhost': ['read', 'realm-owner'], + }, + fileSystem: { + 'person.gts': ` + import { contains, field, CardDef, Component } from "@cardstack/base/card-api"; + import StringField from "@cardstack/base/string"; + + export class Person extends CardDef { + @field firstName = contains(StringField); + static isolated = class Isolated extends Component { + + } + static embedded = class Embedded extends Component { + + } + static fitted = class Fitted extends Component { + + } + } + `, + 'person-1.json': { + data: { + type: 'card', + attributes: { firstName: 'Mango' }, + meta: { + adoptsFrom: { + module: rri(`${readOnlyRealm.href}person`), + name: 'Person', + }, + }, + }, + }, + }, + onRealmSetup(args) { + realm = args.testRealm; + request = withRealmPath(args.request, readOnlyRealm); + }, + }); + + // The permission is derived from the method before a body is read, so these + // two send the same batch and differ only in how it is sent. + let batch = () => + JSON.stringify({ + 'boxel:operations': [ + { op: 'invoke', 'boxel:name': 'read', href: '/person-1' }, + ], + }); + + test('a POST needs realm write', async function (assert) { + let response = await request + .post('/_operations') + .set('Accept', OPERATIONS) + .set('Content-Type', OPERATIONS) + .set('Authorization', `Bearer ${createJWT(realm, 'reader', ['read'])}`) + .send(batch()); + + assert.strictEqual(response.status, 403, 'HTTP 403 status'); + }); + + test('a QUERY needs only realm read', async function (assert) { + let response = await request + .post('/_operations') + .set('Accept', OPERATIONS) + .set('Content-Type', OPERATIONS) + .set('X-HTTP-Method-Override', 'QUERY') + .set('Authorization', `Bearer ${createJWT(realm, 'reader', ['read'])}`) + .send(batch()); + + assert.strictEqual(response.status, 200, 'HTTP 200 status'); + assert.strictEqual( + response.body['atomic:results'][0].data.attributes.firstName, + 'Mango', + 'the read a reader is authorized for is carried out', + ); + }); +}); diff --git a/packages/runtime-common/card-operations/bxl-emit.ts b/packages/runtime-common/card-operations/bxl-emit.ts index e7e1b6e64a9..ed6ec37f4b4 100644 --- a/packages/runtime-common/card-operations/bxl-emit.ts +++ b/packages/runtime-common/card-operations/bxl-emit.ts @@ -164,3 +164,31 @@ export function paramKeysRead(source: string): string[] { } return keys; } + +// Whether a program reads the invoking actor. Read the same way the two scans +// above read their names, and for the same reason: one token pass covers both +// program flavors, a name inside a string literal is a `str` token and never +// matches, and the neighbours rule out a field access (`.actor`) and an object +// key (`{actor: 1}`). +// +// A program this cannot tokenize answers false. Such a program cannot run +// either, and it is refused for being unreadable — reporting it as needing an +// identity would answer a question about the caller that the program's own +// text never raised. +export function callsActor(source: string): boolean { + let tokens: { type: string; value: unknown }[]; + try { + tokens = tokenizeNativeJq(source, { readableSyntax: false }) as { + type: string; + value: unknown; + }[]; + } catch { + return false; + } + return tokens.some((token, index) => { + if (token.type !== 'ident' || String(token.value) !== 'actor') { + return false; + } + return tokens[index - 1]?.value !== '.' && tokens[index + 1]?.value !== ':'; + }); +} diff --git a/packages/runtime-common/card-operations/dispatch.ts b/packages/runtime-common/card-operations/dispatch.ts index 1529ab483a5..a602de4ab75 100644 --- a/packages/runtime-common/card-operations/dispatch.ts +++ b/packages/runtime-common/card-operations/dispatch.ts @@ -322,23 +322,46 @@ const ALLOWED_BASE_OPERATIONS: Readonly< // for, though, and both writes here work on them: an `update` replaces the // content wholesale and an `appendLine` adds one line to the end of a text // file without reading what is already there. - // - // Reaching either of those writes depends on a target being classified a - // file, and `defKindFor` classifies an instance target by its extension — - // which does not name every stored file. A `.log`, a `.css`, a `.yml` holds - // bytes and serves them, and each is classified `card-def` here, so a - // file-only behavior on one is refused before its executor runs. `read` - // survives that because a card carries a read too, and its executor falls - // back to the file-metadata document for exactly these paths. A write has no - // such overlap to fall back through, so the discrimination has to move to - // where it can be made: the executor, which knows whether the path holds a - // card's `.json` or plain bytes and already has to judge the content type. 'file-def': carriedBy('file-def'), // A field's instances have no URL, so nothing is invocable on one. Field // data is reached through the operations of the card that contains it. 'field-def': carriedBy('field-def'), }; +// The file-only behaviors an instance target carries whatever its URL is +// classified as. +// +// `defKindFor` classifies an instance target by its extension, and the +// registered-extension table does not name every stored file: a `.log`, a +// `.css`, a `.yml` holds bytes and serves them, and each classifies `card-def`. +// A `read` survives that because a card carries a read too and its executor +// falls back to the file-metadata document for exactly those paths. A +// file-only write has no such overlap, so it is admitted here and the +// discrimination is made where the answer is available: the executor, which +// reads whether the path holds a card's `.json` or plain bytes and already has +// to judge the content type to decide whether a line may be appended at all. +// +// Only an instance target, and only for what a URL can under-report. A type +// target's kind comes from its definition rather than from an extension, so +// nothing about it is uncertain, and there is no instance behind it for a +// stored-bytes write to reach. +const FILE_WRITES_ON_ANY_INSTANCE: Readonly< + Partial> +> = { appendLine: true }; + +function carries( + target: OperationTarget, + kind: DefKind, + base: BaseOperation, +): boolean { + if (own(ALLOWED_BASE_OPERATIONS[kind], base)) { + return true; + } + return ( + target.kind === 'instance' && own(FILE_WRITES_ON_ANY_INSTANCE, base) != null + ); +} + // The base operations that resolve without consulting a definition. // // A definition is consulted for two reasons — to find a declaration of the @@ -470,7 +493,7 @@ export async function resolveOperation( `serves the bytes stored at the target's URL`, }); } - if (!own(ALLOWED_BASE_OPERATIONS[kind], declared.base)) { + if (!carries(target, kind, declared.base)) { throw notAllowed(target, name, kind, declared.base); } return declared; @@ -484,7 +507,7 @@ export async function resolveOperation( detail: `there is no operation named "${name}" on ${describeTarget(target)}`, }); } - if (!own(ALLOWED_BASE_OPERATIONS[kind], name)) { + if (!carries(target, kind, name)) { throw notAllowed(target, name, kind, name); } // The built-in behavior, undeclared. It has no program and no params, and @@ -836,7 +859,7 @@ function notAllowed( // target's own terms are accurate either way. let because = target.kind === 'instance' && kind - ? `a ${kind} allows ${describeAllowed(kind)}` + ? `a ${kind} allows ${describeAllowed(kind, target)}` : `a "${base}" runs against an instance, and a type is not one`; return new OperationFailure({ id: targetId(target), @@ -849,8 +872,13 @@ function notAllowed( }); } -function describeAllowed(kind: DefKind): string { - let allowed = Object.keys(ALLOWED_BASE_OPERATIONS[kind] ?? {}); +function describeAllowed(kind: DefKind, target: OperationTarget): string { + // Asked through `carries` rather than read off the table, so a refusal lists + // what this target actually carries — which for an instance target includes + // the file writes the table admits on top of its kind. + let allowed = (Object.keys(ALL_BASE_OPERATIONS) as BaseOperation[]).filter( + (base) => carries(target, kind, base), + ); return allowed.length > 0 ? allowed.join(', ') : 'no operations'; } diff --git a/packages/runtime-common/card-operations/envelope.ts b/packages/runtime-common/card-operations/envelope.ts new file mode 100644 index 00000000000..f0f9f7ab73b --- /dev/null +++ b/packages/runtime-common/card-operations/envelope.ts @@ -0,0 +1,636 @@ +import { BOXEL_OPERATIONS_EXT } from '../supported-mime-type.ts'; +import { RealmPaths, type LocalPath } from '../paths.ts'; +import { callsActor } from './bxl-emit.ts'; +import { + OperationFailure, + isDocumentResult, + isOperationFailure, + type BaseOperation, + type OperationDefinition, + type OperationError, + type OperationResult, + type OperationTarget, + type OperationTemplate, +} from './types.ts'; +import type { BatchEntryResult } from './coordinator.ts'; +import type { BatchEntry } from './executors.ts'; +import type { CodeRef } from '../code-ref.ts'; +import type { CardResource } from '../resource-types.ts'; + +// ============================================================================ +// The operations envelope: reading a batch off the wire, and writing its +// answer back. +// +// Everything here is text and plain data — a request body in, entries and +// results out. It resolves no identifier, reads no file and consults no index, +// which is what keeps it runnable wherever the operation core runs and lets it +// be tested without a realm. The two questions it cannot answer are asked by +// whoever calls it: which behavior each entry's name resolves to (the target's +// definition decides that) and whether the batch commits. +// +// The entry vocabulary is deliberately a discriminated union on `op` with one +// member. `invoke` names an operation to run; the extension also defines group +// verbs whose members are batches in their own right, and they are read here +// as verbs this endpoint does not carry rather than as malformed entries, so +// adding one is a new arm rather than a reshaping of this parse. +// ============================================================================ + +// A JSON:API media type carries its extensions in the `ext` parameter, as a +// space-separated list of URIs. The router matches the one spelling the +// envelope is sent under; this reads the parameter itself, so a body sent as +// plain `application/vnd.api+json` — the near miss a client makes — is told +// what it is missing instead of falling through to a route that does not +// exist. +export function carriesOperationsExt(contentType: string | null): boolean { + if (!contentType) { + return false; + } + let [mediaType, ...parameters] = contentType.split(';'); + if (mediaType.trim().toLowerCase() !== 'application/vnd.api+json') { + return false; + } + for (let parameter of parameters) { + let separator = parameter.indexOf('='); + if (separator === -1) { + continue; + } + if (parameter.slice(0, separator).trim().toLowerCase() !== 'ext') { + continue; + } + let value = parameter.slice(separator + 1).trim(); + // Quoted in every spelling that carries more than one URI, since the + // separator between them is a space. + if (value.startsWith('"') && value.endsWith('"')) { + value = value.slice(1, -1); + } + if (value.split(/\s+/).includes(BOXEL_OPERATIONS_EXT)) { + return true; + } + } + return false; +} + +// One `invoke` entry, with the parts the wire spells resolved into the terms +// the core takes: an `href` that has been resolved against this realm and +// found to be inside it, and the local id read out of the payload. +export interface EnvelopeEntry { + // Where the entry sat in the batch. Every refusal is labelled with it, so a + // caller reading one error knows which of the entries it sent produced it. + index: number; + name: string; + // Absolute, and inside this realm. Absent on an entry that names no existing + // resource, which is a create of a card that does not exist yet. + href?: string; + data?: Record; + // The caller's own id for a card this batch mints, read from `data.lid` — + // the resource-level member JSON:API already reserves for exactly this, and + // the key later entries link to the new card by. + lid?: string; +} + +// The request body, read as a batch. +// +// An empty list is a batch that asks for nothing, and it is carried out as +// one: the coordinator takes no lock and announces nothing for it, so the +// envelope answers with no results rather than inventing a refusal for a +// request that would change nothing either way. +export function parseOperationsEnvelope( + body: unknown, + realmURL: string, +): EnvelopeEntry[] { + if (!isPlainRecord(body)) { + throw refuse(`the request body is not a JSON:API document`); + } + let operations = body['boxel:operations']; + if (!Array.isArray(operations)) { + throw refuse( + `the request body carries no "boxel:operations" list of operations`, + ); + } + let paths = new RealmPaths(new URL(realmURL)); + return operations.map((operation, index) => + parseEntry(operation, index, paths), + ); +} + +function parseEntry( + operation: unknown, + index: number, + paths: RealmPaths, +): EnvelopeEntry { + if (!isPlainRecord(operation)) { + throw refuse(`entry ${index} is not an operation`, index); + } + let op = operation.op; + switch (op) { + case 'invoke': + return parseInvocation(operation, index, paths); + case 'parallel': + case 'serial': + throw refuse( + `entry ${index} is a "${op}" group, and this endpoint carries ` + + `"invoke" entries`, + index, + ); + default: + throw refuse( + `entry ${index} names ${ + typeof op === 'string' ? `operation "${op}"` : 'no operation' + }, and an entry in this envelope is an "invoke"`, + index, + ); + } +} + +function parseInvocation( + operation: Record, + index: number, + paths: RealmPaths, +): EnvelopeEntry { + let name = operation['boxel:name']; + if (typeof name !== 'string' || name.length === 0) { + throw refuse( + `entry ${index} carries no "boxel:name" naming the operation to invoke`, + index, + ); + } + let data: Record | undefined; + if (operation.data !== undefined) { + if (!isPlainRecord(operation.data)) { + throw refuse( + `entry ${index} carries a "data" that is not an object`, + index, + ); + } + data = operation.data; + } + let lid = data?.lid; + if (lid !== undefined && typeof lid !== 'string') { + throw refuse( + `entry ${index} carries a local id that is not a string`, + index, + ); + } + return { + index, + name, + ...(operation.href === undefined + ? {} + : { href: hrefIn(operation.href, index, paths) }), + ...(data ? { data } : {}), + ...(lid === undefined ? {} : { lid }), + }; +} + +// An entry's `href` as an absolute URL inside this realm. +// +// The endpoint is realm-scoped, so a relative href is relative to the realm +// rather than to its origin: `/reports/x` under a realm mounted at +// `https://…/my-realm/` is that realm's `reports/x`, not the server's. Both +// spellings are resolved through the realm's own path helper, so the one URL +// the entry ends up naming is the one the realm addresses it by. +// +// A batch commits under one realm's write lock, so an href resolving outside +// this realm is not something the endpoint can carry out rather than something +// it declines to: there is no lock it could take that would make the write +// atomic with the rest of the batch. +function hrefIn(href: unknown, index: number, paths: RealmPaths): string { + if (typeof href !== 'string' || href.length === 0) { + throw refuse(`entry ${index} carries an "href" that is not a URL`, index); + } + let absolute: URL; + try { + absolute = new URL(href); + } catch { + // Relative, so it names a path within the realm. A leading slash is the + // spelling the extension documents and means the realm's root, not the + // origin's, which is why this resolves through `fileURL` rather than + // through URL resolution against the realm. + try { + absolute = paths.fileURL(href.replace(/^\/+/, '') as LocalPath); + } catch { + throw refuse(`entry ${index} carries an "href" that is not a URL`, index); + } + } + try { + paths.local(absolute); + } catch { + throw refuse( + `entry ${index} targets ${absolute.href}, which realm ${paths.url} does ` + + `not contain; a batch commits to one realm`, + index, + absolute.href, + ); + } + return absolute.href; +} + +// What the entry runs against. An entry naming an href targets that resource; +// one naming none is creating a card that does not exist yet, and names the +// type it mints the same way a card's stored JSON names its own. +export function targetFor( + entry: EnvelopeEntry, + realmURL: string, +): OperationTarget { + if (entry.href !== undefined) { + return { kind: 'instance', url: entry.href }; + } + let adoptsFrom = asRecord(asRecord(entry.data?.meta)?.adoptsFrom); + if (!adoptsFrom) { + throw refuse( + `entry ${entry.index} names no "href" and no type in ` + + `"data.meta.adoptsFrom", so there is nothing for it to run against`, + entry.index, + ); + } + return { + kind: 'type', + codeRef: adoptsFrom as unknown as CodeRef, + realm: realmURL, + }; +} + +// The behaviors that change stored state, which is what decides the permission +// the request needed and therefore which method may carry the batch. +// +// Exhaustive over the base operations on purpose: a further behavior has to +// say here whether it writes, rather than defaulting to "read" and reaching a +// commit from a request that was only authorized to read. +const WRITES: Readonly> = { + read: false, + readSource: false, + query: false, + create: true, + update: true, + delete: true, + transform: true, + appendContainsMany: true, + appendLine: true, +}; + +export function isWrite(base: BaseOperation): boolean { + return WRITES[base]; +} + +// The two behaviors that are reached somewhere other than here. +// +// Both are refusals about the entry point rather than about the operation: a +// query is planned and run on the search engine, and stored bytes are served +// by the source and byte routes. Neither has a representation in a JSON batch +// — one answers a collection and the other answers bytes — so an entry naming +// either is told where it belongs instead of being carried out differently +// here. +export function assertTravelsInEnvelope( + entry: EnvelopeEntry, + definition: OperationDefinition, +): void { + if (definition.base !== 'query' && definition.base !== 'readSource') { + return; + } + throw new OperationFailure({ + ...(entry.href ? { id: entry.href } : {}), + status: 400, + code: 'wrong-entry-point', + title: 'Operation not carried here', + detail: + definition.base === 'query' + ? `operation "${entry.name}" is a query, which runs on the search ` + + `engine rather than in a batch` + : `operation "${entry.name}" reads stored bytes, which the card ` + + `source and byte routes serve rather than a JSON batch`, + meta: { entry: entry.index }, + }); +} + +// Whether carrying this operation out needs to know who the caller is. +// +// Decidable before anything runs, because every place an operation can read +// the actor is part of its stored definition: the programs it runs, and the +// template a named create fills. That is what makes it worth asking here — +// an anonymous caller on a realm that lets anyone write is told its request +// needs an identity, once for the whole batch, rather than having an entry +// refuse part-way through for a reason that reads as a payload problem. +export function needsActor(definition: OperationDefinition): boolean { + for (let program of [ + definition.program, + definition.input, + definition.output, + ]) { + if (program && callsActor(program.source)) { + return true; + } + } + return definition.fill !== undefined && templateReadsActor(definition.fill); +} + +function templateReadsActor(template: OperationTemplate): boolean { + if (Array.isArray(template)) { + return template.some(templateReadsActor); + } + if (template === null || typeof template !== 'object') { + return false; + } + if ((template as Record).$ref === 'actor') { + return true; + } + return Object.values(template).some(templateReadsActor); +} + +// One entry as the batch coordinator takes it. +// +// The wire never names a base operation — an entry names the operation and the +// target's definition says what it is — so this is where the one `data` member +// the envelope carries is read as whatever that behavior expects of it. Each +// arm reads only what the envelope alone can decide; how well the payload fits +// the behavior is the executor's to judge, and it judges it against the state +// it reads under the write lock. +export function batchEntryFor( + entry: EnvelopeEntry, + definition: OperationDefinition, +): BatchEntry { + let { index, name } = entry; + switch (definition.base) { + case 'create': { + if (definition.of) { + // A named create stages its card from the type and template its + // declaration carries, so its payload is the operation's params and + // its href — when it has one — is the card it reads for context. + return { + op: 'create', + definition, + params: paramsFor(entry), + ...(entry.href ? { href: entry.href } : {}), + ...(entry.lid === undefined ? {} : { lid: entry.lid }), + }; + } + // The base behavior mints the card the payload describes, so the payload + // is a JSON:API resource — the same one a `POST` of a new card carries, + // local id included. Handed over as it arrived: what makes a resource a + // card is the executor's to judge, against the type it resolves. + if (entry.href !== undefined) { + throw refuse( + `entry ${index} invokes "${name}", which mints a card, and names an ` + + `href; a create has no existing resource to target`, + index, + entry.href, + ); + } + return { + op: 'create', + definition, + document: { data: entry.data as unknown as CardResource }, + }; + } + case 'update': { + let href = hrefRequired(entry, definition.base); + let content = entry.data?.content; + if (content !== undefined) { + // A file's content replaces its bytes wholesale. It travels as UTF-8 + // text, which is the only form a JSON body can carry — the facade's + // upload routes are where bytes that are not text are replaced, and + // they are also the only callers that replace bytes verbatim, so + // nothing here asks for that. + if (typeof content !== 'string') { + throw refuse( + `entry ${index} replaces the content of ${href} with something ` + + `that is not text`, + index, + href, + ); + } + return { op: 'update', definition, href, content }; + } + return { + op: 'update', + definition, + href, + document: { data: entry.data as unknown as CardResource }, + }; + } + case 'delete': + return { + op: 'delete', + definition, + href: hrefRequired(entry, 'delete'), + }; + case 'transform': + return { + op: 'transform', + definition, + params: paramsFor(entry), + href: hrefRequired(entry, 'transform'), + name, + }; + case 'appendLine': + return { + op: 'appendLine', + definition, + params: paramsFor(entry), + href: hrefRequired(entry, 'appendLine'), + }; + case 'appendContainsMany': + return { + op: 'appendContainsMany', + definition, + params: paramsFor(entry), + href: hrefRequired(entry, 'appendContainsMany'), + // Named the way the append executor names them, so the fields and + // their items are handed through rather than restated. + ...(entry.data?.field === undefined + ? {} + : { field: entry.data.field as string }), + ...(entry.data?.items === undefined + ? {} + : { items: entry.data.items as unknown[] }), + ...(entry.data?.fields === undefined + ? {} + : { fields: entry.data.fields as Record }), + }; + default: + // Unreachable for a base that writes, which is the only kind of entry + // staged. Present because falling off the switch returns `undefined`, + // and an undefined entry reaches the coordinator as an entry naming no + // operation — a refusal that describes the realm rather than the request. + throw new OperationFailure({ + status: 500, + code: 'internal-error', + title: 'Unstageable entry', + detail: + `operation "${name}" is a "${definition.base}", which is not a ` + + `behavior a batch stages`, + meta: { entry: index }, + }); + } +} + +// The payload an operation's own params are read from. +// +// `lid` is the one member of `data` that is never a param: it is the caller's +// id for the card the entry mints, which is what other entries link to it by, +// and an operation declaring a param under that name would have the two +// meanings arrive in one key. +export function paramsFor(entry: EnvelopeEntry): Record { + if (entry.lid === undefined) { + return entry.data ?? {}; + } + let { lid: _lid, ...params } = entry.data ?? {}; + return params; +} + +function hrefRequired(entry: EnvelopeEntry, base: BaseOperation): string { + if (entry.href === undefined) { + throw refuse( + `entry ${entry.index} invokes "${entry.name}", which is a "${base}" and ` + + `runs against an existing resource, and names no href`, + entry.index, + ); + } + return entry.href; +} + +// --------------------------------------------------------------------------- +// The answer +// --------------------------------------------------------------------------- + +// One positional element of `atomic:results`. +// +// A write reports an identity rather than a document: the caller wrote the +// state and the common case is reconciling against the version, so reprinting +// what was written costs an assembly nobody asked for. A read reports the +// document, which is the whole of what it was asked for. A delete reports +// `null`, since there is no state left to describe. +export type EnvelopeResult = + | { data: Record | null } + | Record; + +export function writeResult(result: BatchEntryResult): EnvelopeResult { + if (!result) { + return { data: null }; + } + // Built member by member rather than by spreading what the coordinator + // reports. The result the core carries between its own collaborators grows + // members as behaviors need them, and the wire is a contract with clients: + // a member reaches it because it was put here, not because it happened to + // be in scope. + return { + data: { + type: 'card', + id: result.id, + ...(result.lid === undefined ? {} : { lid: result.lid }), + meta: { + version: result.meta.version, + generation: result.meta.generation, + lastModified: result.meta.lastModified, + ...(result.meta.baseMatched === undefined + ? {} + : { baseMatched: result.meta.baseMatched }), + }, + }, + }; +} + +export function readResult( + entry: EnvelopeEntry, + result: OperationResult, +): EnvelopeResult { + if (!isDocumentResult(result)) { + throw new OperationFailure({ + ...(entry.href ? { id: entry.href } : {}), + status: 500, + code: 'internal-error', + title: 'Unreadable result', + detail: `the read in entry ${entry.index} answered with no document`, + meta: { entry: entry.index }, + }); + } + return result.document as unknown as Record; +} + +// Every entry answers, and its answer sits where the entry did. +// +// An unfilled position would serialize as `null`, which is what a delete +// reports — so a batch that ran fewer entries than it read would answer with a +// removal nobody asked for rather than saying that something did not run. +export function answered( + result: EnvelopeResult | undefined, + index: number, +): EnvelopeResult { + if (result === undefined) { + throw new OperationFailure({ + status: 500, + code: 'internal-error', + title: 'Missing result', + detail: `entry ${index} was read from the batch and produced no result`, + meta: { entry: index }, + }); + } + return result; +} + +// A refusal as a JSON:API error document. The batch is all-or-nothing, so one +// error is the whole answer: there is no partial outcome to describe alongside +// it and nothing was written. +export function errorsDocument(error: OperationError): { + errors: [OperationError]; +} { + return { errors: [error] }; +} + +// Label a refusal with the position of the entry that produced it, so a caller +// reading one error knows which of the entries it sent is wrong. +// +// A refusal that already carries a position keeps it. The coordinator labels +// what it stages with the position in the batch it was handed, which is not +// the position in the envelope when the batch holds only the entries that +// write — so `at` is what the two are reconciled through, and relabelling one +// that arrived correct would overwrite an answer with a guess. +export function labelEntry( + err: unknown, + at: (index: number) => number, +): unknown { + if (!isOperationFailure(err)) { + return err; + } + let entry = err.error.meta?.entry; + if (typeof entry !== 'number') { + return err; + } + return new OperationFailure({ + ...err.error, + meta: { ...err.error.meta, entry: at(entry) }, + }); +} + +// The same, for a refusal raised where the position is known outright — an +// entry parsed, resolved or staged one at a time. +export function atEntry(err: unknown, index: number): unknown { + if (!isOperationFailure(err)) { + return err; + } + if (typeof err.error.meta?.entry === 'number') { + return err; + } + return new OperationFailure({ + ...err.error, + meta: { ...err.error.meta, entry: index }, + }); +} + +function refuse(detail: string, index?: number, id?: string): OperationFailure { + return new OperationFailure({ + ...(id ? { id } : {}), + status: 400, + code: 'invalid-params', + title: 'Invalid operations envelope', + detail, + ...(index === undefined ? {} : { meta: { entry: index } }), + }); +} + +function isPlainRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} + +function asRecord(value: unknown): Record | undefined { + return isPlainRecord(value) ? value : undefined; +} diff --git a/packages/runtime-common/card-operations/index.ts b/packages/runtime-common/card-operations/index.ts index 5df0d969cd0..5c9798dfbe4 100644 --- a/packages/runtime-common/card-operations/index.ts +++ b/packages/runtime-common/card-operations/index.ts @@ -79,6 +79,23 @@ export { } from './json-splice.ts'; export type { CardSourceLayout, StoredContainer } from './json-splice.ts'; export { readSourceOperation } from './read-source.ts'; +export { + answered, + assertTravelsInEnvelope, + atEntry, + batchEntryFor, + carriesOperationsExt, + errorsDocument, + isWrite, + labelEntry, + needsActor, + paramsFor, + parseOperationsEnvelope, + readResult, + targetFor, + writeResult, +} from './envelope.ts'; +export type { EnvelopeEntry, EnvelopeResult } from './envelope.ts'; export { lowerQueryOperation } from './query.ts'; export type { QueryInvocation } from './query.ts'; export { diff --git a/packages/runtime-common/card-operations/types.ts b/packages/runtime-common/card-operations/types.ts index ce844d53731..852bcfca611 100644 --- a/packages/runtime-common/card-operations/types.ts +++ b/packages/runtime-common/card-operations/types.ts @@ -514,6 +514,10 @@ export type OperationErrorCode = // The request named a `baseVersion` the target is no longer at, on an // operation that requires the base to match. | 'version-conflict' + // The operation reads the invoking actor and the request authenticated + // nobody. Distinct from `invalid-params` because nothing the caller sent is + // wrong: the remedy is credentials, which is what its 401 says. + | 'actor-required' // The bytes an operation would store are over the realm's ceiling for a // card or a file of that kind. Separate from `invalid-params` because the // payload is well formed and the remedy is to send less of it, and because diff --git a/packages/runtime-common/realm.ts b/packages/runtime-common/realm.ts index 93e9de51402..284a0b560d3 100644 --- a/packages/runtime-common/realm.ts +++ b/packages/runtime-common/realm.ts @@ -171,22 +171,47 @@ import { type LooseCardResource, type FileMetaResource, } from './index.ts'; -import { runOperation } from './card-operations/dispatch.ts'; +import { + newOperationScope, + resolveOperation, + runOperation, +} from './card-operations/dispatch.ts'; import type { OperationCore, + OperationScope, OperationStoredFile, OperationStoredFileMeta, } from './card-operations/dispatch.ts'; import { commitBatch } from './card-operations/coordinator.ts'; import { + answered, + assertTravelsInEnvelope, + atEntry, + batchEntryFor, + carriesOperationsExt, + errorsDocument, + isWrite, + labelEntry, + needsActor, + paramsFor, + parseOperationsEnvelope, + readResult, + targetFor, + writeResult, + type EnvelopeEntry, + type EnvelopeResult, +} from './card-operations/envelope.ts'; +import { + OperationFailure, isDocumentResult, isHeadResult, isOperationFailure, isSourceResult, - type OperationFailure, + type OperationDefinition, type OperationRequest, type OperationResult, type OperationSourceResult, + type OperationTarget, } from './card-operations/types.ts'; import { erroredTargetRow } from './card-operations/read.ts'; import type { BatchCore } from './card-operations/coordinator.ts'; @@ -1810,6 +1835,38 @@ export class Realm { SupportedMimeType.JSONAPI, this.handleAtomicOperations.bind(this), ) + // The operations envelope, under the media type that carries its + // extension and under the plain JSON:API one it extends. Both reach the + // same handler, which reads the `ext` parameter itself: matching only the + // extended spelling would leave a body sent as plain + // `application/vnd.api+json` — the near miss a client makes — falling + // through to a path nothing serves, and answering "no such route" to a + // request that named this one. + // + // The method chooses the permission the realm checks, so the two verbs + // are what separates a batch that may write from one that may not: + // `POST` needs realm write, `QUERY` needs realm read, and a `QUERY` + // carrying a write is refused by the handler. + .post( + '/_operations', + SupportedMimeType.BoxelOperations, + this.handleOperations.bind(this), + ) + .query( + '/_operations', + SupportedMimeType.BoxelOperations, + this.handleOperations.bind(this), + ) + .post( + '/_operations', + SupportedMimeType.JSONAPI, + this.handleOperations.bind(this), + ) + .query( + '/_operations', + SupportedMimeType.JSONAPI, + this.handleOperations.bind(this), + ) .post( '/_cancel-indexing-job', SupportedMimeType.JSON, @@ -3991,6 +4048,239 @@ export class Realm { }); } + // The operations envelope: a batch of named operations, committed all or + // nothing. + // + // This is the second front door onto the operation core and it adds no + // behavior of its own. It reads the batch off the wire, asks the core which + // behavior each entry's name resolves to for its target, runs the reads and + // hands the writes to the coordinator — the same `runOperation` and + // `commitBatch` the card verbs dispatch into, so the two transports cannot + // drift into meaning different things by the same operation. + // + // **Access posture.** Operations are identity-aware but not access-enforced. + // The realm's own read/write permission is the whole of what is checked: any + // caller who may write the realm may invoke any operation that writes it, + // and any caller who may read it may invoke any read. An operation's program + // can read `actor()` and an `assert` can refuse on what it finds, but the + // realm verifies no claim beyond the one its permission check already made, + // and refuses nothing on the strength of who is asking. Treat every + // operation's result as reachable by any permitted caller of this realm. + private async handleOperations( + request: Request, + requestContext: RequestContext, + ): Promise { + try { + return await this.#runOperationsBatch(request, requestContext); + } catch (err: unknown) { + if (!isOperationFailure(err)) { + throw err; + } + // A batch is all-or-nothing, so the first refusal is the whole answer: + // nothing was written, no index job was enqueued and no event was + // broadcast, whichever stage produced it. + return this.#operationsResponse( + errorsDocument(err.error), + err.error.status, + requestContext, + ); + } + } + + async #runOperationsBatch( + request: Request, + requestContext: RequestContext, + ): Promise { + if (!carriesOperationsExt(request.headers.get('Content-Type'))) { + throw new OperationFailure({ + status: 400, + code: 'invalid-params', + title: 'Invalid content type', + detail: + `a batch of operations is sent as ` + + `"${SupportedMimeType.BoxelOperations}"; this request's ` + + `content type does not name that extension`, + }); + } + let body: unknown; + try { + body = JSON.parse(await request.text()); + } catch { + throw new OperationFailure({ + status: 400, + code: 'invalid-params', + title: 'Invalid operations envelope', + detail: `the request body is not valid JSON`, + }); + } + let entries = parseOperationsEnvelope(body, this.url); + let caller = this.#callerOf(request, requestContext); + // One row peek per target for the whole resolution pass: entries often + // name the same card, and which behavior a name resolves to is read off + // the target's stored type. + let scope = newOperationScope(this.operationCore); + let resolved = await Promise.all( + entries.map((entry) => this.#resolveEnvelopeEntry(entry, scope)), + ); + + // `QUERY` is the read-only spelling, and the realm derives the permission + // it checks from the method — so a write reaching here arrived on a + // request that was only authorized to read. Refused before anything is + // staged rather than let through to a permission check that already + // passed for the wrong question. + if (request.method === 'QUERY') { + let write = resolved.find(({ definition }) => isWrite(definition.base)); + if (write) { + throw new OperationFailure({ + ...(write.entry.href ? { id: write.entry.href } : {}), + status: 400, + code: 'wrong-entry-point', + title: 'Write in a read-only batch', + detail: + `operation "${write.entry.name}" writes, and a QUERY batch is ` + + `authorized to read; send a batch that writes as a POST`, + meta: { entry: write.entry.index }, + }); + } + } + + // An anonymous caller on a realm anyone may read or write has no identity + // for an operation to read, and whether an operation reads one is settled + // by its stored definition — so the batch is refused here, before any of + // it runs, rather than part-way through by whichever entry reached the + // actor first. No identity is invented to stand in: a fabricated id would + // be written into cards and compared in filters as though someone had + // acted. + if (!caller.actor) { + let needing = resolved.find(({ definition }) => needsActor(definition)); + if (needing) { + throw new OperationFailure({ + ...(needing.entry.href ? { id: needing.entry.href } : {}), + status: 401, + code: 'actor-required', + title: 'Operation needs an identity', + detail: + `operation "${needing.entry.name}" reads the invoking actor, and ` + + `this request authenticated nobody`, + meta: { entry: needing.entry.index }, + }); + } + } + + // Reads run first and against the state the batch started from, which is + // what "an entry sees pre-batch state" means for a mixed batch: a read + // entry never observes what a write entry in the same batch stages, and + // reading before the coordinator takes the write lock is what keeps a read + // from waiting on one. + let results: (EnvelopeResult | undefined)[] = new Array(entries.length); + for (let { entry, target, definition } of resolved) { + if (isWrite(definition.base)) { + continue; + } + let result: OperationResult; + try { + result = await runOperation(this.operationCore, { + target, + name: entry.name, + ...(entry.data ? { params: paramsFor(entry) } : {}), + ...caller, + }); + } catch (err: unknown) { + throw atEntry(err, entry.index); + } + results[entry.index] = readResult(entry, result); + } + + let writes = resolved.filter(({ definition }) => isWrite(definition.base)); + if (writes.length > 0) { + let staged = writes.map(({ entry, definition }) => { + try { + return batchEntryFor(entry, definition); + } catch (err: unknown) { + throw atEntry(err, entry.index); + } + }); + let committed: Awaited>; + try { + committed = await commitBatch(this.batchCore, staged, { + clientRequestId: caller.clientRequestId || null, + actor: requestContext.authenticatedUser ?? undefined, + }); + } catch (err: unknown) { + // The coordinator labels a refusal with the position in the batch it + // was handed, and that batch holds only the entries that write — so a + // mixed batch's positions are not the envelope's, and the caller is + // told about an entry it did not send unless they are mapped back. + throw labelEntry(err, (index) => writes[index].entry.index); + } + for (let [index, { entry }] of writes.entries()) { + results[entry.index] = writeResult(committed[index]); + } + } + + return this.#operationsResponse( + { + 'atomic:results': results.map((result, index) => + answered(result, index), + ), + }, + 200, + requestContext, + ); + } + + // Which behavior one entry's name means for its target, labelled with the + // entry's position. + // + // The name is resolved against the target's own definition, never read off + // the wire — the envelope carries the operation's name, and what a `delete` + // does is whatever the card's type says it does. + async #resolveEnvelopeEntry( + entry: EnvelopeEntry, + scope: OperationScope, + ): Promise<{ + entry: EnvelopeEntry; + target: OperationTarget; + definition: OperationDefinition; + }> { + try { + let target = targetFor(entry, this.url); + let definition = await resolveOperation( + this.operationCore, + target, + entry.name, + scope, + ); + assertTravelsInEnvelope(entry, definition); + return { entry, target, definition }; + } catch (err: unknown) { + throw atEntry(err, entry.index); + } + } + + // A batch's answer is never HTTP-cached. It is not a resource with a + // validator: the same request run twice writes twice, and a read entry's + // document is served without the index-time validator the card+json `GET` + // builds — so there is nothing here a conditional request could be answered + // against, and no ETag is emitted for one to be compared with. + #operationsResponse( + body: unknown, + status: number, + requestContext: RequestContext, + ): Response { + return createResponse({ + body: JSON.stringify(body, null, 2), + init: { + status, + headers: { + 'content-type': SupportedMimeType.BoxelOperations, + 'cache-control': 'no-store', + }, + }, + requestContext, + }); + } + // we track our own writes so that we can eliminate echoes in the file watcher // Write a file whose content is described as an edit of its own bytes. diff --git a/packages/runtime-common/supported-mime-type.ts b/packages/runtime-common/supported-mime-type.ts index 490a4d74458..5e75d629017 100644 --- a/packages/runtime-common/supported-mime-type.ts +++ b/packages/runtime-common/supported-mime-type.ts @@ -1,3 +1,10 @@ +// The JSON:API extension the operations envelope is defined by. It names the +// `invoke` verb and the `boxel:`-prefixed members the envelope carries, which +// is what a plain `application/vnd.api+json` body does not have — so the media +// type below carries it as the `ext` parameter and the router matches on the +// whole string. +export const BOXEL_OPERATIONS_EXT = 'https://boxel.ai/ext/operations'; + // A `const` object (rather than a TS `enum`) so the declaration is // erasable and runs under Node's native `--experimental-strip-types`. // The merged type below keeps `SupportedMimeType` usable as both a value @@ -22,6 +29,12 @@ export const SupportedMimeType = { HTML: 'text/html', Markdown: 'text/markdown', JSONAPI: 'application/vnd.api+json', + // The operations envelope: a JSON:API document extended with the `invoke` + // verb and the `boxel:operations` member. Spelled from the extension URI + // above so the media type and the parameter a handler validates cannot drift + // apart. + BoxelOperations: + `application/vnd.api+json;ext="${BOXEL_OPERATIONS_EXT}"` as const, JSON: 'application/json', CardDependencies: 'application/json', CardTypeSummary: 'application/json', From 58b302e4b43615c0deff416378b17fffe2c92d79 Mon Sep 17 00:00:00 2001 From: Hassan Abdel-Rahman Date: Wed, 16 Sep 2026 18:41:30 -0400 Subject: [PATCH 02/13] Name the earliest entry a batch got wrong, not the first to answer Co-Authored-By: Claude Opus 5 (1M context) --- .../card-operations/envelope.ts | 9 +++++++ .../runtime-common/card-operations/index.ts | 6 ++++- packages/runtime-common/realm.ts | 25 ++++++++++++------- 3 files changed, 30 insertions(+), 10 deletions(-) diff --git a/packages/runtime-common/card-operations/envelope.ts b/packages/runtime-common/card-operations/envelope.ts index f0f9f7ab73b..62f57039568 100644 --- a/packages/runtime-common/card-operations/envelope.ts +++ b/packages/runtime-common/card-operations/envelope.ts @@ -250,6 +250,15 @@ export function targetFor( }; } +// One entry with the behavior its name resolved to. The name is the whole of +// what the wire says; which behavior that is comes from the target's own +// definition, so an entry is only actionable once the two are together. +export interface ResolvedEnvelopeEntry { + entry: EnvelopeEntry; + target: OperationTarget; + definition: OperationDefinition; +} + // The behaviors that change stored state, which is what decides the permission // the request needed and therefore which method may carry the batch. // diff --git a/packages/runtime-common/card-operations/index.ts b/packages/runtime-common/card-operations/index.ts index 5c9798dfbe4..d67a4422ab4 100644 --- a/packages/runtime-common/card-operations/index.ts +++ b/packages/runtime-common/card-operations/index.ts @@ -95,7 +95,11 @@ export { targetFor, writeResult, } from './envelope.ts'; -export type { EnvelopeEntry, EnvelopeResult } from './envelope.ts'; +export type { + EnvelopeEntry, + EnvelopeResult, + ResolvedEnvelopeEntry, +} from './envelope.ts'; export { lowerQueryOperation } from './query.ts'; export type { QueryInvocation } from './query.ts'; export { diff --git a/packages/runtime-common/realm.ts b/packages/runtime-common/realm.ts index 284a0b560d3..36183e0a1d7 100644 --- a/packages/runtime-common/realm.ts +++ b/packages/runtime-common/realm.ts @@ -200,6 +200,7 @@ import { writeResult, type EnvelopeEntry, type EnvelopeResult, + type ResolvedEnvelopeEntry, } from './card-operations/envelope.ts'; import { OperationFailure, @@ -207,11 +208,9 @@ import { isHeadResult, isOperationFailure, isSourceResult, - type OperationDefinition, type OperationRequest, type OperationResult, type OperationSourceResult, - type OperationTarget, } from './card-operations/types.ts'; import { erroredTargetRow } from './card-operations/read.ts'; import type { BatchCore } from './card-operations/coordinator.ts'; @@ -4119,9 +4118,21 @@ export class Realm { // name the same card, and which behavior a name resolves to is read off // the target's stored type. let scope = newOperationScope(this.operationCore); - let resolved = await Promise.all( + // Settled rather than raced, so the entry a refusal names is the earliest + // one the caller got wrong rather than whichever index read came back + // first. A batch with two bad entries would otherwise report a different + // one run to run. + let outcomes = await Promise.allSettled( entries.map((entry) => this.#resolveEnvelopeEntry(entry, scope)), ); + let refused = outcomes.find((outcome) => outcome.status === 'rejected'); + if (refused) { + throw refused.reason; + } + let resolved = outcomes.map( + (outcome) => + (outcome as PromiseFulfilledResult).value, + ); // `QUERY` is the read-only spelling, and the realm derives the permission // it checks from the method — so a write reaching here arrived on a @@ -4204,7 +4215,7 @@ export class Realm { try { committed = await commitBatch(this.batchCore, staged, { clientRequestId: caller.clientRequestId || null, - actor: requestContext.authenticatedUser ?? undefined, + actor: caller.actor || undefined, }); } catch (err: unknown) { // The coordinator labels a refusal with the position in the batch it @@ -4238,11 +4249,7 @@ export class Realm { async #resolveEnvelopeEntry( entry: EnvelopeEntry, scope: OperationScope, - ): Promise<{ - entry: EnvelopeEntry; - target: OperationTarget; - definition: OperationDefinition; - }> { + ): Promise { try { let target = targetFor(entry, this.url); let definition = await resolveOperation( From 3fef59439d756891c8840d3c5bb825b74375dcec Mon Sep 17 00:00:00 2001 From: Hassan Abdel-Rahman Date: Wed, 16 Sep 2026 18:42:52 -0400 Subject: [PATCH 03/13] Answer a write with the id the realm serves that card under Co-Authored-By: Claude Opus 5 (1M context) --- packages/runtime-common/card-operations/envelope.ts | 13 +++++++++++-- packages/runtime-common/realm.ts | 4 +++- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/packages/runtime-common/card-operations/envelope.ts b/packages/runtime-common/card-operations/envelope.ts index 62f57039568..634b774a4fb 100644 --- a/packages/runtime-common/card-operations/envelope.ts +++ b/packages/runtime-common/card-operations/envelope.ts @@ -512,7 +512,16 @@ export type EnvelopeResult = | { data: Record | null } | Record; -export function writeResult(result: BatchEntryResult): EnvelopeResult { +export function writeResult( + result: BatchEntryResult, + // The card's id in the form the realm serves ids in. A realm reached through + // a registered prefix answers every other surface's ids in that form — a + // read's document, a created card's `POST` response — so a batch result + // spelling the same card differently would hand back an id the caller cannot + // send back as a target. Supplied by the realm, which owns identifier + // resolution; nothing here resolves one. + canonical: (url: string) => string, +): EnvelopeResult { if (!result) { return { data: null }; } @@ -524,7 +533,7 @@ export function writeResult(result: BatchEntryResult): EnvelopeResult { return { data: { type: 'card', - id: result.id, + id: canonical(result.id), ...(result.lid === undefined ? {} : { lid: result.lid }), meta: { version: result.meta.version, diff --git a/packages/runtime-common/realm.ts b/packages/runtime-common/realm.ts index 36183e0a1d7..30654ecfca2 100644 --- a/packages/runtime-common/realm.ts +++ b/packages/runtime-common/realm.ts @@ -4225,7 +4225,9 @@ export class Realm { throw labelEntry(err, (index) => writes[index].entry.index); } for (let [index, { entry }] of writes.entries()) { - results[entry.index] = writeResult(committed[index]); + results[entry.index] = writeResult(committed[index], (url) => + this.#virtualNetwork.unresolveURL(url), + ); } } From cff95250b2986eb08a8d2c9dafdc11d0e10e6552 Mon Sep 17 00:00:00 2001 From: Hassan Abdel-Rahman Date: Wed, 16 Sep 2026 18:44:38 -0400 Subject: [PATCH 04/13] Name the suite the way the realm-endpoint suites are named Co-Authored-By: Claude Opus 5 (1M context) --- .../tests/realm-endpoints/operations-test.ts | 1186 +++++++++-------- 1 file changed, 595 insertions(+), 591 deletions(-) diff --git a/packages/realm-server/tests/realm-endpoints/operations-test.ts b/packages/realm-server/tests/realm-endpoints/operations-test.ts index ab8825ac10e..9fa1fbda175 100644 --- a/packages/realm-server/tests/realm-endpoints/operations-test.ts +++ b/packages/realm-server/tests/realm-endpoints/operations-test.ts @@ -180,609 +180,612 @@ function makeFileSystem(): Record { }; } -module(basename(import.meta.filename), function (hooks) { - let realm: Realm; - let testDbAdapter: DBAdapter; - let request: RealmRequest; - let serverRequest: SuperTest; - let testRealmHttpServer: Server; - let dir: DirResult; - - setupPermissionedRealmCached(hooks, { - mode: 'before', - realmURL: testRealm, - permissions: { - '*': ['read', 'write'], - '@node-test_realm:localhost': ['read', 'write', 'realm-owner'], - }, - subscribeToRealmEvents: true, - fileSystem: makeFileSystem(), - onRealmSetup(args) { - realm = args.testRealm; - testDbAdapter = args.dbAdapter; - request = withRealmPath(args.request, testRealm); - serverRequest = args.request; - testRealmHttpServer = args.testRealmHttpServer; - dir = args.dir; - }, - }); +module(`realm-endpoints/${basename(import.meta.filename)}`, function () { + module('a realm anyone may read and write', function (hooks) { + let realm: Realm; + let testDbAdapter: DBAdapter; + let request: RealmRequest; + let serverRequest: SuperTest; + let testRealmHttpServer: Server; + let dir: DirResult; + + setupPermissionedRealmCached(hooks, { + mode: 'before', + realmURL: testRealm, + permissions: { + '*': ['read', 'write'], + '@node-test_realm:localhost': ['read', 'write', 'realm-owner'], + }, + subscribeToRealmEvents: true, + fileSystem: makeFileSystem(), + onRealmSetup(args) { + realm = args.testRealm; + testDbAdapter = args.dbAdapter; + request = withRealmPath(args.request, testRealm); + serverRequest = args.request; + testRealmHttpServer = args.testRealmHttpServer; + dir = args.dir; + }, + }); - let { getMessagesSince } = setupMatrixRoom(hooks, () => ({ - testRealm: realm, - testRealmHttpServer, - request, - serverRequest, - dir, - dbAdapter: testDbAdapter as PgAdapter, - })); - - function realmFile(localPath: string): string { - return join(dir.name, 'realm_server_1', 'test', localPath); - } - - function storedCard(localPath: string): LooseSingleCardDocument { - return JSON.parse(readFileSync(realmFile(localPath), 'utf8')); - } - - async function indexJobIds(): Promise { - let rows = (await testDbAdapter.execute( - `select id from jobs where job_type = 'incremental-index' + let { getMessagesSince } = setupMatrixRoom(hooks, () => ({ + testRealm: realm, + testRealmHttpServer, + request, + serverRequest, + dir, + dbAdapter: testDbAdapter as PgAdapter, + })); + + function realmFile(localPath: string): string { + return join(dir.name, 'realm_server_1', 'test', localPath); + } + + function storedCard(localPath: string): LooseSingleCardDocument { + return JSON.parse(readFileSync(realmFile(localPath), 'utf8')); + } + + async function indexJobIds(): Promise { + let rows = (await testDbAdapter.execute( + `select id from jobs where job_type = 'incremental-index' and concurrency_group = $1 order by id`, - { bind: [`indexing:${realm.url}`] }, - )) as { id: number | string }[]; - return rows.map((row) => Number(row.id)); - } - - async function incrementalIndexEventsSince( - since: number, - ): Promise { - let messages = await getMessagesSince(since); - return messages - .filter((message) => message.type === APP_BOXEL_REALM_EVENT_TYPE) - .map((message) => message.content as RealmEventContent) - .filter( - (event): event is IncrementalIndexEventContent => - event.eventName === 'index' && event.indexType === 'incremental', - ); - } - - const TESTER = '@tester:localhost'; - - // The realm lets anyone read and write it, so a batch reaches the endpoint - // with or without credentials — which is the difference the identity module - // is about. Everywhere else the caller is authenticated, since that is the - // ordinary case and it is what gives an operation an actor to read. - function post(body: string) { - return anonymousPost(body).set( - 'Authorization', - `Bearer ${createJWT(realm, TESTER, ['read', 'write'])}`, - ); - } - - function anonymousPost(body: string) { - return request - .post('/_operations') - .set('Accept', OPERATIONS) - .set('Content-Type', OPERATIONS) - .send(body); - } - - // Sent as a `POST` carrying the override header, which is the spelling for - // clients that cannot send a `QUERY` method; the realm reads it back into a - // `QUERY` before it decides which permission the request needs. - function query(body: string) { - return post(body).set('X-HTTP-Method-Override', 'QUERY'); - } - - module('validation', function () { - test('an href outside this realm is refused, naming the entry', async function (assert) { - let response = await post( - envelope( - invoke('escalate', { href: '/report-kept' }), - invoke('escalate', { href: 'http://127.0.0.1:4999/other/report-x' }), - ), - ); + { bind: [`indexing:${realm.url}`] }, + )) as { id: number | string }[]; + return rows.map((row) => Number(row.id)); + } + + async function incrementalIndexEventsSince( + since: number, + ): Promise { + let messages = await getMessagesSince(since); + return messages + .filter((message) => message.type === APP_BOXEL_REALM_EVENT_TYPE) + .map((message) => message.content as RealmEventContent) + .filter( + (event): event is IncrementalIndexEventContent => + event.eventName === 'index' && event.indexType === 'incremental', + ); + } - assert.strictEqual(response.status, 400, 'HTTP 400 status'); - let [error] = response.body.errors; - assert.strictEqual(error.code, 'invalid-params'); - assert.strictEqual( - error.meta.entry, - 1, - 'the error names the entry that carries the foreign href', - ); - assert.true( - error.detail.includes('http://127.0.0.1:4999/other/report-x'), - `detail names the href: ${error.detail}`, - ); - assert.strictEqual( - storedCard('report-kept.json').data.attributes?.status, - 'open', - 'the entry that would have succeeded wrote nothing', - ); - }); + const TESTER = '@tester:localhost'; - test('a name the target does not carry is refused', async function (assert) { - let response = await post( - envelope(invoke('unheardOf', { href: '/report-kept' })), + // The realm lets anyone read and write it, so a batch reaches the endpoint + // with or without credentials — which is the difference the identity module + // is about. Everywhere else the caller is authenticated, since that is the + // ordinary case and it is what gives an operation an actor to read. + function post(body: string) { + return anonymousPost(body).set( + 'Authorization', + `Bearer ${createJWT(realm, TESTER, ['read', 'write'])}`, ); + } - assert.strictEqual(response.status, 404, 'HTTP 404 status'); - let [error] = response.body.errors; - assert.strictEqual(error.code, 'unknown-operation'); - assert.strictEqual(error.meta.entry, 0); - }); - - test('a body sent without the operations extension is told what it is missing', async function (assert) { - let response = await request + function anonymousPost(body: string) { + return request .post('/_operations') - .set('Accept', SupportedMimeType.JSONAPI) - .set('Content-Type', SupportedMimeType.JSONAPI) - .send(envelope(invoke('escalate', { href: '/report-kept' }))); - - assert.strictEqual(response.status, 400, 'HTTP 400 status'); - let [error] = response.body.errors; - assert.strictEqual(error.title, 'Invalid content type'); - assert.true( - error.detail.includes('ext='), - `detail names the extension parameter: ${error.detail}`, - ); - }); - - test('an entry naming a query is sent to the search engine', async function (assert) { - let response = await query( - envelope(invoke('openReports', { href: '/report-kept' })), - ); + .set('Accept', OPERATIONS) + .set('Content-Type', OPERATIONS) + .send(body); + } + + // Sent as a `POST` carrying the override header, which is the spelling for + // clients that cannot send a `QUERY` method; the realm reads it back into a + // `QUERY` before it decides which permission the request needs. + function query(body: string) { + return post(body).set('X-HTTP-Method-Override', 'QUERY'); + } + + module('validation', function () { + test('an href outside this realm is refused, naming the entry', async function (assert) { + let response = await post( + envelope( + invoke('escalate', { href: '/report-kept' }), + invoke('escalate', { + href: 'http://127.0.0.1:4999/other/report-x', + }), + ), + ); - assert.strictEqual(response.status, 400, 'HTTP 400 status'); - let [error] = response.body.errors; - assert.strictEqual(error.code, 'wrong-entry-point'); - assert.strictEqual(error.meta.entry, 0); - assert.true( - error.detail.includes('search engine'), - `detail says where a query runs: ${error.detail}`, - ); - }); + assert.strictEqual(response.status, 400, 'HTTP 400 status'); + let [error] = response.body.errors; + assert.strictEqual(error.code, 'invalid-params'); + assert.strictEqual( + error.meta.entry, + 1, + 'the error names the entry that carries the foreign href', + ); + assert.true( + error.detail.includes('http://127.0.0.1:4999/other/report-x'), + `detail names the href: ${error.detail}`, + ); + assert.strictEqual( + storedCard('report-kept.json').data.attributes?.status, + 'open', + 'the entry that would have succeeded wrote nothing', + ); + }); - test('an entry naming readSource is sent to the byte routes', async function (assert) { - let response = await query( - envelope(invoke('readSource', { href: '/notes.md' })), - ); + test('a name the target does not carry is refused', async function (assert) { + let response = await post( + envelope(invoke('unheardOf', { href: '/report-kept' })), + ); - assert.strictEqual(response.status, 400, 'HTTP 400 status'); - let [error] = response.body.errors; - assert.strictEqual(error.code, 'wrong-entry-point'); - assert.strictEqual(error.meta.entry, 0); - }); + assert.strictEqual(response.status, 404, 'HTTP 404 status'); + let [error] = response.body.errors; + assert.strictEqual(error.code, 'unknown-operation'); + assert.strictEqual(error.meta.entry, 0); + }); + + test('a body sent without the operations extension is told what it is missing', async function (assert) { + let response = await request + .post('/_operations') + .set('Accept', SupportedMimeType.JSONAPI) + .set('Content-Type', SupportedMimeType.JSONAPI) + .send(envelope(invoke('escalate', { href: '/report-kept' }))); + + assert.strictEqual(response.status, 400, 'HTTP 400 status'); + let [error] = response.body.errors; + assert.strictEqual(error.title, 'Invalid content type'); + assert.true( + error.detail.includes('ext='), + `detail names the extension parameter: ${error.detail}`, + ); + }); - test('a QUERY batch carrying a write is refused', async function (assert) { - let response = await query( - envelope( - invoke('read', { href: '/report-kept' }), - invoke('escalate', { href: '/report-kept' }), - ), - ); + test('an entry naming a query is sent to the search engine', async function (assert) { + let response = await query( + envelope(invoke('openReports', { href: '/report-kept' })), + ); - assert.strictEqual(response.status, 400, 'HTTP 400 status'); - let [error] = response.body.errors; - assert.strictEqual(error.code, 'wrong-entry-point'); - assert.strictEqual( - error.meta.entry, - 1, - 'the error names the entry that writes', - ); - assert.strictEqual( - storedCard('report-kept.json').data.attributes?.status, - 'open', - 'nothing was written', - ); - }); + assert.strictEqual(response.status, 400, 'HTTP 400 status'); + let [error] = response.body.errors; + assert.strictEqual(error.code, 'wrong-entry-point'); + assert.strictEqual(error.meta.entry, 0); + assert.true( + error.detail.includes('search engine'), + `detail says where a query runs: ${error.detail}`, + ); + }); - test('an entry that is not an invocation is refused', async function (assert) { - let response = await post( - envelope({ op: 'parallel', 'boxel:operations': [] }), - ); + test('an entry naming readSource is sent to the byte routes', async function (assert) { + let response = await query( + envelope(invoke('readSource', { href: '/notes.md' })), + ); - assert.strictEqual(response.status, 400, 'HTTP 400 status'); - let [error] = response.body.errors; - assert.strictEqual(error.meta.entry, 0); - assert.true( - error.detail.includes('invoke'), - `detail names the verb this endpoint carries: ${error.detail}`, - ); - }); + assert.strictEqual(response.status, 400, 'HTTP 400 status'); + let [error] = response.body.errors; + assert.strictEqual(error.code, 'wrong-entry-point'); + assert.strictEqual(error.meta.entry, 0); + }); - test('a body that is not an envelope is refused', async function (assert) { - let response = await post(JSON.stringify({ 'atomic:operations': [] })); + test('a QUERY batch carrying a write is refused', async function (assert) { + let response = await query( + envelope( + invoke('read', { href: '/report-kept' }), + invoke('escalate', { href: '/report-kept' }), + ), + ); - assert.strictEqual(response.status, 400, 'HTTP 400 status'); - let [error] = response.body.errors; - assert.true( - error.detail.includes('boxel:operations'), - `detail names the member it looked for: ${error.detail}`, - ); - }); + assert.strictEqual(response.status, 400, 'HTTP 400 status'); + let [error] = response.body.errors; + assert.strictEqual(error.code, 'wrong-entry-point'); + assert.strictEqual( + error.meta.entry, + 1, + 'the error names the entry that writes', + ); + assert.strictEqual( + storedCard('report-kept.json').data.attributes?.status, + 'open', + 'nothing was written', + ); + }); - test('two entries claiming one local id are refused', async function (assert) { - let response = await post( - envelope( - invoke('create', { - data: { - lid: 'twin', - type: 'card', - attributes: { firstName: 'Mango' }, - meta: { adoptsFrom: PERSON }, - }, - }), - invoke('create', { - data: { - lid: 'twin', - type: 'card', - attributes: { firstName: 'Van Gogh' }, - meta: { adoptsFrom: PERSON }, - }, - }), - ), - ); + test('an entry that is not an invocation is refused', async function (assert) { + let response = await post( + envelope({ op: 'parallel', 'boxel:operations': [] }), + ); - assert.strictEqual(response.status, 400, 'HTTP 400 status'); - let [error] = response.body.errors; - assert.strictEqual(error.meta.entry, 1); - assert.true( - error.detail.includes('twin'), - `detail names the local id: ${error.detail}`, - ); - }); + assert.strictEqual(response.status, 400, 'HTTP 400 status'); + let [error] = response.body.errors; + assert.strictEqual(error.meta.entry, 0); + assert.true( + error.detail.includes('invoke'), + `detail names the verb this endpoint carries: ${error.detail}`, + ); + }); - test('a local id no entry creates is refused', async function (assert) { - let response = await post( - envelope( - invoke('create', { - data: { - type: 'card', - attributes: { firstName: 'Mango' }, - relationships: { - friend: { data: { lid: 'nobody', type: 'card' } }, - }, - meta: { adoptsFrom: PERSON }, - }, - }), - ), - ); + test('a body that is not an envelope is refused', async function (assert) { + let response = await post(JSON.stringify({ 'atomic:operations': [] })); - assert.strictEqual(response.status, 400, 'HTTP 400 status'); - let [error] = response.body.errors; - assert.strictEqual(error.meta.entry, 0); - assert.true( - error.detail.includes('nobody'), - `detail names the local id: ${error.detail}`, - ); - }); - }); + assert.strictEqual(response.status, 400, 'HTTP 400 status'); + let [error] = response.body.errors; + assert.true( + error.detail.includes('boxel:operations'), + `detail names the member it looked for: ${error.detail}`, + ); + }); - module('invocation', function () { - test('a named transform runs and answers with the identity it wrote', async function (assert) { - let response = await post( - envelope( - invoke('addComment', { - href: '/report-named', - data: { body: 'Reviewed.' }, - }), - ), - ); + test('two entries claiming one local id are refused', async function (assert) { + let response = await post( + envelope( + invoke('create', { + data: { + lid: 'twin', + type: 'card', + attributes: { firstName: 'Mango' }, + meta: { adoptsFrom: PERSON }, + }, + }), + invoke('create', { + data: { + lid: 'twin', + type: 'card', + attributes: { firstName: 'Van Gogh' }, + meta: { adoptsFrom: PERSON }, + }, + }), + ), + ); - assert.strictEqual(response.status, 200, 'HTTP 200 status'); - let [result] = response.body['atomic:results']; - assert.strictEqual( - result.data.id, - `${testRealmHref}report-named`, - 'the result names the card the entry targeted', - ); - assert.strictEqual( - typeof result.data.meta.version, - 'string', - 'the result carries the version the card now holds', - ); - assert.strictEqual( - result.data.attributes, - undefined, - 'a write answers with an identity rather than a document', - ); - assert.deepEqual( - storedCard('report-named.json').data.attributes?.comments, - [{ body: 'Reviewed.', postedBy: TESTER }], - 'the comment the operation appends is on disk, recording the caller ' + - 'the realm verified as the actor', - ); - }); + assert.strictEqual(response.status, 400, 'HTTP 400 status'); + let [error] = response.body.errors; + assert.strictEqual(error.meta.entry, 1); + assert.true( + error.detail.includes('twin'), + `detail names the local id: ${error.detail}`, + ); + }); - test('a create names the type it mints and echoes the local id', async function (assert) { - let response = await post( - envelope( - invoke('create', { - data: { - lid: 'author', - type: 'card', - attributes: { firstName: 'Mango' }, - meta: { adoptsFrom: PERSON }, - }, - }), - invoke('create', { - data: { - lid: 'sidekick', - type: 'card', - attributes: { firstName: 'Van Gogh' }, - relationships: { - friend: { data: { lid: 'author', type: 'card' } }, + test('a local id no entry creates is refused', async function (assert) { + let response = await post( + envelope( + invoke('create', { + data: { + type: 'card', + attributes: { firstName: 'Mango' }, + relationships: { + friend: { data: { lid: 'nobody', type: 'card' } }, + }, + meta: { adoptsFrom: PERSON }, }, - meta: { adoptsFrom: PERSON }, - }, - }), - ), - ); + }), + ), + ); - assert.strictEqual(response.status, 200, 'HTTP 200 status'); - let [author, sidekick] = response.body['atomic:results']; - assert.deepEqual( - [author.data.lid, sidekick.data.lid], - ['author', 'sidekick'], - 'each create echoes the local id the client named it with', - ); - assert.true( - author.data.id.startsWith(`${testRealmHref}Person/`), - `the minted id is under the type's directory: ${author.data.id}`, - ); - assert.strictEqual( - author.data.type, - 'card', - 'a write answers with a card identity', - ); - let stored = JSON.parse( - readFileSync( - realmFile(`${sidekick.data.id.slice(testRealmHref.length)}.json`), - 'utf8', - ), - ); - assert.strictEqual( - new URL( - stored.data.relationships.friend.links.self, - `${sidekick.data.id}.json`, - ).href, - author.data.id, - 'the second card links to the one the first entry minted', - ); + assert.strictEqual(response.status, 400, 'HTTP 400 status'); + let [error] = response.body.errors; + assert.strictEqual(error.meta.entry, 0); + assert.true( + error.detail.includes('nobody'), + `detail names the local id: ${error.detail}`, + ); + }); }); - test('a delete answers with no state and removes the file', async function (assert) { - let response = await post( - envelope(invoke('delete', { href: '/report-deleted' })), - ); + module('invocation', function () { + test('a named transform runs and answers with the identity it wrote', async function (assert) { + let response = await post( + envelope( + invoke('addComment', { + href: '/report-named', + data: { body: 'Reviewed.' }, + }), + ), + ); - assert.strictEqual(response.status, 200, 'HTTP 200 status'); - assert.deepEqual( - response.body['atomic:results'], - [{ data: null }], - 'a delete leaves no state to describe', - ); - assert.false( - existsSync(realmFile('report-deleted.json')), - 'the card is gone from disk', - ); - }); + assert.strictEqual(response.status, 200, 'HTTP 200 status'); + let [result] = response.body['atomic:results']; + assert.strictEqual( + result.data.id, + `${testRealmHref}report-named`, + 'the result names the card the entry targeted', + ); + assert.strictEqual( + typeof result.data.meta.version, + 'string', + 'the result carries the version the card now holds', + ); + assert.strictEqual( + result.data.attributes, + undefined, + 'a write answers with an identity rather than a document', + ); + assert.deepEqual( + storedCard('report-named.json').data.attributes?.comments, + [{ body: 'Reviewed.', postedBy: TESTER }], + 'the comment the operation appends is on disk, recording the caller ' + + 'the realm verified as the actor', + ); + }); - test('an href relative to the realm names a card inside it', async function (assert) { - let response = await post( - envelope(invoke('escalate', { href: '/report-relative' })), - ); + test('a create names the type it mints and echoes the local id', async function (assert) { + let response = await post( + envelope( + invoke('create', { + data: { + lid: 'author', + type: 'card', + attributes: { firstName: 'Mango' }, + meta: { adoptsFrom: PERSON }, + }, + }), + invoke('create', { + data: { + lid: 'sidekick', + type: 'card', + attributes: { firstName: 'Van Gogh' }, + relationships: { + friend: { data: { lid: 'author', type: 'card' } }, + }, + meta: { adoptsFrom: PERSON }, + }, + }), + ), + ); - assert.strictEqual(response.status, 200, 'HTTP 200 status'); - assert.strictEqual( - response.body['atomic:results'][0].data.id, - `${testRealmHref}report-relative`, - 'a leading slash names the realm root rather than the origin', - ); - }); + assert.strictEqual(response.status, 200, 'HTTP 200 status'); + let [author, sidekick] = response.body['atomic:results']; + assert.deepEqual( + [author.data.lid, sidekick.data.lid], + ['author', 'sidekick'], + 'each create echoes the local id the client named it with', + ); + assert.true( + author.data.id.startsWith(`${testRealmHref}Person/`), + `the minted id is under the type's directory: ${author.data.id}`, + ); + assert.strictEqual( + author.data.type, + 'card', + 'a write answers with a card identity', + ); + let stored = JSON.parse( + readFileSync( + realmFile(`${sidekick.data.id.slice(testRealmHref.length)}.json`), + 'utf8', + ), + ); + assert.strictEqual( + new URL( + stored.data.relationships.friend.links.self, + `${sidekick.data.id}.json`, + ).href, + author.data.id, + 'the second card links to the one the first entry minted', + ); + }); - test('a line is appended to a file whose extension the realm does not register', async function (assert) { - let response = await post( - envelope( - invoke('appendLine', { - href: '/telemetry.log', - data: { line: 'deployed' }, - }), - ), - ); + test('a delete answers with no state and removes the file', async function (assert) { + let response = await post( + envelope(invoke('delete', { href: '/report-deleted' })), + ); - assert.strictEqual(response.status, 200, 'HTTP 200 status'); - assert.strictEqual( - readFileSync(realmFile('telemetry.log'), 'utf8'), - 'boot\ndeployed\n', - 'the line is on the end of the file', - ); - }); + assert.strictEqual(response.status, 200, 'HTTP 200 status'); + assert.deepEqual( + response.body['atomic:results'], + [{ data: null }], + 'a delete leaves no state to describe', + ); + assert.false( + existsSync(realmFile('report-deleted.json')), + 'the card is gone from disk', + ); + }); - test('a read answers with the document, and a write on a file is refused', async function (assert) { - let read = await query(envelope(invoke('read', { href: '/notes.md' }))); + test('an href relative to the realm names a card inside it', async function (assert) { + let response = await post( + envelope(invoke('escalate', { href: '/report-relative' })), + ); - assert.strictEqual(read.status, 200, 'HTTP 200 status'); - assert.strictEqual( - read.body['atomic:results'][0].data.type, - 'file-meta', - 'a file reads as its metadata document', - ); + assert.strictEqual(response.status, 200, 'HTTP 200 status'); + assert.strictEqual( + response.body['atomic:results'][0].data.id, + `${testRealmHref}report-relative`, + 'a leading slash names the realm root rather than the origin', + ); + }); - let write = await post( - envelope(invoke('transform', { href: '/notes.md' })), - ); - assert.strictEqual(write.status, 405, 'HTTP 405 status'); - assert.strictEqual( - write.body.errors[0].code, - 'operation-not-allowed', - 'a file carries no transform', - ); - }); - }); + test('a line is appended to a file whose extension the realm does not register', async function (assert) { + let response = await post( + envelope( + invoke('appendLine', { + href: '/telemetry.log', + data: { line: 'deployed' }, + }), + ), + ); - module('atomicity', function () { - test('a failing entry leaves the batch unwritten, unindexed and unannounced', async function (assert) { - let jobsBefore = await indexJobIds(); - let since = Date.now(); - - let response = await post( - envelope( - invoke('read', { href: '/report-kept' }), - invoke('escalate', { href: '/report-rollback' }), - invoke('delete', { href: '/does-not-exist' }), - ), - ); + assert.strictEqual(response.status, 200, 'HTTP 200 status'); + assert.strictEqual( + readFileSync(realmFile('telemetry.log'), 'utf8'), + 'boot\ndeployed\n', + 'the line is on the end of the file', + ); + }); - assert.strictEqual(response.status, 404, 'HTTP 404 status'); - assert.strictEqual( - response.body.errors[0].meta.entry, - 2, - 'the error names the entry the caller sent, not the position it took ' + - 'among the entries that write', - ); - assert.strictEqual( - storedCard('report-rollback.json').data.attributes?.status, - 'open', - 'the entry that could have been carried out wrote nothing', - ); - assert.deepEqual( - await indexJobIds(), - jobsBefore, - 'no index job was enqueued', - ); - assert.deepEqual( - await incrementalIndexEventsSince(since), - [], - 'no index event was broadcast', - ); - }); + test('a read answers with the document, and a write on a file is refused', async function (assert) { + let read = await query(envelope(invoke('read', { href: '/notes.md' }))); - test('a read in a mixed batch answers with the pre-batch document', async function (assert) { - let response = await post( - envelope( - invoke('read', { href: '/report-mixed' }), - invoke('escalate', { href: '/report-mixed' }), - invoke('read', { href: '/notes.md' }), - ), - ); + assert.strictEqual(read.status, 200, 'HTTP 200 status'); + assert.strictEqual( + read.body['atomic:results'][0].data.type, + 'file-meta', + 'a file reads as its metadata document', + ); - assert.strictEqual(response.status, 200, 'HTTP 200 status'); - let [before, write, file] = response.body['atomic:results']; - assert.strictEqual( - before.data.attributes.status, - 'open', - 'the read answers with the state the batch started from', - ); - assert.strictEqual(write.data.id, `${testRealmHref}report-mixed`); - assert.strictEqual( - file.data.type, - 'file-meta', - 'a file read sits alongside card writes', - ); - assert.strictEqual( - storedCard('report-mixed.json').data.attributes?.status, - 'escalated', - 'the write in the same batch landed', - ); + let write = await post( + envelope(invoke('transform', { href: '/notes.md' })), + ); + assert.strictEqual(write.status, 405, 'HTTP 405 status'); + assert.strictEqual( + write.body.errors[0].code, + 'operation-not-allowed', + 'a file carries no transform', + ); + }); }); - test('an answer is never cached', async function (assert) { - let written = await post( - envelope(invoke('escalate', { href: '/report-uncached' })), - ); - assert.strictEqual(written.status, 200, 'HTTP 200 status'); - assert.strictEqual(written.get('Cache-Control'), 'no-store'); - assert.strictEqual(written.get('ETag'), undefined); - assert.strictEqual(written.get('Content-Type'), OPERATIONS); + module('atomicity', function () { + test('a failing entry leaves the batch unwritten, unindexed and unannounced', async function (assert) { + let jobsBefore = await indexJobIds(); + let since = Date.now(); - let read = await query( - envelope(invoke('read', { href: '/report-uncached' })), - ); - assert.strictEqual(read.status, 200, 'HTTP 200 status'); - assert.strictEqual(read.get('Cache-Control'), 'no-store'); - assert.strictEqual(read.get('ETag'), undefined); - }); - }); + let response = await post( + envelope( + invoke('read', { href: '/report-kept' }), + invoke('escalate', { href: '/report-rollback' }), + invoke('delete', { href: '/does-not-exist' }), + ), + ); - module('identity', function () { - test('an operation that reads the actor refuses a request that authenticated nobody', async function (assert) { - let response = await post( - envelope( - invoke('addComment', { - href: '/report-anonymous', - data: { body: 'Who said this?' }, - }), - ), - ); + assert.strictEqual(response.status, 404, 'HTTP 404 status'); + assert.strictEqual( + response.body.errors[0].meta.entry, + 2, + 'the error names the entry the caller sent, not the position it took ' + + 'among the entries that write', + ); + assert.strictEqual( + storedCard('report-rollback.json').data.attributes?.status, + 'open', + 'the entry that could have been carried out wrote nothing', + ); + assert.deepEqual( + await indexJobIds(), + jobsBefore, + 'no index job was enqueued', + ); + assert.deepEqual( + await incrementalIndexEventsSince(since), + [], + 'no index event was broadcast', + ); + }); - assert.strictEqual(response.status, 401, 'HTTP 401 status'); - let [error] = response.body.errors; - assert.strictEqual(error.code, 'actor-required'); - assert.strictEqual(error.meta.entry, 0); - assert.deepEqual( - storedCard('report-anonymous.json').data.attributes?.comments, - [], - 'nothing was written for a caller with no identity', - ); - }); + test('a read in a mixed batch answers with the pre-batch document', async function (assert) { + let response = await post( + envelope( + invoke('read', { href: '/report-mixed' }), + invoke('escalate', { href: '/report-mixed' }), + invoke('read', { href: '/notes.md' }), + ), + ); - test('an operation that reads no actor is carried out for an anonymous caller', async function (assert) { - let response = await post( - envelope(invoke('escalate', { href: '/report-anonymous' })), - ); + assert.strictEqual(response.status, 200, 'HTTP 200 status'); + let [before, write, file] = response.body['atomic:results']; + assert.strictEqual( + before.data.attributes.status, + 'open', + 'the read answers with the state the batch started from', + ); + assert.strictEqual(write.data.id, `${testRealmHref}report-mixed`); + assert.strictEqual( + file.data.type, + 'file-meta', + 'a file read sits alongside card writes', + ); + assert.strictEqual( + storedCard('report-mixed.json').data.attributes?.status, + 'escalated', + 'the write in the same batch landed', + ); + }); - assert.strictEqual(response.status, 200, 'HTTP 200 status'); - assert.strictEqual( - storedCard('report-anonymous.json').data.attributes?.status, - 'escalated', - 'a realm anyone may write carries out a batch that needs no identity', - ); + test('an answer is never cached', async function (assert) { + let written = await post( + envelope(invoke('escalate', { href: '/report-uncached' })), + ); + assert.strictEqual(written.status, 200, 'HTTP 200 status'); + assert.strictEqual(written.get('Cache-Control'), 'no-store'); + assert.strictEqual(written.get('ETag'), undefined); + assert.strictEqual(written.get('Content-Type'), OPERATIONS); + + let read = await query( + envelope(invoke('read', { href: '/report-uncached' })), + ); + assert.strictEqual(read.status, 200, 'HTTP 200 status'); + assert.strictEqual(read.get('Cache-Control'), 'no-store'); + assert.strictEqual(read.get('ETag'), undefined); + }); }); - test('the actor an operation reads is the user the realm authenticated', async function (assert) { - let response = await request - .post('/_operations') - .set('Accept', OPERATIONS) - .set('Content-Type', OPERATIONS) - .set( - 'Authorization', - `Bearer ${createJWT(realm, '@tester:localhost', ['read', 'write'])}`, - ) - .send( + module('identity', function () { + test('an operation that reads the actor refuses a request that authenticated nobody', async function (assert) { + let response = await post( envelope( invoke('addComment', { - href: '/report-identified', - data: { body: 'Reviewed.' }, + href: '/report-anonymous', + data: { body: 'Who said this?' }, }), ), ); - assert.strictEqual(response.status, 200, 'HTTP 200 status'); - assert.deepEqual( - storedCard('report-identified.json').data.attributes?.comments, - [{ body: 'Reviewed.', postedBy: '@tester:localhost' }], - 'the comment records the caller the realm verified', - ); + assert.strictEqual(response.status, 401, 'HTTP 401 status'); + let [error] = response.body.errors; + assert.strictEqual(error.code, 'actor-required'); + assert.strictEqual(error.meta.entry, 0); + assert.deepEqual( + storedCard('report-anonymous.json').data.attributes?.comments, + [], + 'nothing was written for a caller with no identity', + ); + }); + + test('an operation that reads no actor is carried out for an anonymous caller', async function (assert) { + let response = await post( + envelope(invoke('escalate', { href: '/report-anonymous' })), + ); + + assert.strictEqual(response.status, 200, 'HTTP 200 status'); + assert.strictEqual( + storedCard('report-anonymous.json').data.attributes?.status, + 'escalated', + 'a realm anyone may write carries out a batch that needs no identity', + ); + }); + + test('the actor an operation reads is the user the realm authenticated', async function (assert) { + let response = await request + .post('/_operations') + .set('Accept', OPERATIONS) + .set('Content-Type', OPERATIONS) + .set( + 'Authorization', + `Bearer ${createJWT(realm, '@tester:localhost', ['read', 'write'])}`, + ) + .send( + envelope( + invoke('addComment', { + href: '/report-identified', + data: { body: 'Reviewed.' }, + }), + ), + ); + + assert.strictEqual(response.status, 200, 'HTTP 200 status'); + assert.deepEqual( + storedCard('report-identified.json').data.attributes?.comments, + [{ body: 'Reviewed.', postedBy: '@tester:localhost' }], + 'the comment records the caller the realm verified', + ); + }); }); }); -}); -module(`${basename(import.meta.filename)} > read-only realm`, function (hooks) { - let realm: Realm; - let request: RealmRequest; + module('a realm anyone may only read', function (hooks) { + let realm: Realm; + let request: RealmRequest; - setupPermissionedRealmCached(hooks, { - mode: 'before', - realmURL: readOnlyRealm, - permissions: { - reader: ['read'], - '@node-test_realm:localhost': ['read', 'realm-owner'], - }, - fileSystem: { - 'person.gts': ` + setupPermissionedRealmCached(hooks, { + mode: 'before', + realmURL: readOnlyRealm, + permissions: { + reader: ['read'], + '@node-test_realm:localhost': ['read', 'realm-owner'], + }, + fileSystem: { + 'person.gts': ` import { contains, field, CardDef, Component } from "@cardstack/base/card-api"; import StringField from "@cardstack/base/string"; @@ -799,59 +802,60 @@ module(`${basename(import.meta.filename)} > read-only realm`, function (hooks) { } } `, - 'person-1.json': { - data: { - type: 'card', - attributes: { firstName: 'Mango' }, - meta: { - adoptsFrom: { - module: rri(`${readOnlyRealm.href}person`), - name: 'Person', + 'person-1.json': { + data: { + type: 'card', + attributes: { firstName: 'Mango' }, + meta: { + adoptsFrom: { + module: rri(`${readOnlyRealm.href}person`), + name: 'Person', + }, }, }, }, }, - }, - onRealmSetup(args) { - realm = args.testRealm; - request = withRealmPath(args.request, readOnlyRealm); - }, - }); - - // The permission is derived from the method before a body is read, so these - // two send the same batch and differ only in how it is sent. - let batch = () => - JSON.stringify({ - 'boxel:operations': [ - { op: 'invoke', 'boxel:name': 'read', href: '/person-1' }, - ], + onRealmSetup(args) { + realm = args.testRealm; + request = withRealmPath(args.request, readOnlyRealm); + }, }); - test('a POST needs realm write', async function (assert) { - let response = await request - .post('/_operations') - .set('Accept', OPERATIONS) - .set('Content-Type', OPERATIONS) - .set('Authorization', `Bearer ${createJWT(realm, 'reader', ['read'])}`) - .send(batch()); + // The permission is derived from the method before a body is read, so these + // two send the same batch and differ only in how it is sent. + let batch = () => + JSON.stringify({ + 'boxel:operations': [ + { op: 'invoke', 'boxel:name': 'read', href: '/person-1' }, + ], + }); - assert.strictEqual(response.status, 403, 'HTTP 403 status'); - }); + test('a POST needs realm write', async function (assert) { + let response = await request + .post('/_operations') + .set('Accept', OPERATIONS) + .set('Content-Type', OPERATIONS) + .set('Authorization', `Bearer ${createJWT(realm, 'reader', ['read'])}`) + .send(batch()); - test('a QUERY needs only realm read', async function (assert) { - let response = await request - .post('/_operations') - .set('Accept', OPERATIONS) - .set('Content-Type', OPERATIONS) - .set('X-HTTP-Method-Override', 'QUERY') - .set('Authorization', `Bearer ${createJWT(realm, 'reader', ['read'])}`) - .send(batch()); - - assert.strictEqual(response.status, 200, 'HTTP 200 status'); - assert.strictEqual( - response.body['atomic:results'][0].data.attributes.firstName, - 'Mango', - 'the read a reader is authorized for is carried out', - ); + assert.strictEqual(response.status, 403, 'HTTP 403 status'); + }); + + test('a QUERY needs only realm read', async function (assert) { + let response = await request + .post('/_operations') + .set('Accept', OPERATIONS) + .set('Content-Type', OPERATIONS) + .set('X-HTTP-Method-Override', 'QUERY') + .set('Authorization', `Bearer ${createJWT(realm, 'reader', ['read'])}`) + .send(batch()); + + assert.strictEqual(response.status, 200, 'HTTP 200 status'); + assert.strictEqual( + response.body['atomic:results'][0].data.attributes.firstName, + 'Mango', + 'the read a reader is authorized for is carried out', + ); + }); }); }); From 66f07b517a93e4280b2d5ea4ad97a1f0314a4e7e Mon Sep 17 00:00:00 2001 From: Hassan Abdel-Rahman Date: Wed, 16 Sep 2026 18:46:47 -0400 Subject: [PATCH 05/13] Exercise the identity cases as the anonymous requests they describe Co-Authored-By: Claude Opus 5 (1M context) --- .../tests/realm-endpoints/operations-test.ts | 41 ++++++++----------- 1 file changed, 17 insertions(+), 24 deletions(-) diff --git a/packages/realm-server/tests/realm-endpoints/operations-test.ts b/packages/realm-server/tests/realm-endpoints/operations-test.ts index 9fa1fbda175..5db14eef419 100644 --- a/packages/realm-server/tests/realm-endpoints/operations-test.ts +++ b/packages/realm-server/tests/realm-endpoints/operations-test.ts @@ -107,7 +107,7 @@ function makeFileSystem(): Record { 'report.gts': ` import { contains, containsMany, field, linksTo, CardDef, FieldDef, Component } from "@cardstack/base/card-api"; import StringField from "@cardstack/base/string"; - import { operation, params, actor, instance } from "@cardstack/base/operations"; + import { operation, params, actor } from "@cardstack/base/operations"; import { Person } from "./person"; export class ReportComment extends FieldDef { @@ -135,9 +135,9 @@ function makeFileSystem(): Record { }, }; - @operation static openReports = { + @operation static knownReviewers = { base: 'query', - query: { filter: { on: ExternalReport, eq: { status: 'open' } } }, + query: { filter: { on: Person, eq: { firstName: 'Reviewer' } } }, }; static isolated = class Isolated extends Component { @@ -334,7 +334,7 @@ module(`realm-endpoints/${basename(import.meta.filename)}`, function () { test('an entry naming a query is sent to the search engine', async function (assert) { let response = await query( - envelope(invoke('openReports', { href: '/report-kept' })), + envelope(invoke('knownReviewers', { href: '/report-kept' })), ); assert.strictEqual(response.status, 400, 'HTTP 400 status'); @@ -712,7 +712,7 @@ module(`realm-endpoints/${basename(import.meta.filename)}`, function () { module('identity', function () { test('an operation that reads the actor refuses a request that authenticated nobody', async function (assert) { - let response = await post( + let response = await anonymousPost( envelope( invoke('addComment', { href: '/report-anonymous', @@ -733,7 +733,7 @@ module(`realm-endpoints/${basename(import.meta.filename)}`, function () { }); test('an operation that reads no actor is carried out for an anonymous caller', async function (assert) { - let response = await post( + let response = await anonymousPost( envelope(invoke('escalate', { href: '/report-anonymous' })), ); @@ -746,28 +746,21 @@ module(`realm-endpoints/${basename(import.meta.filename)}`, function () { }); test('the actor an operation reads is the user the realm authenticated', async function (assert) { - let response = await request - .post('/_operations') - .set('Accept', OPERATIONS) - .set('Content-Type', OPERATIONS) - .set( - 'Authorization', - `Bearer ${createJWT(realm, '@tester:localhost', ['read', 'write'])}`, - ) - .send( - envelope( - invoke('addComment', { - href: '/report-identified', - data: { body: 'Reviewed.' }, - }), - ), - ); + let response = await post( + envelope( + invoke('addComment', { + href: '/report-identified', + data: { body: 'Reviewed.' }, + }), + ), + ); assert.strictEqual(response.status, 200, 'HTTP 200 status'); assert.deepEqual( storedCard('report-identified.json').data.attributes?.comments, - [{ body: 'Reviewed.', postedBy: '@tester:localhost' }], - 'the comment records the caller the realm verified', + [{ body: 'Reviewed.', postedBy: TESTER }], + 'the comment records the caller the realm verified, and no other ' + + 'identity is invented for it', ); }); }); From 61ca7102ae799f469f517ab27e793d21f909c62c Mon Sep 17 00:00:00 2001 From: Hassan Abdel-Rahman Date: Wed, 16 Sep 2026 18:50:29 -0400 Subject: [PATCH 06/13] Let a file-content write reach the executor that can tell a card from bytes Co-Authored-By: Claude Opus 5 (1M context) --- .../tests/card-operations-dispatch-test.ts | 26 +++++++++++++------ 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/packages/realm-server/tests/card-operations-dispatch-test.ts b/packages/realm-server/tests/card-operations-dispatch-test.ts index aa7ef816904..0734478674c 100644 --- a/packages/realm-server/tests/card-operations-dispatch-test.ts +++ b/packages/realm-server/tests/card-operations-dispatch-test.ts @@ -480,9 +480,7 @@ module(basename(import.meta.filename), function () { test('a file def carries the two writes that work on its bytes', async function (assert) { // A file's metadata is content-derived and read-only, so what a write on // one reaches is the bytes: an `update` replaces them wholesale, and an - // `appendLine` adds a line to the end of a text file. Appending a line is - // the one behavior that goes the other way — a line appended to a card's - // stored file leaves behind something that is no longer a card. + // `appendLine` adds a line to the end of a text file. let file = stub(); for (let name of ['update', 'appendLine']) { let resolved = await resolveOperation(file.core, FILE, name); @@ -492,13 +490,25 @@ module(basename(import.meta.filename), function () { `a file carries "${name}", undeclared, as a base operation`, ); } - + }); + test('appending a line is admitted for any instance, whatever its URL says', async function (assert) { + // A line appended to a card's stored file leaves behind something that is + // no longer a card, so this is the one write a card must not carry — and + // it is refused by `stageAppendLine`, not here. Dispatch classifies an + // instance target by its extension, and the registered-extension table + // does not name every stored file: a `.log`, a `.css`, a `.yml` holds + // bytes and serves them, and each classifies as a card. Refusing here + // would turn those away before the only code that can tell them from a + // card — the executor, which reads whether the path holds a card's + // `.json` and what content type its bytes are. let card = stub(); - let onCard = await refusalFrom(() => - resolveOperation(card.core, CARD, 'appendLine'), + let resolved = await resolveOperation(card.core, CARD, 'appendLine'); + assert.strictEqual( + resolved.base, + 'appendLine', + 'the behavior resolves, and what it may be applied to is the ' + + "executor's to decide", ); - assert.strictEqual(onCard.code, 'operation-not-allowed'); - assert.strictEqual(onCard.status, 405); }); test('a write is carried out by the coordinator rather than by this dispatch', async function (assert) { // Every write takes the realm's write lock once for the whole batch it From 9a947904e631508cc42020880f8e41c9552c2a5f Mon Sep 17 00:00:00 2001 From: Hassan Abdel-Rahman Date: Wed, 16 Sep 2026 18:51:45 -0400 Subject: [PATCH 07/13] Say which type a class-scoped entry names, and who resolves it Co-Authored-By: Claude Opus 5 (1M context) --- .../runtime-common/card-operations/envelope.ts | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/packages/runtime-common/card-operations/envelope.ts b/packages/runtime-common/card-operations/envelope.ts index 634b774a4fb..9a872415a64 100644 --- a/packages/runtime-common/card-operations/envelope.ts +++ b/packages/runtime-common/card-operations/envelope.ts @@ -225,9 +225,19 @@ function hrefIn(href: unknown, index: number, paths: RealmPaths): string { return absolute.href; } -// What the entry runs against. An entry naming an href targets that resource; -// one naming none is creating a card that does not exist yet, and names the -// type it mints the same way a card's stored JSON names its own. +// What the entry runs against. +// +// An entry naming an href targets that resource. One naming none is scoped to +// a type rather than to an instance — a create, which has no existing card to +// bind to — and names that type in `data.meta.adoptsFrom`, the same member a +// card's stored JSON names its own type in. The type named is the one whose +// operations are being invoked; what a named create actually mints is its +// declaration's, which the realm reads from the definition rather than from +// the wire. +// +// The ref is handed on unchecked. Resolving one is the realm's, and a ref that +// names nothing comes back from the lookup as a type that cannot be resolved, +// which is the answer a caller needs either way. export function targetFor( entry: EnvelopeEntry, realmURL: string, From 94d6158680059011b4cbf6380d0f99e28dbacb3c Mon Sep 17 00:00:00 2001 From: Hassan Abdel-Rahman Date: Wed, 16 Sep 2026 19:25:21 -0400 Subject: [PATCH 08/13] Ask the actor question where the declaration is, and keep bxl out of the realm's graph Whether an operation reads the invoking actor is a property of its declaration, so lowering records it next to `deterministic` and the envelope reads a boolean. That also covers every template a marker can hide in rather than the four members one function remembered, and it keeps the BXL package out of the typecheck program of everything that reaches the realm. Co-Authored-By: Claude Opus 5 (1M context) --- .../tests/realm-endpoints/operations-test.ts | 160 ++++++++++++- .../card-operations/coordinator.ts | 135 +++++++---- .../card-operations/envelope.ts | 212 +++++++++++------- .../card-operations/executors.ts | 8 + .../runtime-common/card-operations/index.ts | 1 - .../card-operations/lowering.ts | 37 +++ .../runtime-common/card-operations/types.ts | 12 + packages/runtime-common/index.ts | 6 +- packages/runtime-common/realm.ts | 56 +++-- packages/runtime-common/router.ts | 78 +++++++ 10 files changed, 542 insertions(+), 163 deletions(-) diff --git a/packages/realm-server/tests/realm-endpoints/operations-test.ts b/packages/realm-server/tests/realm-endpoints/operations-test.ts index 5db14eef419..ca80890aeea 100644 --- a/packages/realm-server/tests/realm-endpoints/operations-test.ts +++ b/packages/realm-server/tests/realm-endpoints/operations-test.ts @@ -7,7 +7,11 @@ import type { Test, SuperTest } from 'supertest'; import type { DirResult } from 'tmp'; import type { PgAdapter } from '@cardstack/postgres'; -import { rri, SupportedMimeType } from '@cardstack/runtime-common'; +import { + BOXEL_OPERATIONS_EXT, + rri, + SupportedMimeType, +} from '@cardstack/runtime-common'; import type { DBAdapter, LooseSingleCardDocument, @@ -107,7 +111,7 @@ function makeFileSystem(): Record { 'report.gts': ` import { contains, containsMany, field, linksTo, CardDef, FieldDef, Component } from "@cardstack/base/card-api"; import StringField from "@cardstack/base/string"; - import { operation, params, actor } from "@cardstack/base/operations"; + import { operation, params, actor, bxl } from "@cardstack/base/operations"; import { Person } from "./person"; export class ReportComment extends FieldDef { @@ -135,6 +139,13 @@ function makeFileSystem(): Record { }, }; + @operation static restate = { + base: 'transform', + params: { headline: StringField }, + input: bxl`{headline: params("headline")}`, + set: { headline: params('headline') }, + }; + @operation static knownReviewers = { base: 'query', query: { filter: { on: Person, eq: { firstName: 'Reviewer' } } }, @@ -171,9 +182,14 @@ function makeFileSystem(): Record { 'report-deleted', 'report-relative', 'report-uncached', + 'report-canonical', + 'report-query', + 'report-unserved', + 'report-position', ].map((name) => [`${name}.json`, reportFile()]), ), 'notes.md': '# Notes\n', + 'positions.log': 'boot\n', // A stored file the registered-extension table does not name, so its URL // classifies as a card and the executor is what tells the two apart. 'telemetry.log': 'boot\n', @@ -316,7 +332,33 @@ module(`realm-endpoints/${basename(import.meta.filename)}`, function () { assert.strictEqual(error.meta.entry, 0); }); - test('a body sent without the operations extension is told what it is missing', async function (assert) { + test('every spelling of the operations media type reaches the endpoint', async function (assert) { + // The router matches a media type by its type and its parameters, so the + // extension is named the same operation whichever way a client writes + // it. Sent as the content type alone in each case — a `fetch` that sets + // one and leaves `Accept` to its default is the shape that has no other + // header to fall back on. + for (let spelling of [ + `application/vnd.api+json;ext="${BOXEL_OPERATIONS_EXT}"`, + `application/vnd.api+json; ext="${BOXEL_OPERATIONS_EXT}"`, + `application/vnd.api+json;ext=${BOXEL_OPERATIONS_EXT}`, + `application/vnd.api+json;ext="https://jsonapi.org/ext/atomic ${BOXEL_OPERATIONS_EXT}"`, + `application/vnd.api+json;charset=utf-8;ext="${BOXEL_OPERATIONS_EXT}"`, + ]) { + let response = await request + .post('/_operations') + .set('Content-Type', spelling) + .set( + 'Authorization', + `Bearer ${createJWT(realm, TESTER, ['read', 'write'])}`, + ) + .send(envelope(invoke('read', { href: '/report-kept' }))); + + assert.strictEqual(response.status, 200, `${spelling} is carried out`); + } + }); + + test('a body sent without the operations extension is told what it is missing', async function (assert) { let response = await request .post('/_operations') .set('Accept', SupportedMimeType.JSONAPI) @@ -406,7 +448,50 @@ module(`realm-endpoints/${basename(import.meta.filename)}`, function () { ); }); - test('two entries claiming one local id are refused', async function (assert) { + test('a type reference that is not one is refused rather than thrown out of', async function (assert) { + let response = await post( + envelope( + invoke('create', { + data: { + type: 'card', + attributes: { firstName: 'Mango' }, + meta: { adoptsFrom: { type: 'fieldOf', card: null, field: 'x' } }, + }, + }), + ), + ); + + assert.strictEqual( + response.status, + 400, + 'a malformed reference is the caller\'s to fix, not a fault to report', + ); + assert.strictEqual(response.body.errors[0].meta.entry, 0); + }); + + test('an operation whose declaration carries a stage a batch does not run is refused', async function (assert) { + let response = await post( + envelope( + invoke('restate', { + href: '/report-unserved', + data: { headline: 'Revised' }, + }), + ), + ); + + assert.strictEqual(response.status, 501, 'HTTP 501 status'); + assert.true( + response.body.errors[0].detail.includes('input'), + `the refusal names the stage: ${response.body.errors[0].detail}`, + ); + assert.strictEqual( + storedCard('report-unserved.json').data.attributes?.headline, + 'Quarterly Review', + 'and nothing was written under a declaration half carried out', + ); + }); + + test('two entries claiming one local id are refused', async function (assert) { let response = await post( envelope( invoke('create', { @@ -556,7 +641,33 @@ module(`realm-endpoints/${basename(import.meta.filename)}`, function () { ); }); - test('a delete answers with no state and removes the file', async function (assert) { + test('an equivalent spelling of an href names the same card', async function (assert) { + // The index is read by exact URL, so a declared operation resolves only + // once the spelling has been folded to the one the realm addresses the + // card by — and the identity the entry answers with is that one too. + for (let [spelling, what] of [ + ['/report-canonical?view=full', 'a query string'], + ['/report-canonical#section', 'a fragment'], + ]) { + let response = await post( + envelope(invoke('escalate', { href: spelling })), + ); + + assert.strictEqual(response.status, 200, `${what} is carried out`); + assert.strictEqual( + response.body['atomic:results'][0].data.id, + `${testRealmHref}report-canonical`, + `${what} answers with the id the realm serves the card under`, + ); + } + assert.strictEqual( + storedCard('report-canonical.json').data.attributes?.status, + 'escalated', + 'and the card the spellings name is the one that changed', + ); + }); + + test('a delete answers with no state and removes the file', async function (assert) { let response = await post( envelope(invoke('delete', { href: '/report-deleted' })), ); @@ -663,7 +774,44 @@ module(`realm-endpoints/${basename(import.meta.filename)}`, function () { ); }); - test('a read in a mixed batch answers with the pre-batch document', async function (assert) { + test('a refusal names the entry the caller sent, in its key and in its prose', async function (assert) { + // The coordinator is handed only the entries that write, so its own + // numbering runs 0,1 where the caller sent 1,2. Both the keys and the + // sentence have to speak the caller's. + let response = await post( + envelope( + invoke('read', { href: '/report-kept' }), + invoke('appendLine', { + href: '/positions.log', + data: { line: 'deployed' }, + }), + invoke('update', { + href: '/positions.log', + data: { content: 'replaced\n' }, + }), + ), + ); + + assert.strictEqual(response.status, 400, 'HTTP 400 status'); + let [error] = response.body.errors; + assert.strictEqual(error.meta.entry, 2, 'the entry that collides'); + assert.strictEqual( + error.meta.conflictsWith, + 1, + 'and the entry it collides with', + ); + assert.true( + error.detail.includes('entry 2') && error.detail.includes('entry 1'), + `the prose names the same two entries: ${error.detail}`, + ); + assert.strictEqual( + readFileSync(realmFile('positions.log'), 'utf8'), + 'boot\n', + 'and nothing was written', + ); + }); + + test('a read in a mixed batch answers with the pre-batch document', async function (assert) { let response = await post( envelope( invoke('read', { href: '/report-mixed' }), diff --git a/packages/runtime-common/card-operations/coordinator.ts b/packages/runtime-common/card-operations/coordinator.ts index 8cc94209182..75ce85c2f16 100644 --- a/packages/runtime-common/card-operations/coordinator.ts +++ b/packages/runtime-common/card-operations/coordinator.ts @@ -252,7 +252,8 @@ export async function commitBatch( // created card's file is named after its `lid`, so its URL is path math // over the type it adopts — no read and no write — which is what lets an // entry link to a card a later entry mints. - let { lids, foreignLids } = indexLids(entries, paths); + let positions = positionsOf(entries); + let { lids, foreignLids } = indexLids(entries, positions, paths); let { stored, storedMeta } = await readPreState(core, entries, paths); // What an append stages for a file it never read whole. Kept beside // `stored` rather than in it: the two describe the same file in different @@ -265,7 +266,7 @@ export async function commitBatch( // entry to touch a file merged over. let baseHashes: (string | undefined)[] = []; for (let [index, entry] of entries.entries()) { - let change = await stageEntry(entry, index, { + let change = await stageEntry(entry, positions[index], { realmURL: core.realmURL, paths, lids, @@ -293,17 +294,35 @@ export async function commitBatch( compose(stored, storedMeta, splices, entry, change); staged.push(change); } - assertWritesAllowed(staged); - assertLinkedCardsSurvive(staged, paths); - assertWritesFit(core, staged); - await assertRemovalsAllowed(core, paths, staged); - await assertDestinationsFree(core, staged); + assertWritesAllowed(staged, positions); + assertLinkedCardsSurvive(staged, paths, positions); + assertWritesFit(core, staged, positions); + await assertRemovalsAllowed(core, paths, staged, positions); + await assertDestinationsFree(core, staged, positions); // Everything above either produced bytes for every entry or threw, and a // throw leaves the realm as it was. - return await commitStaged(core, entries, staged, baseHashes, opts); + return await commitStaged( + core, + entries, + staged, + baseHashes, + positions, + opts, + ); }); } +// The position each entry is reported under, in batch order. +// +// An entry that names its own is reported under that one everywhere a position +// appears — the key on a refusal, the key beside it naming the entry it +// collides with, and the prose — so the three cannot disagree with each other. +// An entry that names none is reported under its position in this batch, which +// is what a caller that sent exactly this list means by it. +function positionsOf(entries: readonly BatchEntry[]): number[] { + return entries.map((entry, index) => entry.label ?? index); +} + // Fold what an entry staged into the state the next entry stages against. A // batch is a sequence, so two entries may name one card and the second is // meant to build on the first: it merges over the bytes the first staged, not @@ -377,11 +396,11 @@ function compose( // which of the entries it sent produced it. async function stageEntry( entry: BatchEntry, - index: number, + position: number, ctx: StagingContext, ): Promise { try { - assertVersionable(entry, index); + assertVersionable(entry, position); switch (entry.op) { case 'create': return await stageCreate(entry, ctx); @@ -400,11 +419,11 @@ async function stageEntry( status: 400, code: 'invalid-params', title: 'Unknown entry', - detail: `entry ${index} names no staged operation`, + detail: `entry ${position} names no staged operation`, }); } } catch (err: unknown) { - throw atEntry(err, index); + throw atEntry(err, position); } } @@ -414,7 +433,7 @@ async function stageEntry( // transform plans a program against it. A create has no prior state to name, a // delete's result carries no state to report a match on, and an append never // reads the state it edits, so it has nothing to compare one against. -function assertVersionable(entry: BatchEntry, index: number): void { +function assertVersionable(entry: BatchEntry, position: number): void { if ( entry.baseVersion === undefined || entry.op === 'update' || @@ -426,25 +445,25 @@ function assertVersionable(entry: BatchEntry, index: number): void { status: 400, code: 'invalid-params', title: 'Invalid base version', - detail: `entry ${index} is a ${entry.op}, which has no base version`, + detail: `entry ${position} is a ${entry.op}, which has no base version`, }); } -function atEntry(err: unknown, index: number): OperationFailure { +function atEntry(err: unknown, position: number): OperationFailure { if (isOperationFailure(err)) { return new OperationFailure({ ...err.error, - meta: { ...err.error.meta, entry: index }, + meta: { ...err.error.meta, entry: position }, }); } return new OperationFailure({ status: 500, code: 'internal-error', title: 'Cannot stage batch', - detail: `entry ${index} could not be staged: ${ + detail: `entry ${position} could not be staged: ${ err instanceof Error ? err.message : String(err) }`, - meta: { entry: index }, + meta: { entry: position }, }); } @@ -461,6 +480,7 @@ function atEntry(err: unknown, index: number): OperationFailure { // as a link to a card nobody sent. function indexLids( entries: BatchEntry[], + positions: readonly number[], paths: RealmPaths, ): { lids: LidIndex; foreignLids: ReadonlySet } { let lids = new Map(); @@ -504,7 +524,7 @@ function indexLids( claim( primaryLid, createIdentity(entry, primary, paths, new Map()), - `entry ${index}`, + `entry ${positions[index]}`, ); } } @@ -529,11 +549,11 @@ function indexLids( entry.op === 'create' ? entry.directory : undefined, paths, ), - `entry ${index}, included[${offset}]`, + `entry ${positions[index]}, included[${offset}]`, ); } } catch (err: unknown) { - throw atEntry(err, index); + throw atEntry(err, positions[index]); } } return { lids, foreignLids }; @@ -763,7 +783,10 @@ function cardSourcePathOf( // them once for all three. Removals are not covered, deliberately: they are // the recovery path for anything already stored under either name, which is // why the other two admit them as well. -function assertWritesAllowed(staged: StagedChange[]): void { +function assertWritesAllowed( + staged: StagedChange[], + positions: readonly number[], +): void { for (let [index, change] of staged.entries()) { for (let write of [...change.writes, ...change.appends]) { let reserved = isCaptureServingPath(write.path) @@ -780,7 +803,7 @@ function assertWritesAllowed(staged: StagedChange[]): void { title: 'Reserved path', detail: `cannot write "${write.path}": ${reserved}`, }), - index, + positions[index], ); } } @@ -801,6 +824,7 @@ function assertWritesAllowed(staged: StagedChange[]): void { function assertLinkedCardsSurvive( staged: StagedChange[], paths: RealmPaths, + positions: readonly number[], ): void { let removed = new Set(); for (let change of staged) { @@ -828,10 +852,10 @@ function assertLinkedCardsSurvive( code: 'invalid-params', title: 'Conflicting entries', detail: - `entry ${index} links to ${id}, which this batch removes; the ` + - `edge would point at nothing once the batch commits`, + `entry ${positions[index]} links to ${id}, which this batch ` + + `removes; the edge would point at nothing once the batch commits`, }), - index, + positions[index], ); } } @@ -848,6 +872,7 @@ async function assertRemovalsAllowed( core: BatchCore, paths: RealmPaths, staged: StagedChange[], + positions: readonly number[], ): Promise { for (let [index, change] of staged.entries()) { for (let path of change.deletes) { @@ -861,7 +886,7 @@ async function assertRemovalsAllowed( `${paths.fileURL(path).href} is under a path the realm ` + `ignores, so it holds no card to remove`, }), - index, + positions[index], ); } } @@ -874,7 +899,11 @@ async function assertRemovalsAllowed( // it enqueues anything — over a payload the caller could have been told about // while the realm was still untouched. So the ceiling is applied to every // staged write here, where a refusal still costs nothing. -function assertWritesFit(core: BatchCore, staged: StagedChange[]): void { +function assertWritesFit( + core: BatchCore, + staged: StagedChange[], + positions: readonly number[], +): void { for (let [index, change] of staged.entries()) { // An append is held to the ceiling by what it adds rather than by what the // file will hold: the limit is over the bytes a caller hands the realm, @@ -900,7 +929,7 @@ function assertWritesFit(core: BatchCore, staged: StagedChange[]): void { detail: err.message, }) : err, - index, + positions[index], ); } } @@ -925,6 +954,7 @@ function assertWritesFit(core: BatchCore, staged: StagedChange[]): void { async function assertDestinationsFree( core: BatchCore, staged: StagedChange[], + positions: readonly number[], ): Promise { let removedBefore: Set[] = []; let removed = new Set(); @@ -956,7 +986,7 @@ async function assertDestinationsFree( detail: `a card is already stored at ${taken.path}; a create mints a card ` + `rather than replacing one`, - meta: { entry: taken.index }, + meta: { entry: positions[taken.index] }, }); } } @@ -966,6 +996,7 @@ async function commitStaged( entries: BatchEntry[], staged: StagedChange[], baseHashes: (string | undefined)[], + positions: readonly number[], opts: CommitBatchOptions, ): Promise { // One entry per file, in batch order. Two entries may name one card, and @@ -997,10 +1028,13 @@ async function commitStaged( code: 'invalid-params', title: 'Conflicting entries', detail: - `entry ${index} replaces the content of ${write.path}, which ` + - `entry ${appender} appends to; a batch appends to a file after ` + - `it writes one, not before`, - meta: { entry: index, conflictsWith: appender }, + `entry ${positions[index]} replaces the content of ` + + `${write.path}, which entry ${positions[appender]} appends to; a ` + + `batch appends to a file after it writes one, not before`, + meta: { + entry: positions[index], + conflictsWith: positions[appender], + }, }); } if (deleted.has(write.path)) { @@ -1013,9 +1047,10 @@ async function commitStaged( code: 'invalid-params', title: 'Conflicting entries', detail: - `entry ${index} writes ${write.path}, which an earlier entry ` + - `removes; a batch cannot both remove a card and write it`, - meta: { entry: index }, + `entry ${positions[index]} writes ${write.path}, which an ` + + `earlier entry removes; a batch cannot both remove a card and ` + + `write it`, + meta: { entry: positions[index] }, }); } let owner = writtenBy.get(write.path); @@ -1032,10 +1067,10 @@ async function commitStaged( code: 'invalid-params', title: 'Conflicting entries', detail: - `entries ${owner} and ${index} both replace the content of ` + - `${write.path}; a replacement composes over nothing, so only ` + - `one of them could land`, - meta: { entry: index, conflictsWith: owner }, + `entries ${positions[owner]} and ${positions[index]} both ` + + `replace the content of ${write.path}; a replacement composes ` + + `over nothing, so only one of them could land`, + meta: { entry: positions[index], conflictsWith: positions[owner] }, }); } if (owner !== undefined && write.path !== change.primaryPath) { @@ -1049,10 +1084,11 @@ async function commitStaged( code: 'invalid-params', title: 'Conflicting entries', detail: - `entries ${owner} and ${index} both write ${write.path}, and ` + - `entry ${index} carries it as a side-load, which replaces the ` + - `card rather than merging over it`, - meta: { entry: index, conflictsWith: owner }, + `entries ${positions[owner]} and ${positions[index]} both ` + + `write ${write.path}, and entry ${positions[index]} carries it ` + + `as a side-load, which replaces the card rather than merging ` + + `over it`, + meta: { entry: positions[index], conflictsWith: positions[owner] }, }); } writes.set(write.path, write.content); @@ -1071,9 +1107,10 @@ async function commitStaged( code: 'invalid-params', title: 'Conflicting entries', detail: - `entry ${index} appends to ${append.path}, which an earlier ` + - `entry removes; a batch cannot both remove a file and add to it`, - meta: { entry: index }, + `entry ${positions[index]} appends to ${append.path}, which an ` + + `earlier entry removes; a batch cannot both remove a file and ` + + `add to it`, + meta: { entry: positions[index] }, }); } appends.set( @@ -1140,7 +1177,7 @@ async function commitStaged( code: 'internal-error', title: 'Missing write result', detail: `the commit reported no result for ${change.primaryPath}`, - meta: { entry: index }, + meta: { entry: positions[index] }, }); } let { baseVersion } = entries[index]; diff --git a/packages/runtime-common/card-operations/envelope.ts b/packages/runtime-common/card-operations/envelope.ts index 9a872415a64..e011a549cbe 100644 --- a/packages/runtime-common/card-operations/envelope.ts +++ b/packages/runtime-common/card-operations/envelope.ts @@ -1,6 +1,5 @@ import { BOXEL_OPERATIONS_EXT } from '../supported-mime-type.ts'; import { RealmPaths, type LocalPath } from '../paths.ts'; -import { callsActor } from './bxl-emit.ts'; import { OperationFailure, isDocumentResult, @@ -10,11 +9,10 @@ import { type OperationError, type OperationResult, type OperationTarget, - type OperationTemplate, } from './types.ts'; import type { BatchEntryResult } from './coordinator.ts'; import type { BatchEntry } from './executors.ts'; -import type { CodeRef } from '../code-ref.ts'; +import { isCodeRef } from '../card-document-shape.ts'; import type { CardResource } from '../resource-types.ts'; // ============================================================================ @@ -94,9 +92,20 @@ export interface EnvelopeEntry { // one: the coordinator takes no lock and announces nothing for it, so the // envelope answers with no results rather than inventing a refusal for a // request that would change nothing either way. +export interface ParseEnvelopeOptions { + // A registered-prefix identifier as the absolute URL it names, and anything + // else unchanged. Identifier resolution belongs to the realm's fetch layer, + // so it arrives as a bound function rather than being reached from here — + // the same reason the operation core takes its code-ref resolver bound. + // Absent, an href is read exactly as it is written, which is what a caller + // with no prefixes registered would get anyway. + resolveIdentifier?: (href: string) => string; +} + export function parseOperationsEnvelope( body: unknown, realmURL: string, + opts: ParseEnvelopeOptions = {}, ): EnvelopeEntry[] { if (!isPlainRecord(body)) { throw refuse(`the request body is not a JSON:API document`); @@ -109,7 +118,7 @@ export function parseOperationsEnvelope( } let paths = new RealmPaths(new URL(realmURL)); return operations.map((operation, index) => - parseEntry(operation, index, paths), + parseEntry(operation, index, paths, opts), ); } @@ -117,6 +126,7 @@ function parseEntry( operation: unknown, index: number, paths: RealmPaths, + opts: ParseEnvelopeOptions, ): EnvelopeEntry { if (!isPlainRecord(operation)) { throw refuse(`entry ${index} is not an operation`, index); @@ -124,7 +134,7 @@ function parseEntry( let op = operation.op; switch (op) { case 'invoke': - return parseInvocation(operation, index, paths); + return parseInvocation(operation, index, paths, opts); case 'parallel': case 'serial': throw refuse( @@ -146,6 +156,7 @@ function parseInvocation( operation: Record, index: number, paths: RealmPaths, + opts: ParseEnvelopeOptions, ): EnvelopeEntry { let name = operation['boxel:name']; if (typeof name !== 'string' || name.length === 0) { @@ -176,7 +187,7 @@ function parseInvocation( name, ...(operation.href === undefined ? {} - : { href: hrefIn(operation.href, index, paths) }), + : { href: hrefIn(operation.href, index, paths, opts) }), ...(data ? { data } : {}), ...(lid === undefined ? {} : { lid }), }; @@ -194,20 +205,30 @@ function parseInvocation( // this realm is not something the endpoint can carry out rather than something // it declines to: there is no lock it could take that would make the write // atomic with the rest of the batch. -function hrefIn(href: unknown, index: number, paths: RealmPaths): string { +function hrefIn( + href: unknown, + index: number, + paths: RealmPaths, + opts: ParseEnvelopeOptions, +): string { if (typeof href !== 'string' || href.length === 0) { throw refuse(`entry ${index} carries an "href" that is not a URL`, index); } + // A realm reached through a registered prefix is addressed by that prefix + // everywhere else — it is the form a result's id comes back in — so an href + // written that way names the card it appears to name rather than a path + // under the realm root that happens to start with it. + let resolved = opts.resolveIdentifier?.(href) ?? href; let absolute: URL; try { - absolute = new URL(href); + absolute = new URL(resolved); } catch { // Relative, so it names a path within the realm. A leading slash is the // spelling the extension documents and means the realm's root, not the // origin's, which is why this resolves through `fileURL` rather than // through URL resolution against the realm. try { - absolute = paths.fileURL(href.replace(/^\/+/, '') as LocalPath); + absolute = paths.fileURL(resolved.replace(/^\/+/, '') as LocalPath); } catch { throw refuse(`entry ${index} carries an "href" that is not a URL`, index); } @@ -235,9 +256,11 @@ function hrefIn(href: unknown, index: number, paths: RealmPaths): string { // declaration's, which the realm reads from the definition rather than from // the wire. // -// The ref is handed on unchecked. Resolving one is the realm's, and a ref that -// names nothing comes back from the lookup as a type that cannot be resolved, -// which is the answer a caller needs either way. +// The ref's shape is checked here, recursively. Whether the type it names +// exists is the lookup's answer and comes back as a type that cannot be +// resolved; whether the thing is a code ref at all is not, and an object that +// is not one reaches the resolver's `'type' in ref` recursion and throws out of +// it — a malformed payload answered as a fault in the realm. export function targetFor( entry: EnvelopeEntry, realmURL: string, @@ -253,11 +276,14 @@ export function targetFor( entry.index, ); } - return { - kind: 'type', - codeRef: adoptsFrom as unknown as CodeRef, - realm: realmURL, - }; + if (!isCodeRef(adoptsFrom)) { + throw refuse( + `entry ${entry.index} names a type in "data.meta.adoptsFrom" that is ` + + `not a code reference`, + entry.index, + ); + } + return { kind: 'type', codeRef: adoptsFrom, realm: realmURL }; } // One entry with the behavior its name resolved to. The name is the whole of @@ -323,36 +349,19 @@ export function assertTravelsInEnvelope( // Whether carrying this operation out needs to know who the caller is. // -// Decidable before anything runs, because every place an operation can read -// the actor is part of its stored definition: the programs it runs, and the -// template a named create fills. That is what makes it worth asking here — -// an anonymous caller on a realm that lets anyone write is told its request -// needs an identity, once for the whole batch, rather than having an entry -// refuse part-way through for a reason that reads as a payload problem. +// Read off the stored definition, where lowering recorded it with the whole +// declaration in hand. Asking it here would mean reading program text, which +// would put this module — and so every package that reaches the realm through +// it — in the BXL package's typecheck program; and it would mean keeping a +// list of the members a marker can hide in, which is the list lowering already +// walks. +// +// The point of asking at all is that it is answerable before anything runs: an +// anonymous caller on a realm that lets anyone write is told the request needs +// an identity once, for the whole batch, rather than by whichever entry +// reached the actor first for a reason that reads as a payload problem. export function needsActor(definition: OperationDefinition): boolean { - for (let program of [ - definition.program, - definition.input, - definition.output, - ]) { - if (program && callsActor(program.source)) { - return true; - } - } - return definition.fill !== undefined && templateReadsActor(definition.fill); -} - -function templateReadsActor(template: OperationTemplate): boolean { - if (Array.isArray(template)) { - return template.some(templateReadsActor); - } - if (template === null || typeof template !== 'object') { - return false; - } - if ((template as Record).$ref === 'actor') { - return true; - } - return Object.values(template).some(templateReadsActor); + return definition.readsActor === true; } // One entry as the batch coordinator takes it. @@ -368,6 +377,11 @@ export function batchEntryFor( definition: OperationDefinition, ): BatchEntry { let { index, name } = entry; + assertStagesAreServed(entry, definition); + // Every entry the coordinator stages carries the position the caller sent it + // under, so a batch holding only some of an envelope's entries still reports + // refusals against the envelope's numbering. + let common = { definition, label: index }; switch (definition.base) { case 'create': { if (definition.of) { @@ -376,7 +390,7 @@ export function batchEntryFor( // its href — when it has one — is the card it reads for context. return { op: 'create', - definition, + ...common, params: paramsFor(entry), ...(entry.href ? { href: entry.href } : {}), ...(entry.lid === undefined ? {} : { lid: entry.lid }), @@ -396,7 +410,7 @@ export function batchEntryFor( } return { op: 'create', - definition, + ...common, document: { data: entry.data as unknown as CardResource }, }; } @@ -417,11 +431,11 @@ export function batchEntryFor( href, ); } - return { op: 'update', definition, href, content }; + return { op: 'update', ...common, href, content }; } return { op: 'update', - definition, + ...common, href, document: { data: entry.data as unknown as CardResource }, }; @@ -429,13 +443,13 @@ export function batchEntryFor( case 'delete': return { op: 'delete', - definition, + ...common, href: hrefRequired(entry, 'delete'), }; case 'transform': return { op: 'transform', - definition, + ...common, params: paramsFor(entry), href: hrefRequired(entry, 'transform'), name, @@ -443,14 +457,14 @@ export function batchEntryFor( case 'appendLine': return { op: 'appendLine', - definition, + ...common, params: paramsFor(entry), href: hrefRequired(entry, 'appendLine'), }; case 'appendContainsMany': return { op: 'appendContainsMany', - definition, + ...common, params: paramsFor(entry), href: hrefRequired(entry, 'appendContainsMany'), // Named the way the append executor names them, so the fields and @@ -484,18 +498,66 @@ export function batchEntryFor( // The payload an operation's own params are read from. // -// `lid` is the one member of `data` that is never a param: it is the caller's -// id for the card the entry mints, which is what other entries link to it by, -// and an operation declaring a param under that name would have the two -// meanings arrive in one key. +// `data` carries two kinds of thing at once: the values an operation declares +// as params, and the members the envelope itself reads to work out what the +// entry is — the local id a later entry links by, the type a class-scoped +// entry is scoped to, and the field and items an append names. The second kind +// is never a param, so an operation declaring one under the same name would +// otherwise be handed the envelope's value instead of the caller's. +// +// The rule is the members this module reads, not a list of names to remember +// to grow: anything added to that set belongs here in the same commit. +const ENVELOPE_MEMBERS = [ + 'lid', + 'meta', + 'field', + 'items', + 'fields', + 'content', +] as const; + export function paramsFor(entry: EnvelopeEntry): Record { - if (entry.lid === undefined) { - return entry.data ?? {}; + if (!entry.data) { + return {}; + } + let params: Record = {}; + for (let [key, value] of Object.entries(entry.data)) { + if ((ENVELOPE_MEMBERS as readonly string[]).includes(key)) { + continue; + } + params[key] = value; } - let { lid: _lid, ...params } = entry.data ?? {}; return params; } +// A declaration may reshape its payload with an `input` program and project +// its result with an `output` one. A batch runs neither, and carrying the +// entry out as though the declaration said nothing answers a different +// question well — the author's `input` was to produce the very value the +// executor then reports as missing. So the refusal names the stage, the way +// the read executor refuses a specialization it does not carry out. +function assertStagesAreServed( + entry: EnvelopeEntry, + definition: OperationDefinition, +): void { + let stages = (['input', 'output'] as const).filter( + (stage) => definition[stage] !== undefined, + ); + if (stages.length === 0) { + return; + } + throw new OperationFailure({ + ...(entry.href ? { id: entry.href } : {}), + status: 501, + code: 'internal-error', + title: 'Operation not implemented', + detail: + `operation "${entry.name}" specializes its behavior with ` + + `${stages.join(' and ')}, which a batch does not run`, + meta: { entry: entry.index }, + }); +} + function hrefRequired(entry: EnvelopeEntry, base: BaseOperation): string { if (entry.href === undefined) { throw refuse( @@ -607,30 +669,10 @@ export function errorsDocument(error: OperationError): { // Label a refusal with the position of the entry that produced it, so a caller // reading one error knows which of the entries it sent is wrong. // -// A refusal that already carries a position keeps it. The coordinator labels -// what it stages with the position in the batch it was handed, which is not -// the position in the envelope when the batch holds only the entries that -// write — so `at` is what the two are reconciled through, and relabelling one -// that arrived correct would overwrite an answer with a guess. -export function labelEntry( - err: unknown, - at: (index: number) => number, -): unknown { - if (!isOperationFailure(err)) { - return err; - } - let entry = err.error.meta?.entry; - if (typeof entry !== 'number') { - return err; - } - return new OperationFailure({ - ...err.error, - meta: { ...err.error.meta, entry: at(entry) }, - }); -} - -// The same, for a refusal raised where the position is known outright — an -// entry parsed, resolved or staged one at a time. +// For a refusal raised where the position is known outright — an entry parsed +// or resolved one at a time. A staged entry carries its position into the +// coordinator instead, which writes it everywhere a position appears rather +// than leaving one key to be corrected afterwards. export function atEntry(err: unknown, index: number): unknown { if (!isOperationFailure(err)) { return err; diff --git a/packages/runtime-common/card-operations/executors.ts b/packages/runtime-common/card-operations/executors.ts index f097b86536a..0fabe0fbe66 100644 --- a/packages/runtime-common/card-operations/executors.ts +++ b/packages/runtime-common/card-operations/executors.ts @@ -166,6 +166,14 @@ export interface BatchDocument { } interface EntryCommon { + // The position this entry holds in whatever the caller composed it from, + // where that is not its position in the batch. A transport that stages only + // some of what it was sent — an envelope holding reads alongside writes — + // hands over a shorter list, so a refusal naming the position in this batch + // would name an entry the caller did not send. Every position the batch + // reports comes from here when it is set: the key on the error, the one + // beside it naming a conflicting entry, and the prose. + label?: number; // The lowered operation, when the entry invokes a named operation rather // than a plain base one. A named `create` stages its card from the // definition's `of` and `fill` instead of from a document. diff --git a/packages/runtime-common/card-operations/index.ts b/packages/runtime-common/card-operations/index.ts index d67a4422ab4..afa7b2bea26 100644 --- a/packages/runtime-common/card-operations/index.ts +++ b/packages/runtime-common/card-operations/index.ts @@ -87,7 +87,6 @@ export { carriesOperationsExt, errorsDocument, isWrite, - labelEntry, needsActor, paramsFor, parseOperationsEnvelope, diff --git a/packages/runtime-common/card-operations/lowering.ts b/packages/runtime-common/card-operations/lowering.ts index d8999fdd56a..f2e4179a923 100644 --- a/packages/runtime-common/card-operations/lowering.ts +++ b/packages/runtime-common/card-operations/lowering.ts @@ -8,6 +8,7 @@ import { checkExpressionProgram, checkMutationProgram, paramKeysRead, + callsActor, usesVolatileCall, } from './bxl-emit.ts'; import { isDefinitionFreeBaseOperation } from './types.ts'; @@ -436,9 +437,45 @@ async function lowerOperation( operation.output, ].every((program) => !program || !usesVolatileCall(program.source)); + if (readsActor(operation)) { + operation.readsActor = true; + } + return operation; } +// Whether anything this operation runs or fills reads the invoking actor. +// +// Asked here, where the whole lowered operation is in hand, rather than at +// invocation: the answer cannot change between one request and the next, and +// every member that can carry an actor is in front of us — the three programs, +// and the two marker-carrying templates. A caller asking the same question +// from a request would have to remember this list, and would be reading +// program text on a path that must not reach the BXL package at all. +function readsActor(operation: OperationDefinition): boolean { + for (let program of [operation.program, operation.input, operation.output]) { + if (program && callsActor(program.source)) { + return true; + } + } + return [operation.fill, operation.items].some( + (template) => template !== undefined && templateReadsActor(template), + ); +} + +function templateReadsActor(template: OperationTemplate): boolean { + if (Array.isArray(template)) { + return template.some(templateReadsActor); + } + if (template === null || typeof template !== 'object') { + return false; + } + if ((template as Record).$ref === 'actor') { + return true; + } + return Object.values(template).some(templateReadsActor); +} + // --------------------------------------------------------------------------- // The payload schema // --------------------------------------------------------------------------- diff --git a/packages/runtime-common/card-operations/types.ts b/packages/runtime-common/card-operations/types.ts index 852bcfca611..ae84e4b4a86 100644 --- a/packages/runtime-common/card-operations/types.ts +++ b/packages/runtime-common/card-operations/types.ts @@ -129,6 +129,18 @@ export interface OperationDefinition { // holds — a non-deterministic one would land a different value locally than // the server computes, and reconciliation would report a phantom conflict. deterministic: boolean; + // Whether carrying this operation out needs to know who the caller is: + // whether any program it runs calls `actor()`, or any template it fills + // carries an actor marker. Recorded here rather than asked at invocation + // because it is a property of the declaration, fixed from the moment the + // module is indexed — so a transport can refuse a request that authenticated + // nobody before any of the batch runs, instead of part-way through by + // whichever entry reached the actor first. + // + // Absent means no, which is also what an entry built before this was + // recorded reports. A definition-cache entry is rebuilt on demand, so such + // an entry is replaced rather than corrected. + readsActor?: true; // Set when lowering found problems. The operation is stored either way, so // invoking it reports what is wrong with it rather than "unknown // operation". diff --git a/packages/runtime-common/index.ts b/packages/runtime-common/index.ts index 35928d28e0b..061e4c23945 100644 --- a/packages/runtime-common/index.ts +++ b/packages/runtime-common/index.ts @@ -1748,7 +1748,11 @@ export const isNode = Object.prototype.toString.call((globalThis as any).process) === '[object process]'; -export { SupportedMimeType, isJsonContentType } from './supported-mime-type.ts'; +export { + BOXEL_OPERATIONS_EXT, + SupportedMimeType, + isJsonContentType, +} from './supported-mime-type.ts'; export { isUrlLike, VirtualNetwork, diff --git a/packages/runtime-common/realm.ts b/packages/runtime-common/realm.ts index 30654ecfca2..038b0b5e132 100644 --- a/packages/runtime-common/realm.ts +++ b/packages/runtime-common/realm.ts @@ -172,6 +172,7 @@ import { type FileMetaResource, } from './index.ts'; import { + canonicalizeTarget, newOperationScope, resolveOperation, runOperation, @@ -191,7 +192,6 @@ import { carriesOperationsExt, errorsDocument, isWrite, - labelEntry, needsActor, paramsFor, parseOperationsEnvelope, @@ -4112,7 +4112,9 @@ export class Realm { detail: `the request body is not valid JSON`, }); } - let entries = parseOperationsEnvelope(body, this.url); + let entries = parseOperationsEnvelope(body, this.url, { + resolveIdentifier: (href) => this.#resolveAtomicHref(href), + }); let caller = this.#callerOf(request, requestContext); // One row peek per target for the whole resolution pass: entries often // name the same card, and which behavior a name resolves to is read off @@ -4211,19 +4213,14 @@ export class Realm { throw atEntry(err, entry.index); } }); - let committed: Awaited>; - try { - committed = await commitBatch(this.batchCore, staged, { - clientRequestId: caller.clientRequestId || null, - actor: caller.actor || undefined, - }); - } catch (err: unknown) { - // The coordinator labels a refusal with the position in the batch it - // was handed, and that batch holds only the entries that write — so a - // mixed batch's positions are not the envelope's, and the caller is - // told about an entry it did not send unless they are mapped back. - throw labelEntry(err, (index) => writes[index].entry.index); - } + // Every staged entry carries the position the caller sent it under, so + // a refusal from the coordinator names that one rather than the position + // it took among the entries that write — in the key, in the key beside + // it naming a conflicting entry, and in the prose. + let committed = await commitBatch(this.batchCore, staged, { + clientRequestId: caller.clientRequestId || null, + actor: caller.actor || undefined, + }); for (let [index, { entry }] of writes.entries()) { results[entry.index] = writeResult(committed[index], (url) => this.#virtualNetwork.unresolveURL(url), @@ -4233,8 +4230,11 @@ export class Realm { return this.#operationsResponse( { - 'atomic:results': results.map((result, index) => - answered(result, index), + // Built by index rather than by mapping the array: an unassigned + // position is a hole, `map` skips one, and the hole then serializes + // as `null` — which is what a delete reports. + 'atomic:results': Array.from({ length: entries.length }, (_, index) => + answered(results[index], index), ), }, 200, @@ -4253,15 +4253,29 @@ export class Realm { scope: OperationScope, ): Promise { try { - let target = targetFor(entry, this.url); + // Canonicalized once, here, and everything downstream sees the result: + // the definition is resolved from it, the read runs against it, and the + // staged entry writes it. A trailing slash, a query string and a + // fragment all name the card they hang off, and the index is read by + // exact URL — so resolving the raw spelling would report a declared + // operation as unknown on a card whose type declares it, and would leave + // the entry's result carrying an id no other surface spells that way. + let target = canonicalizeTarget( + this.operationCore, + targetFor(entry, this.url), + ); + let canonical = + target.kind === 'instance' && target.url !== entry.href + ? { ...entry, href: target.url } + : entry; let definition = await resolveOperation( this.operationCore, target, - entry.name, + canonical.name, scope, ); - assertTravelsInEnvelope(entry, definition); - return { entry, target, definition }; + assertTravelsInEnvelope(canonical, definition); + return { entry: canonical, target, definition }; } catch (err: unknown) { throw atEntry(err, entry.index); } diff --git a/packages/runtime-common/router.ts b/packages/runtime-common/router.ts index 84897541d4b..d063a4040a4 100644 --- a/packages/runtime-common/router.ts +++ b/packages/runtime-common/router.ts @@ -86,9 +86,87 @@ export function extractSupportedMimeType( return candidateMimeType as SupportedMimeType; } } + // A media type is its type and subtype; its parameters qualify it without + // making it a different type. The pass above compares whole header values, + // so it answers only for the one spelling a registered type is written in — + // which is every spelling anything sent until a registered type carried a + // parameter of its own. One does now: the operations envelope is + // `application/vnd.api+json` with an `ext` naming its extension, and the + // ways a client legitimately writes that (a space after the semicolon, an + // unquoted value, a `charset` alongside, its URI in a list with another + // extension's) are all the same media type and none of them is that string. + // + // So a second pass compares the parts. It only ever adds a match where the + // first pass found none, and it prefers the registered type whose own + // extensions the candidate carries — otherwise a body that named the + // envelope's extension would route to the plain JSON:API family and be + // answered as though it had named none. + for (const candidateMimeType of acceptMimeTypes) { + let matched = matchParameterized(candidateMimeType, supportedMimeTypes); + if (matched) { + return matched; + } + } return undefined; } +interface ParsedMediaType { + type: string; + // The extension URIs the `ext` parameter names, which JSON:API writes as a + // space-separated list. Compared case-sensitively — they are URIs — while + // the type and the parameter's name are not. + extensions: string[]; +} + +function parseMediaType(value: string): ParsedMediaType { + let [type, ...parameters] = value.split(';'); + let extensions: string[] = []; + for (let parameter of parameters) { + let separator = parameter.indexOf('='); + if (separator === -1) { + continue; + } + if (parameter.slice(0, separator).trim().toLowerCase() !== 'ext') { + continue; + } + let raw = parameter.slice(separator + 1).trim(); + if (raw.startsWith('"') && raw.endsWith('"')) { + raw = raw.slice(1, -1); + } + extensions.push(...raw.split(/\s+/).filter(Boolean)); + } + return { type: type.trim().toLowerCase(), extensions }; +} + +function matchParameterized( + candidate: string, + supportedMimeTypes: SupportedMimeType[], +): SupportedMimeType | undefined { + let sent = parseMediaType(candidate); + if (!sent.type) { + return undefined; + } + let plain: SupportedMimeType | undefined; + for (let supported of supportedMimeTypes) { + let registered = parseMediaType(supported); + if (registered.type !== sent.type) { + continue; + } + if (registered.extensions.length === 0) { + plain ??= supported; + continue; + } + if ( + registered.extensions.every((extension) => + sent.extensions.includes(extension), + ) + ) { + return supported; + } + } + return plain; +} + export type RouteTable = Map>>; export function lookupRouteTable( From 285c4a952562bd95660ad3a14cd7c541089e4ca6 Mon Sep 17 00:00:00 2001 From: Hassan Abdel-Rahman Date: Wed, 16 Sep 2026 19:29:36 -0400 Subject: [PATCH 09/13] Match a media type by its parts, and fold an href before it is resolved A registered media type now carries a parameter of its own, so comparing whole header values answered for one spelling of it and left the rest unrouted. The lookup compares the type and its extensions, preferring the registered type whose extensions the candidate carries. An entry's href is resolved out of registered-prefix form and canonicalized once, before the definition is resolved, so a declared operation is found on every spelling of a card's URL and the identity an entry answers with is the one the realm serves that card under. A type reference that is not one is refused as the caller's mistake rather than thrown out of the resolver, a staged entry carries the caller's position into the coordinator instead of having one key corrected afterwards, params are what is left when the members the envelope reads for itself are taken out, and a declaration carrying a stage a batch does not run is refused by name. Co-Authored-By: Claude Opus 5 (1M context) --- .../tests/realm-endpoints/operations-test.ts | 256 +++++++++--------- 1 file changed, 133 insertions(+), 123 deletions(-) diff --git a/packages/realm-server/tests/realm-endpoints/operations-test.ts b/packages/realm-server/tests/realm-endpoints/operations-test.ts index ca80890aeea..b949badeabd 100644 --- a/packages/realm-server/tests/realm-endpoints/operations-test.ts +++ b/packages/realm-server/tests/realm-endpoints/operations-test.ts @@ -142,7 +142,7 @@ function makeFileSystem(): Record { @operation static restate = { base: 'transform', params: { headline: StringField }, - input: bxl`{headline: params("headline")}`, + input: bxl\`{headline: params("headline")}\`, set: { headline: params('headline') }, }; @@ -333,32 +333,36 @@ module(`realm-endpoints/${basename(import.meta.filename)}`, function () { }); test('every spelling of the operations media type reaches the endpoint', async function (assert) { - // The router matches a media type by its type and its parameters, so the - // extension is named the same operation whichever way a client writes - // it. Sent as the content type alone in each case — a `fetch` that sets - // one and leaves `Accept` to its default is the shape that has no other - // header to fall back on. - for (let spelling of [ - `application/vnd.api+json;ext="${BOXEL_OPERATIONS_EXT}"`, - `application/vnd.api+json; ext="${BOXEL_OPERATIONS_EXT}"`, - `application/vnd.api+json;ext=${BOXEL_OPERATIONS_EXT}`, - `application/vnd.api+json;ext="https://jsonapi.org/ext/atomic ${BOXEL_OPERATIONS_EXT}"`, - `application/vnd.api+json;charset=utf-8;ext="${BOXEL_OPERATIONS_EXT}"`, - ]) { - let response = await request - .post('/_operations') - .set('Content-Type', spelling) - .set( - 'Authorization', - `Bearer ${createJWT(realm, TESTER, ['read', 'write'])}`, - ) - .send(envelope(invoke('read', { href: '/report-kept' }))); - - assert.strictEqual(response.status, 200, `${spelling} is carried out`); - } - }); + // The router matches a media type by its type and its parameters, so the + // extension is named the same operation whichever way a client writes + // it. Sent as the content type alone in each case — a `fetch` that sets + // one and leaves `Accept` to its default is the shape that has no other + // header to fall back on. + for (let spelling of [ + `application/vnd.api+json;ext="${BOXEL_OPERATIONS_EXT}"`, + `application/vnd.api+json; ext="${BOXEL_OPERATIONS_EXT}"`, + `application/vnd.api+json;ext=${BOXEL_OPERATIONS_EXT}`, + `application/vnd.api+json;ext="https://jsonapi.org/ext/atomic ${BOXEL_OPERATIONS_EXT}"`, + `application/vnd.api+json;charset=utf-8;ext="${BOXEL_OPERATIONS_EXT}"`, + ]) { + let response = await request + .post('/_operations') + .set('Content-Type', spelling) + .set( + 'Authorization', + `Bearer ${createJWT(realm, TESTER, ['read', 'write'])}`, + ) + .send(envelope(invoke('read', { href: '/report-kept' }))); + + assert.strictEqual( + response.status, + 200, + `${spelling} is carried out`, + ); + } + }); - test('a body sent without the operations extension is told what it is missing', async function (assert) { + test('a body sent without the operations extension is told what it is missing', async function (assert) { let response = await request .post('/_operations') .set('Accept', SupportedMimeType.JSONAPI) @@ -449,49 +453,51 @@ module(`realm-endpoints/${basename(import.meta.filename)}`, function () { }); test('a type reference that is not one is refused rather than thrown out of', async function (assert) { - let response = await post( - envelope( - invoke('create', { - data: { - type: 'card', - attributes: { firstName: 'Mango' }, - meta: { adoptsFrom: { type: 'fieldOf', card: null, field: 'x' } }, - }, - }), - ), - ); + let response = await post( + envelope( + invoke('create', { + data: { + type: 'card', + attributes: { firstName: 'Mango' }, + meta: { + adoptsFrom: { type: 'fieldOf', card: null, field: 'x' }, + }, + }, + }), + ), + ); - assert.strictEqual( - response.status, - 400, - 'a malformed reference is the caller\'s to fix, not a fault to report', - ); - assert.strictEqual(response.body.errors[0].meta.entry, 0); - }); + assert.strictEqual( + response.status, + 400, + "a malformed reference is the caller's to fix, not a fault to report", + ); + assert.strictEqual(response.body.errors[0].meta.entry, 0); + }); - test('an operation whose declaration carries a stage a batch does not run is refused', async function (assert) { - let response = await post( - envelope( - invoke('restate', { - href: '/report-unserved', - data: { headline: 'Revised' }, - }), - ), - ); + test('an operation whose declaration carries a stage a batch does not run is refused', async function (assert) { + let response = await post( + envelope( + invoke('restate', { + href: '/report-unserved', + data: { headline: 'Revised' }, + }), + ), + ); - assert.strictEqual(response.status, 501, 'HTTP 501 status'); - assert.true( - response.body.errors[0].detail.includes('input'), - `the refusal names the stage: ${response.body.errors[0].detail}`, - ); - assert.strictEqual( - storedCard('report-unserved.json').data.attributes?.headline, - 'Quarterly Review', - 'and nothing was written under a declaration half carried out', - ); - }); + assert.strictEqual(response.status, 501, 'HTTP 501 status'); + assert.true( + response.body.errors[0].detail.includes('input'), + `the refusal names the stage: ${response.body.errors[0].detail}`, + ); + assert.strictEqual( + storedCard('report-unserved.json').data.attributes?.headline, + 'Quarterly Review', + 'and nothing was written under a declaration half carried out', + ); + }); - test('two entries claiming one local id are refused', async function (assert) { + test('two entries claiming one local id are refused', async function (assert) { let response = await post( envelope( invoke('create', { @@ -642,32 +648,32 @@ module(`realm-endpoints/${basename(import.meta.filename)}`, function () { }); test('an equivalent spelling of an href names the same card', async function (assert) { - // The index is read by exact URL, so a declared operation resolves only - // once the spelling has been folded to the one the realm addresses the - // card by — and the identity the entry answers with is that one too. - for (let [spelling, what] of [ - ['/report-canonical?view=full', 'a query string'], - ['/report-canonical#section', 'a fragment'], - ]) { - let response = await post( - envelope(invoke('escalate', { href: spelling })), - ); - - assert.strictEqual(response.status, 200, `${what} is carried out`); + // The index is read by exact URL, so a declared operation resolves only + // once the spelling has been folded to the one the realm addresses the + // card by — and the identity the entry answers with is that one too. + for (let [spelling, what] of [ + ['/report-canonical?view=full', 'a query string'], + ['/report-canonical#section', 'a fragment'], + ]) { + let response = await post( + envelope(invoke('escalate', { href: spelling })), + ); + + assert.strictEqual(response.status, 200, `${what} is carried out`); + assert.strictEqual( + response.body['atomic:results'][0].data.id, + `${testRealmHref}report-canonical`, + `${what} answers with the id the realm serves the card under`, + ); + } assert.strictEqual( - response.body['atomic:results'][0].data.id, - `${testRealmHref}report-canonical`, - `${what} answers with the id the realm serves the card under`, + storedCard('report-canonical.json').data.attributes?.status, + 'escalated', + 'and the card the spellings name is the one that changed', ); - } - assert.strictEqual( - storedCard('report-canonical.json').data.attributes?.status, - 'escalated', - 'and the card the spellings name is the one that changed', - ); - }); + }); - test('a delete answers with no state and removes the file', async function (assert) { + test('a delete answers with no state and removes the file', async function (assert) { let response = await post( envelope(invoke('delete', { href: '/report-deleted' })), ); @@ -775,43 +781,47 @@ module(`realm-endpoints/${basename(import.meta.filename)}`, function () { }); test('a refusal names the entry the caller sent, in its key and in its prose', async function (assert) { - // The coordinator is handed only the entries that write, so its own - // numbering runs 0,1 where the caller sent 1,2. Both the keys and the - // sentence have to speak the caller's. - let response = await post( - envelope( - invoke('read', { href: '/report-kept' }), - invoke('appendLine', { - href: '/positions.log', - data: { line: 'deployed' }, - }), - invoke('update', { - href: '/positions.log', - data: { content: 'replaced\n' }, - }), - ), - ); + // The coordinator is handed only the entries that write, so its own + // numbering runs 0,1 where the caller sent 1,2. Both the keys and the + // sentence have to speak the caller's. + let response = await post( + envelope( + invoke('read', { href: '/report-kept' }), + invoke('appendLine', { + href: '/positions.log', + data: { line: 'deployed' }, + }), + invoke('update', { + href: '/positions.log', + data: { content: 'replaced\n' }, + }), + ), + ); - assert.strictEqual(response.status, 400, 'HTTP 400 status'); - let [error] = response.body.errors; - assert.strictEqual(error.meta.entry, 2, 'the entry that collides'); - assert.strictEqual( - error.meta.conflictsWith, - 1, - 'and the entry it collides with', - ); - assert.true( - error.detail.includes('entry 2') && error.detail.includes('entry 1'), - `the prose names the same two entries: ${error.detail}`, - ); - assert.strictEqual( - readFileSync(realmFile('positions.log'), 'utf8'), - 'boot\n', - 'and nothing was written', - ); - }); + assert.strictEqual(response.status, 400, 'HTTP 400 status'); + let [error] = response.body.errors; + assert.strictEqual(error.meta.entry, 2, 'the entry that collides'); + assert.strictEqual( + error.meta.conflictsWith, + 1, + 'and the entry it collides with', + ); + assert.true( + error.detail.includes('entry 2'), + `the prose names the entry that collides: ${error.detail}`, + ); + assert.true( + error.detail.includes('entry 1'), + `and the entry it collides with: ${error.detail}`, + ); + assert.strictEqual( + readFileSync(realmFile('positions.log'), 'utf8'), + 'boot\n', + 'and nothing was written', + ); + }); - test('a read in a mixed batch answers with the pre-batch document', async function (assert) { + test('a read in a mixed batch answers with the pre-batch document', async function (assert) { let response = await post( envelope( invoke('read', { href: '/report-mixed' }), From bf50eb528c89133e30cec21a89490d1f2c27ef41 Mon Sep 17 00:00:00 2001 From: Hassan Abdel-Rahman Date: Wed, 16 Sep 2026 19:52:40 -0400 Subject: [PATCH 10/13] Pin the actor flag on a program, a fill and a projection Co-Authored-By: Claude Opus 5 (1M context) --- packages/host/tests/unit/operation-lowering-test.ts | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/packages/host/tests/unit/operation-lowering-test.ts b/packages/host/tests/unit/operation-lowering-test.ts index 8bfa8d3b8cc..8f4366723a5 100644 --- a/packages/host/tests/unit/operation-lowering-test.ts +++ b/packages/host/tests/unit/operation-lowering-test.ts @@ -211,8 +211,9 @@ module('Unit | operation lowering', function (hooks) { syntax: 'solidified', }, deterministic: true, + readsActor: true, }, - 'a link-typed member becomes a card identity while a scalar one stays the bare builtin', + 'a link-typed member becomes a card identity while a scalar one stays the bare builtin, and a program naming the actor says so', ); }); @@ -424,8 +425,9 @@ module('Unit | operation lowering', function (hooks) { postedBy: { $ref: 'actor' }, }, deterministic: true, + readsActor: true, }, - 'a linkTo param is a link entry and fill stays data for the coordinator to substitute', + 'a linkTo param is a link entry, fill stays data for the coordinator to substitute, and an actor marker inside it is reported the same as one in a program', ); }); @@ -528,8 +530,9 @@ module('Unit | operation lowering', function (hooks) { base: 'read', output: { source: '{label:actor()}', syntax: 'solidified' }, deterministic: true, + readsActor: true, }, - 'a declarative projection and a raw program reach the realm in one form', + 'a declarative projection and a raw program reach the realm in one form, and a projection reading the actor is reported like any other stage', ); }); From 6c3aacaa200f49718d8b6f85a8f88e67edee8986 Mon Sep 17 00:00:00 2001 From: Hassan Abdel-Rahman Date: Wed, 16 Sep 2026 21:46:59 -0400 Subject: [PATCH 11/13] Pin that an append's field and items reach its executor Co-Authored-By: Claude Opus 5 (1M context) --- .../tests/realm-endpoints/operations-test.ts | 61 +++++++++++++++++++ 1 file changed, 61 insertions(+) diff --git a/packages/realm-server/tests/realm-endpoints/operations-test.ts b/packages/realm-server/tests/realm-endpoints/operations-test.ts index b949badeabd..008c8ba9a82 100644 --- a/packages/realm-server/tests/realm-endpoints/operations-test.ts +++ b/packages/realm-server/tests/realm-endpoints/operations-test.ts @@ -162,6 +162,40 @@ function makeFileSystem(): Record { } } `, + 'event-log.gts': ` + import { contains, containsMany, field, CardDef, FieldDef, Component } from "@cardstack/base/card-api"; + import StringField from "@cardstack/base/string"; + + export class LogEvent extends FieldDef { + @field label = contains(StringField); + } + + export class EventLog extends CardDef { + @field title = contains(StringField); + @field events = containsMany(LogEvent); + static isolated = class Isolated extends Component { + + } + static embedded = class Embedded extends Component { + + } + static fitted = class Fitted extends Component { + + } + } + `, + 'deploys.json': { + data: { + type: 'card', + attributes: { title: 'Deploys', events: [] }, + meta: { + adoptsFrom: { + module: rri(`${testRealmHref}event-log`), + name: 'EventLog', + }, + }, + }, + }, 'reviewer.json': { data: { type: 'card', @@ -721,6 +755,33 @@ module(`realm-endpoints/${basename(import.meta.filename)}`, function () { ); }); + test('an append hands the field and its items through to the executor', async function (assert) { + // The one entry shape whose payload is neither a document nor params: + // the field and the items are named in `data` and handed to the append + // executor under the names it reads them by. They are also the members + // taken out of what an operation sees as its params, so passing them + // through and keeping them out of params are the same change. + let response = await post( + envelope( + invoke('appendContainsMany', { + href: '/deploys', + data: { field: 'events', items: [{ label: 'shipped' }] }, + }), + ), + ); + + assert.strictEqual(response.status, 200, 'HTTP 200 status'); + assert.strictEqual( + response.body['atomic:results'][0].data.id, + `${testRealmHref}deploys`, + ); + assert.deepEqual( + storedCard('deploys.json').data.attributes?.events, + [{ label: 'shipped' }], + 'the item the entry named is on the end of the field it named', + ); + }); + test('a read answers with the document, and a write on a file is refused', async function (assert) { let read = await query(envelope(invoke('read', { href: '/notes.md' }))); From 18df8b16c6c2f05522673772b20a31eb75ac60de Mon Sep 17 00:00:00 2001 From: Hassan Abdel-Rahman Date: Wed, 16 Sep 2026 22:19:28 -0400 Subject: [PATCH 12/13] Pin that a lowered operation's actor flag survives the prerender channel Co-Authored-By: Claude Opus 5 (1M context) --- .../realm-server/tests/prerendering-test.ts | 76 +++++++++++++++++++ 1 file changed, 76 insertions(+) diff --git a/packages/realm-server/tests/prerendering-test.ts b/packages/realm-server/tests/prerendering-test.ts index 1eab856d2bf..b1f35d70c37 100644 --- a/packages/realm-server/tests/prerendering-test.ts +++ b/packages/realm-server/tests/prerendering-test.ts @@ -618,6 +618,82 @@ module(basename(import.meta.filename), function () { ); }); + test("a module's lowered operations report whether they read the actor", async function (assert) { + // Lowering runs in the prerender host, and this is where the realm + // server sees what it produced: the visit hands back the definitions it + // built, so a member computed at lowering time is asserted on directly + // rather than through indexing and an endpoint. `readsActor` is what + // lets a transport refuse an operation needing an identity before any of + // a batch runs, so it has to survive this channel to be worth anything. + const moduleURL = `${realmURL}report.gts`; + await realmAdapter.write( + 'report.gts', + ` + import { CardDef, FieldDef, field, contains, containsMany, StringField, Component } from '@cardstack/base/card-api'; + import { operation, params, actor } from '@cardstack/base/operations'; + + export class ReportComment extends FieldDef { + @field body = contains(StringField); + @field postedBy = contains(StringField); + } + + export class ExternalReport extends CardDef { + static displayName = "External Report"; + @field status = contains(StringField); + @field comments = containsMany(ReportComment); + + @operation static addComment = { + base: 'transform', + params: { body: StringField }, + append: { + to: 'comments', + value: { body: params('body'), postedBy: actor() }, + }, + }; + + @operation static escalate = { + base: 'transform', + set: { status: 'escalated' }, + }; + + static isolated = class extends Component { + + } + } + `, + ); + realm.__testOnlyClearCaches(); + + let result = await prerenderer.prerenderModule({ + affinityType: 'realm', + affinityValue: realmURL, + realm: realmURL, + url: moduleURL, + auth: auth(), + renderOptions: { clearCache: true }, + }); + + let key = `${trimExecutableExtension(rri(moduleURL))}/ExternalReport`; + let entry = result.response.definitions[key]; + if (entry?.type === 'definition') { + let operations = entry.definition.operations ?? {}; + let issues = operations.addComment?.issues ?? []; + assert.deepEqual(issues, [], 'the declaration lowers cleanly'); + assert.true( + operations.addComment?.readsActor, + 'an operation whose program names actor() says so', + ); + assert.strictEqual( + operations.escalate?.readsActor, + undefined, + 'and one that never reads it carries nothing, which is the absence ' + + 'a transport reads as "no identity needed"', + ); + } else { + assert.ok(false, "the visit should carry the type's definition"); + } + }); + test('module prerender reuses pooled page after updates', async function (assert) { const moduleURL = `${realmURL}person.gts`; From 7143d3711615de521a95e981da6e046f1efbeb15 Mon Sep 17 00:00:00 2001 From: Hassan Abdel-Rahman Date: Thu, 17 Sep 2026 09:12:55 -0400 Subject: [PATCH 13/13] Subtract from a payload only what the entry's own behavior reads A named operation declaring a param under one of the members the envelope reads for itself became uninvokable: the value was taken out of its payload and the executor then refused the operation for carrying none. Only the local id and the type are read for every entry; the rest belong to one arm and are subtracted there. The route lookup falls through to the content type on failing to match a route rather than on the family holding none for the method, so an Accept that selects a family with routes for other paths no longer hides the content type that named the endpoint. Co-Authored-By: Claude Opus 5 (1M context) --- .../tests/realm-endpoints/operations-test.ts | 19 +++++ .../card-operations/envelope.ts | 37 +++++----- packages/runtime-common/router.ts | 70 ++++++++++++------- .../runtime-common/supported-mime-type.ts | 4 +- 4 files changed, 84 insertions(+), 46 deletions(-) diff --git a/packages/realm-server/tests/realm-endpoints/operations-test.ts b/packages/realm-server/tests/realm-endpoints/operations-test.ts index 008c8ba9a82..990c27dee72 100644 --- a/packages/realm-server/tests/realm-endpoints/operations-test.ts +++ b/packages/realm-server/tests/realm-endpoints/operations-test.ts @@ -396,6 +396,25 @@ module(`realm-endpoints/${basename(import.meta.filename)}`, function () { } }); + test('an Accept naming another family does not hide the content type that named this endpoint', async function (assert) { + // The lookup prefers `Accept` and falls back to `Content-Type`, and the + // fall-through is on failing to match a route rather than on the family + // holding none for the method. `application/json` holds POST routes — + // for other paths — so stopping at the family would answer "no such + // route" to a request whose content type named this one. + let response = await request + .post('/_operations') + .set('Accept', 'application/json; charset=utf-8') + .set('Content-Type', OPERATIONS) + .set( + 'Authorization', + `Bearer ${createJWT(realm, TESTER, ['read', 'write'])}`, + ) + .send(envelope(invoke('read', { href: '/report-kept' }))); + + assert.strictEqual(response.status, 200, 'HTTP 200 status'); + }); + test('a body sent without the operations extension is told what it is missing', async function (assert) { let response = await request .post('/_operations') diff --git a/packages/runtime-common/card-operations/envelope.ts b/packages/runtime-common/card-operations/envelope.ts index e011a549cbe..cae2e602a76 100644 --- a/packages/runtime-common/card-operations/envelope.ts +++ b/packages/runtime-common/card-operations/envelope.ts @@ -465,7 +465,7 @@ export function batchEntryFor( return { op: 'appendContainsMany', ...common, - params: paramsFor(entry), + params: paramsFor(entry, ['field', 'items', 'fields']), href: hrefRequired(entry, 'appendContainsMany'), // Named the way the append executor names them, so the fields and // their items are handed through rather than restated. @@ -500,29 +500,32 @@ export function batchEntryFor( // // `data` carries two kinds of thing at once: the values an operation declares // as params, and the members the envelope itself reads to work out what the -// entry is — the local id a later entry links by, the type a class-scoped -// entry is scoped to, and the field and items an append names. The second kind -// is never a param, so an operation declaring one under the same name would -// otherwise be handed the envelope's value instead of the caller's. +// entry is. The second kind is never a param, so an operation declaring one +// under the same name would be handed the envelope's value instead of the +// caller's. // -// The rule is the members this module reads, not a list of names to remember -// to grow: anything added to that set belongs here in the same commit. -const ENVELOPE_MEMBERS = [ - 'lid', - 'meta', - 'field', - 'items', - 'fields', - 'content', -] as const; +// Subtracted per entry rather than as one flat set, because which members the +// envelope reads depends on the behavior in hand. Two are read for every +// entry: the local id a later entry links by, and the type a class-scoped +// entry is scoped to. The rest belong to one arm, and taking them out +// everywhere would make an operation declaring a param under one of those +// names uninvokable — its executor would refuse a value the caller did send, +// which is the failure the unserved-stage refusal exists to avoid. +const ENVELOPE_MEMBERS = ['lid', 'meta'] as const; -export function paramsFor(entry: EnvelopeEntry): Record { +export function paramsFor( + entry: EnvelopeEntry, + // The members this entry's own arm reads out of `data`, on top of the two + // every entry carries. + alsoRead: readonly string[] = [], +): Record { if (!entry.data) { return {}; } + let envelopeMembers = [...ENVELOPE_MEMBERS, ...alsoRead]; let params: Record = {}; for (let [key, value] of Object.entries(entry.data)) { - if ((ENVELOPE_MEMBERS as readonly string[]).includes(key)) { + if (envelopeMembers.includes(key)) { continue; } params[key] = value; diff --git a/packages/runtime-common/router.ts b/packages/runtime-common/router.ts index d063a4040a4..cb942bd56d5 100644 --- a/packages/runtime-common/router.ts +++ b/packages/runtime-common/router.ts @@ -174,35 +174,9 @@ export function lookupRouteTable( paths: RealmPaths, request: Request, ) { - let acceptMimeType = extractSupportedMimeType( - request.headers.get('Accept') as unknown as null | string | [string], - ); if (!isHTTPMethod(request.method)) { return; } - let routes = acceptMimeType - ? routeTable.get(acceptMimeType)?.get(request.method) - : undefined; - // Fall back to Content-Type when Accept doesn't match a route. This - // supports POST/PATCH routes where the request body type (e.g. - // application/octet-stream) is the meaningful discriminator rather than the - // desired response type. - if (!routes) { - let contentType = extractSupportedMimeType( - request.headers.get('Content-Type') as unknown as - | null - | string - | [string], - ); - if (!contentType) { - return; - } - routes = routeTable.get(contentType)?.get(request.method); - if (!routes) { - return; - } - } - // we construct a new URL within RealmPath.local() param that strips off the query string let requestPath = `/${paths.local(new URL(request.url))}`; // add a leading and trailing slashes back so we can match on routing rules for directories. @@ -210,6 +184,48 @@ export function lookupRouteTable( request.url.endsWith('/') && requestPath !== '/' ? `${requestPath}/` : requestPath; + + let acceptMimeType = extractSupportedMimeType( + request.headers.get('Accept') as unknown as null | string | [string], + ); + let matched = acceptMimeType + ? matchRoute( + routeTable.get(acceptMimeType)?.get(request.method), + requestPath, + ) + : undefined; + if (matched !== undefined) { + return matched; + } + // Fall back to Content-Type when `Accept` doesn't match a route. This + // supports POST/PATCH routes where the request body type (e.g. + // application/octet-stream) is the meaningful discriminator rather than the + // desired response type. + // + // The fall-through is on failing to match a *route*, not on the family + // holding no routes for the method. A family can hold routes for this + // method and none for this path — `application/json` has three `POST` + // routes, all of them other paths — and stopping there would answer "no + // such route" to a request whose `Content-Type` named one. + let contentType = extractSupportedMimeType( + request.headers.get('Content-Type') as unknown as null | string | [string], + ); + if (!contentType || contentType === acceptMimeType) { + return; + } + return matchRoute( + routeTable.get(contentType)?.get(request.method), + requestPath, + ); +} + +function matchRoute( + routes: Map | undefined, + requestPath: string, +): T | undefined { + if (!routes) { + return undefined; + } for (let [route, value] of routes) { // let's take care of auto escaping '/' and anchoring in our route regex's // to make it more readable in our config @@ -218,7 +234,7 @@ export function lookupRouteTable( return value; } } - return; + return undefined; } export class Router { diff --git a/packages/runtime-common/supported-mime-type.ts b/packages/runtime-common/supported-mime-type.ts index 5e75d629017..16afbf1aec9 100644 --- a/packages/runtime-common/supported-mime-type.ts +++ b/packages/runtime-common/supported-mime-type.ts @@ -1,8 +1,8 @@ // The JSON:API extension the operations envelope is defined by. It names the // `invoke` verb and the `boxel:`-prefixed members the envelope carries, which // is what a plain `application/vnd.api+json` body does not have — so the media -// type below carries it as the `ext` parameter and the router matches on the -// whole string. +// type below carries it as the `ext` parameter, and the router matches a media +// type by its type and the extensions its `ext` names. export const BOXEL_OPERATIONS_EXT = 'https://boxel.ai/ext/operations'; // A `const` object (rather than a TS `enum`) so the declaration is