diff --git a/.changeset/deslop-dead-modules.md b/.changeset/deslop-dead-modules.md new file mode 100644 index 000000000..4dd2d9faa --- /dev/null +++ b/.changeset/deslop-dead-modules.md @@ -0,0 +1,18 @@ +--- +"agent-bundle": patch +"@agent-bundle/runtime": patch +--- + +Remove nineteen unreferenced modules left behind by extractions that never +rewired their callers, collapse the surviving duplicated helpers onto their +canonical owners, and fix the three defects that drift had caused: the +Workbench Logs view now shows `lifecycle.replay.started`, `.completed`, and +`.failed` Dev Log records and records carrying `routeId` (the browser log +client's private copy of the `agent-bundle/contracts/dev-logs` vocabulary had +omitted them); the Workbench now subscribes to `dev.host.sync` project events, +which `project-client` had left out of its SSE listener list; and the +Playground trace store redacts with the shared `core/credentials` classifier, +which adds the provider environment-variable patterns its local copy lacked. +`@agent-bundle/runtime` drops the internal, never-exported +`expectCanonicalPayload` helper from `state/contract`. No public export, route, +diagnostic code, or runtime behavior changes otherwise. (#451) diff --git a/AGENTS.md b/AGENTS.md index 28ddb24b5..bfc3a815a 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,5 +1,52 @@ # Repository guidance +## Code hygiene + +- **Extract and rewire in one change.** Every dead module this repo has had to + delete was born the same way: a refactor lifted helpers into a new file and + never switched the original over, so the monolith kept its inline copy and + the new file had zero importers from its first commit. If a commit adds + `foo-codec.ts`, the same commit deletes the code it replaced and leaves + `foo.ts` importing it. The follow-up PR that "wires it up" does not arrive. +- **A module with no production importer is not delivered.** This repo's + dominant failure mode is a thoroughly tested service that nothing mounts. + Before believing a capability exists, find the production caller, not the + test. Before opening a PR, confirm every file it adds is reachable from + `src/index.ts`, a route, a CLI entry, or a hook — a passing suite proves + nothing about whether the code runs. +- **Look for the helper before writing it.** `dev/http.ts` owns request and + response helpers (`diagnostic`, `requestError`, `isRequestDiagnostic`, + `responseDiagnostic`, `responseJson`, `singleHeader`, `isJsonRequest`, + `readBody`, `readJsonBody`, `rawPathname`, `decodedOpaqueSegment`); + `core/strict-json.ts`, `core/errors.ts`, `core/paths.ts`, and + `core/freeze.ts` own their equivalents. A route module that defines its own + `readBody` has forked a security-relevant bound that will be fixed in one + copy and not the other. +- **Never copy a helper to dodge an import cycle.** Move it to a leaf module + both sides import — `config/conventional-entry.ts` is the pattern. A comment + explaining why the copy exists documents the debt; it does not discharge it. +- **One class per name.** Two identical `class FooError` declarations in two + modules are not interchangeable: `instanceof` against the wrong one silently + returns `false`, so the `catch` that was supposed to handle it falls through. + Error classes live with the code that throws them, exported once. +- **Delete on sight.** Unreferenced code is not free — it is read during + review, matched by search, and copied by the next author who finds it before + the live version. Removing it is a `patch` changeset, not a project. +- Neither `pnpm lint` nor `pnpm typecheck` reports an unreferenced module, so + check by hand when a change adds or moves files: + + ```sh + # any tracked file that mentions the module, other than itself + git grep -l '' -- ':!repos' + ``` + + One hit means the module only mentions itself and nothing imports it. Watch + for false positives from prose in `docs/**` and from strings that merely + contain the name: `Symbol('epoch-staging')` in `dev/epoch-store.ts` was the + only match for a 343-line dead file, which is why it read as reachable. +- Gate before pushing: `pnpm typecheck && pnpm lint && pnpm test:unit`, plus + `pnpm build` first if the change touches `packages/rsc-runtime`. + ## Workbench platform scope - The developer Workbench is a desktop-only application. diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 000000000..0b83f64e7 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,6 @@ +Repository guidance for this project lives in [AGENTS.md](./AGENTS.md) — read it +first. It is the single source for code hygiene, Workbench scope, public +examples, the docsite, changesets, pull requests, and vendored `repos/`. + +Keep it that way: add project rules to `AGENTS.md`, never here. A second copy +of the guidance is the same duplication the hygiene section exists to prevent. diff --git a/examples/audiobook-curator/README.md b/examples/audiobook-curator/README.md index 117510010..8358bb960 100644 --- a/examples/audiobook-curator/README.md +++ b/examples/audiobook-curator/README.md @@ -101,7 +101,7 @@ instead of maintaining separate MCP and CLI presenters: | --- | --- | --- | | `DataList`, `Field`, `Callout`, and `FileList` | Provide atomic report fields, prose callouts, and file-list blocks throughout the component library and directly in the catalog resource, curate prompt, cache route, and library audit | Provide the same primitives through the shared components and directly in `library-audit` | | `FileCard` and `EditionCard`, fed by `view-models` | Render file and edition models in `audit_library`, shelf, and ranking views | Reached through the receipt-specific shelves and ranking components | -| `InspectionShelf`, `InventoryShelf`, `AuditShelf`, and `SelectionShelf` | Compose receipt-specific inspection, inventory, audit, and selection reports; the inspection, inventory, and selection shelves are used directly by their MCP routes | `InventoryShelf` and `SelectionShelf` compose `inventory` and `select` | +| `InspectionShelf`, `InventoryShelf`, and `SelectionShelf` | Compose receipt-specific inspection, inventory, and selection reports, each used directly by its MCP route; `audit_library` composes `AuditSummary` and `AuditFileCards` alongside the asynchronous `LibraryAnalysis` instead of a shelf | `InventoryShelf` and `SelectionShelf` compose `inventory` and `select` | | `SearchRanking`, `IdentifyRanking`, and `SelectionRanking` | Render the statically typed ranking for `search_audible`, `identify_audible_sample`, and `select_audible_edition` | `SearchRanking` composes `audible-search` | | `AcousticTrail`, `IdentifyTrail`, and `WhisperTrail` | Render the statically typed evidence for acoustic verification, acoustic identification, and Whisper verification | No authored rendered counterpart; those compatibility commands remain plain `.ts` routes | | `MetadataMutation`, `ChapterMutation`, `ConversionMutation`, and `PrepareMutation` | Render each statically typed metadata, chapter, conversion, or preparation mutation | `ConversionMutation` composes `convert` | diff --git a/examples/audiobook-curator/src/components/library-shelf.tsx b/examples/audiobook-curator/src/components/library-shelf.tsx index ce71b64e3..a98e9f4c9 100644 --- a/examples/audiobook-curator/src/components/library-shelf.tsx +++ b/examples/audiobook-curator/src/components/library-shelf.tsx @@ -90,36 +90,6 @@ export const AuditFileCards = ({ receipt }: AuditShelfProps) => ( ); -export const AuditShelf = ({ receipt }: AuditShelfProps) => { - return ( - <> - - - {receipt.duplicateCandidates.slice(0, 10).map((group) => ( - - ))} - {receipt.multipartCandidates.slice(0, 10).map((group) => ( - `part ${String(file.part)} ${file.path}`)} - identityKey={group.identityKey} - key={`${group.directory}/${group.identityKey}`} - kind="multipart" - reviewNote={receipt.reviewNote} - /> - ))} - {receipt.duplicateCandidates.length === 0 && receipt.multipartCandidates.length === 0 - ? {receipt.reviewNote} - : null} - - ); -}; - export interface SelectionShelfProps { readonly receipt: SelectionReceipt; } diff --git a/examples/audiobook-curator/src/state.ts b/examples/audiobook-curator/src/state.ts index f2d660cd5..bd6363b2f 100644 --- a/examples/audiobook-curator/src/state.ts +++ b/examples/audiobook-curator/src/state.ts @@ -24,8 +24,6 @@ export const CurationShelfStateSchema = z.object({ selections: z.array(ShelfSelectionSchema), }).strict(); -export type ShelfSelection = z.output; -export type ShelfMutation = z.output; export type CurationShelfState = z.output; export const curationShelfEventSchemas = { diff --git a/examples/rsc-agent-runtime/src/runtime/contracts.ts b/examples/rsc-agent-runtime/src/runtime/contracts.ts index a69998996..e103ce5b4 100644 --- a/examples/rsc-agent-runtime/src/runtime/contracts.ts +++ b/examples/rsc-agent-runtime/src/runtime/contracts.ts @@ -113,7 +113,6 @@ export interface DevRuntimeInspectionResponse { readonly inspection: DevRuntimeInspectionEnvelope; } -export type McpTimeline = RuntimeSnapshot; export interface ToolAnnotations { readOnlyHint: boolean; diff --git a/examples/rsc-agent-runtime/src/runtime/state-definition.ts b/examples/rsc-agent-runtime/src/runtime/state-definition.ts index 4fcd00e69..91b488c36 100644 --- a/examples/rsc-agent-runtime/src/runtime/state-definition.ts +++ b/examples/rsc-agent-runtime/src/runtime/state-definition.ts @@ -31,7 +31,6 @@ export const RecordedEditSchema = z }) .strict(); -export type RecordedEdit = z.output; const JsonValueSchema: z.ZodType = z.lazy(() => z.union([ diff --git a/examples/worktree-proximity/src/state.ts b/examples/worktree-proximity/src/state.ts index 0f51e26af..1bee532b1 100644 --- a/examples/worktree-proximity/src/state.ts +++ b/examples/worktree-proximity/src/state.ts @@ -57,7 +57,6 @@ export const TopologyStateSchema = z export type Actor = z.output; export type Activity = z.output; -export type EdgeRefusal = z.output; export type TopologyState = z.output; const actorObservedSchema = ActorSchema.omit({ worktreeRoot: true }).strict(); diff --git a/packages/agent-bundle/src/config/conventional-entry.ts b/packages/agent-bundle/src/config/conventional-entry.ts new file mode 100644 index 000000000..bab75abea --- /dev/null +++ b/packages/agent-bundle/src/config/conventional-entry.ts @@ -0,0 +1,22 @@ +import { existsSync, statSync } from 'node:fs'; +import { resolve } from 'node:path'; + +const conventionalEntryExtensions = ['.ts', '.tsx'] as const; + +/** + * Probe for a conventional entry source file. A leaf module so both + * config/normalize.ts and routes/graph.ts can share one rule without closing + * the discover.ts -> routes/graph.ts -> normalize.ts -> discover.ts cycle. + */ +export const conventionalEntryAt = (root: string, ...segments: string[]): string | undefined => { + const stem = resolve(root, ...segments); + for (const extension of conventionalEntryExtensions) { + const candidate = `${stem}${extension}`; + try { + if (existsSync(candidate) && statSync(candidate).isFile()) return candidate; + } catch { + // A racing deletion means the convention does not apply. + } + } + return undefined; +}; diff --git a/packages/agent-bundle/src/config/normalize.ts b/packages/agent-bundle/src/config/normalize.ts index aef037c87..a0a113a3e 100644 --- a/packages/agent-bundle/src/config/normalize.ts +++ b/packages/agent-bundle/src/config/normalize.ts @@ -17,6 +17,7 @@ import { } from '../core/runtime.ts'; import { developmentFallbackVersion, snapshotPackageIdentity } from '../core/project-context.ts'; import { isRecord } from '../core/strict-json.ts'; +import { conventionalEntryAt } from './conventional-entry.ts'; import { canonicalHookEvents, isPrebuiltEntryInput, @@ -155,21 +156,6 @@ const mcpEntryName = (name: string): string => { /** Anchored alias contract for generated target-local MCP entry modules. */ export const mcpEntryAliasPattern = /^mcp\/(mcp-[a-z0-9-]+-[a-f\d]{8}\.mjs)$/u; -const conventionalEntryExtensions = ['.ts', '.tsx'] as const; - -const conventionalEntryAt = (root: string, ...segments: string[]): string | undefined => { - const stem = resolve(root, ...segments); - for (const extension of conventionalEntryExtensions) { - const candidate = `${stem}${extension}`; - try { - if (existsSync(candidate) && statSync(candidate).isFile()) return candidate; - } catch { - // A racing deletion means the convention does not apply. - } - } - return undefined; -}; - /** * The `src/mcp/.ts` convention: the stdio entry for a declared MCP * server that names no entry, command, or url. Config always wins — an diff --git a/packages/agent-bundle/src/contracts/dev-logs.ts b/packages/agent-bundle/src/contracts/dev-logs.ts index 433499dc7..50f6a0a23 100644 --- a/packages/agent-bundle/src/contracts/dev-logs.ts +++ b/packages/agent-bundle/src/contracts/dev-logs.ts @@ -3,5 +3,5 @@ * vocabulary (kinds, levels, producers) is dependency-free runtime code; * the record shapes are type-only because the log service touches Node. */ -export { devLogKinds, devLogLevels, devLogProducers, hasControlOrSeparators } from '../dev/logs/dev-log-kinds.ts'; +export { devLogKinds, devLogLevels, devLogProducers, hasControlOrSeparators, safeContextKeys } from '../dev/logs/dev-log-kinds.ts'; export type { DevLogMessage, DevLogRecord, DevLogReplay, DevLogReplayGap } from '../dev/logs/dev-log-service.ts'; diff --git a/packages/agent-bundle/src/dev/agent-api-wire.ts b/packages/agent-bundle/src/dev/agent-api-wire.ts deleted file mode 100644 index d543543d4..000000000 --- a/packages/agent-bundle/src/dev/agent-api-wire.ts +++ /dev/null @@ -1,340 +0,0 @@ -// Typed wire projections for the Agent API. -// -// Each exported projection maps an already-typed internal value onto the exact -// DTO permitted on the wire, so the compiler verifies what can reach the wire. -// Redaction still fails closed at runtime: every input is detached through an -// accessor-free strict-JSON snapshot and every field must pass an allowlist -// validator before it is copied, so the wire stays path-free and secret-free -// even when a caller's value violates its compile-time type. - -import { isRecord, snapshotStrictJsonValue } from '../core/strict-json.ts'; -import type { EvalRunRecord } from '../eval/run-store.ts'; -import type { ProjectStatus } from './types.ts'; - -/** Deliberately path-free epoch identity permitted on the Agent API wire. */ -export interface AgentApiEpochSummary { - readonly configDigest?: string; - readonly createdAt?: string; - readonly diagnostics?: Readonly<{ readonly errors: number; readonly infos: number; readonly warnings: number }>; - readonly id: string; - readonly modelDigest?: string; - readonly projectRevision?: string; - readonly targetDigests?: Readonly>; -} - -export interface AgentApiDiagnostic { - readonly code: string; - readonly message: string; - readonly recovery?: string; - readonly severity: 'error' | 'info' | 'warning'; - readonly target?: string; -} - -export interface AgentApiRunningBuildAttempt { - readonly diagnostics: readonly AgentApiDiagnostic[]; - readonly id: string; - readonly outcome: 'running'; - readonly sourceRevision: string; - readonly startedAt: string; -} - -export interface AgentApiCompletedBuildAttempt { - readonly completedAt: string; - readonly diagnostics: readonly AgentApiDiagnostic[]; - readonly id: string; - readonly outcome: 'failed' | 'succeeded'; - /** Present only for a succeeded attempt whose epoch identity is safe to name. */ - readonly result?: Readonly<{ readonly epoch: AgentApiEpochSummary }>; - readonly sourceRevision: string; - readonly startedAt: string; -} - -export type AgentApiBuildAttempt = AgentApiCompletedBuildAttempt | AgentApiRunningBuildAttempt; - -export type AgentApiArtifactStatus = - | Readonly<{ readonly state: 'missing' }> - | Readonly<{ - readonly activeEpoch?: AgentApiEpochSummary; - readonly currentSourceRevision?: string; - readonly state: 'active' | 'stale'; - }>; - -export type AgentApiBuildStatus = - | Readonly<{ - readonly activeAttempt: AgentApiBuildAttempt; - readonly lastAttempt?: AgentApiBuildAttempt; - readonly state: 'building'; - }> - | Readonly<{ readonly lastAttempt: AgentApiBuildAttempt; readonly state: 'failed' }> - | Readonly<{ readonly lastAttempt?: AgentApiBuildAttempt; readonly state: 'idle' }>; - -export interface AgentApiSourceStatus { - readonly diagnostics: readonly AgentApiDiagnostic[]; - readonly packageName?: string; - readonly packageVersion?: string; - readonly revision?: string; - readonly state: 'invalid' | 'ready' | 'unknown'; -} - -export interface AgentApiProjectStatus { - readonly artifact: AgentApiArtifactStatus; - readonly build: AgentApiBuildStatus; - readonly source: AgentApiSourceStatus; -} - -/** Durable, path-free acknowledgement returned when an eval background job is admitted. */ -export interface AgentApiEvalRunAdmission { - readonly id: string; - readonly status: 'admitted'; -} - -type AgentApiJsonRecord = Readonly>; - -const maximumDiagnosticTextLength = 4_096; -const safeDiagnosticCodePattern = /^[a-z0-9][a-z0-9._-]{0,127}$/iu; -const safeDigestPattern = /^[a-f0-9]{64}$/iu; -const safeEpochIdPattern = /^[a-z0-9][a-z0-9._-]{0,127}$/iu; -const safeTargetPattern = /^[a-z0-9][a-z0-9._-]{0,127}$/iu; -const safeTimestampPattern = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$/u; -const secretAssignmentPattern = /\b(?:api[_ -]?key|authorization|password|secret|token)\s*(?:=|:)/iu; -const diagnosticMessageFallback = 'Diagnostic details are available in the local workbench.'; -const diagnosticRecoveryFallback = 'Recovery guidance is available in the local workbench.'; - -const wireError = (code: string, message: string): Error & Readonly<{ readonly code: string }> => - Object.assign(new Error(message), { code }); - -const hasControlCharacter = (value: string): boolean => { - for (let index = 0; index < value.length; index += 1) { - const code = value.charCodeAt(index); - if (code <= 0x1f || (code >= 0x7f && code <= 0x9f)) return true; - } - return false; -}; - -const snapshotValue = (value: unknown): unknown => { - try { - return snapshotStrictJsonValue(value); - } catch { - return undefined; - } -}; - -const snapshotRecord = (value: unknown): AgentApiJsonRecord | undefined => { - const snapshot = snapshotValue(value); - return isRecord(snapshot) ? snapshot as AgentApiJsonRecord : undefined; -}; - -const snapshotArray = (value: unknown): readonly unknown[] | undefined => { - const snapshot = snapshotValue(value); - return Array.isArray(snapshot) ? snapshot : undefined; -}; - -const safeDigest = (value: unknown): string | undefined => - typeof value === 'string' && safeDigestPattern.test(value) ? value : undefined; - -const safeDiagnosticCode = (value: unknown): string | undefined => - typeof value === 'string' && safeDiagnosticCodePattern.test(value) ? value : undefined; - -const safeEpochId = (value: unknown): string | undefined => - typeof value === 'string' && safeEpochIdPattern.test(value) ? value : undefined; - -const safeTarget = (value: unknown): string | undefined => - typeof value === 'string' && safeTargetPattern.test(value) ? value : undefined; - -const safeTimestamp = (value: unknown): string | undefined => - typeof value === 'string' && safeTimestampPattern.test(value) && Number.isFinite(Date.parse(value)) - ? value - : undefined; - -/** Deliberately excludes run provenance, artifact bindings, and later execution results. */ -export const evalRunAdmissionWireDto = (run: EvalRunRecord): AgentApiEvalRunAdmission => { - const record = snapshotRecord(run); - const id = safeEpochId(record?.id); - if (id === undefined) { - throw wireError('AGENT_API_OPERATION_FAILED', 'Eval admission did not return a durable run identity.'); - } - return Object.freeze({ id, status: 'admitted' }); -}; - -/** Messages fail closed: any path-like, control, or secret-assignment text is never partially redacted. */ -const safeDiagnosticText = (value: unknown, fallback: string): string => - typeof value === 'string' && value.length <= maximumDiagnosticTextLength && - !value.includes('/') && !value.includes('\\') && !hasControlCharacter(value) && - !secretAssignmentPattern.test(value) - ? value - : fallback; - -/** Dedicated, detached DTO: only diagnostic fields that can be safely named reach the wire. */ -const diagnosticWireDto = (value: unknown): AgentApiDiagnostic | undefined => { - const diagnostic = snapshotRecord(value); - if (diagnostic === undefined) return undefined; - const code = safeDiagnosticCode(diagnostic.code); - const severity = diagnostic.severity; - if (code === undefined || (severity !== 'error' && severity !== 'info' && severity !== 'warning')) return undefined; - const recovery = typeof diagnostic.recovery === 'string' - ? safeDiagnosticText(diagnostic.recovery, diagnosticRecoveryFallback) - : undefined; - const target = safeTarget(diagnostic.target); - return Object.freeze({ - code, - message: safeDiagnosticText(diagnostic.message, diagnosticMessageFallback), - ...(recovery === undefined ? {} : { recovery }), - severity, - ...(target === undefined ? {} : { target }), - }); -}; - -const diagnosticWireDtos = (value: unknown): readonly AgentApiDiagnostic[] => Object.freeze( - (snapshotArray(value) ?? []).flatMap((diagnostic) => { - const projected = diagnosticWireDto(diagnostic); - return projected === undefined ? [] : [projected]; - }), -); - -const diagnosticSummaryWireDto = (value: unknown): AgentApiEpochSummary['diagnostics'] | undefined => { - const summary = snapshotRecord(value); - if (summary === undefined) return undefined; - const errors = summary.errors; - const infos = summary.infos; - const warnings = summary.warnings; - if (![errors, infos, warnings].every((count) => Number.isSafeInteger(count) && (count as number) >= 0)) return undefined; - return Object.freeze({ errors: errors as number, infos: infos as number, warnings: warnings as number }); -}; - -const targetDigestsWireDto = (value: unknown): Readonly> | undefined => { - const targetDigests = snapshotRecord(value); - if (targetDigests === undefined) return undefined; - const entries = Object.entries(targetDigests); - if (entries.length === 0 || entries.some(([target, digest]) => safeTarget(target) === undefined || safeDigest(digest) === undefined)) { - return undefined; - } - return Object.freeze(Object.fromEntries(entries.map(([target, digest]) => [target, digest as string]))); -}; - -/** Explicit safe epoch identity; manifest/root/source fields are intentionally not represented. */ -const epochWireIdentity = (value: unknown): AgentApiEpochSummary | undefined => { - const epoch = snapshotRecord(value); - if (epoch === undefined) return undefined; - const id = safeEpochId(epoch.id); - if (id === undefined) return undefined; - const configDigest = safeDigest(epoch.configDigest); - const createdAt = safeTimestamp(epoch.createdAt); - const diagnostics = diagnosticSummaryWireDto(epoch.diagnostics); - const modelDigest = safeDigest(epoch.modelDigest); - const projectRevision = safeDigest(epoch.projectRevision); - const targetDigests = targetDigestsWireDto(epoch.targetDigests); - return Object.freeze({ - ...(configDigest === undefined ? {} : { configDigest }), - ...(createdAt === undefined ? {} : { createdAt }), - ...(diagnostics === undefined ? {} : { diagnostics }), - id, - ...(modelDigest === undefined ? {} : { modelDigest }), - ...(projectRevision === undefined ? {} : { projectRevision }), - ...(targetDigests === undefined ? {} : { targetDigests }), - }); -}; - -export const epochWireIdentities = (epochs: readonly AgentApiEpochSummary[]): readonly AgentApiEpochSummary[] => Object.freeze( - (snapshotArray(epochs) ?? []).flatMap((epoch) => { - const projected = epochWireIdentity(epoch); - return projected === undefined ? [] : [projected]; - }), -); - -const sourceWireDto = (value: unknown): AgentApiSourceStatus => { - const source = snapshotRecord(value); - const state = source?.state; - const revision = safeDigest(source?.revision); - const packageName = typeof source?.packageName === 'string' && source.packageName.length > 0 - ? source.packageName - : undefined; - const packageVersion = typeof source?.packageVersion === 'string' && source.packageVersion.length > 0 - ? source.packageVersion - : undefined; - return Object.freeze({ - diagnostics: diagnosticWireDtos(source?.diagnostics), - ...(packageName === undefined ? {} : { packageName }), - ...(packageVersion === undefined ? {} : { packageVersion }), - ...(revision === undefined ? {} : { revision }), - state: state === 'invalid' || state === 'ready' || state === 'unknown' ? state : 'unknown', - }); -}; - -const buildAttemptWireDto = (value: unknown): AgentApiBuildAttempt | undefined => { - const attempt = snapshotRecord(value); - if (attempt === undefined) return undefined; - const outcome = attempt.outcome; - const id = safeEpochId(attempt.id); - const sourceRevision = safeDigest(attempt.sourceRevision); - const startedAt = safeTimestamp(attempt.startedAt); - if (id === undefined || sourceRevision === undefined || startedAt === undefined || - (outcome !== 'failed' && outcome !== 'running' && outcome !== 'succeeded')) return undefined; - const completedAt = safeTimestamp(attempt.completedAt); - if (outcome === 'running') { - return Object.freeze({ diagnostics: diagnosticWireDtos(attempt.diagnostics), id, outcome, sourceRevision, startedAt }); - } - if (completedAt === undefined) return undefined; - const result = snapshotRecord(attempt.result); - const epoch = epochWireIdentity(result?.epoch); - return Object.freeze({ - completedAt, - diagnostics: diagnosticWireDtos(attempt.diagnostics), - id, - outcome, - ...(outcome === 'succeeded' && epoch !== undefined ? { result: Object.freeze({ epoch }) } : {}), - sourceRevision, - startedAt, - }); -}; - -const artifactWireDto = (value: unknown): AgentApiArtifactStatus => { - const artifact = snapshotRecord(value); - const state = artifact?.state; - if (artifact === undefined || (state !== 'active' && state !== 'stale')) return Object.freeze({ state: 'missing' }); - const activeEpoch = epochWireIdentity(artifact.activeEpoch); - const currentSourceRevision = safeDigest(artifact.currentSourceRevision); - return Object.freeze({ - ...(activeEpoch === undefined ? {} : { activeEpoch }), - ...(currentSourceRevision === undefined ? {} : { currentSourceRevision }), - state, - }); -}; - -const buildWireDto = (value: unknown): AgentApiBuildStatus => { - const build = snapshotRecord(value); - const state = build?.state; - const activeAttempt = buildAttemptWireDto(build?.activeAttempt); - const lastAttempt = buildAttemptWireDto(build?.lastAttempt); - if (state === 'building' && activeAttempt !== undefined) { - return Object.freeze({ activeAttempt, ...(lastAttempt === undefined ? {} : { lastAttempt }), state }); - } - if (state === 'failed' && lastAttempt !== undefined) return Object.freeze({ lastAttempt, state }); - return Object.freeze({ ...(lastAttempt === undefined ? {} : { lastAttempt }), state: 'idle' }); -}; - -/** Explicit status DTO that carries only safe state, epoch identity, and projected diagnostics. */ -export const projectStatusWireDto = (status: ProjectStatus): AgentApiProjectStatus => { - const record = snapshotRecord(status); - return Object.freeze({ - artifact: artifactWireDto(record?.artifact), - build: buildWireDto(record?.build), - source: sourceWireDto(record?.source), - }); -}; - -/** Flattens only known diagnostic arrays from a direct service result or a ProjectStatus-shaped result. */ -export const diagnosticsListWireDto = (value: unknown): readonly AgentApiDiagnostic[] => { - const result = snapshotRecord(value); - if (result === undefined) return Object.freeze([]); - const direct = snapshotArray(result.diagnostics); - if (direct !== undefined) return diagnosticWireDtos(direct); - const source = snapshotRecord(result.source); - const build = snapshotRecord(result.build); - const activeAttempt = snapshotRecord(build?.activeAttempt); - const lastAttempt = snapshotRecord(build?.lastAttempt); - return Object.freeze([ - ...diagnosticWireDtos(source?.diagnostics), - ...diagnosticWireDtos(activeAttempt?.diagnostics), - ...diagnosticWireDtos(lastAttempt?.diagnostics), - ]); -}; diff --git a/packages/agent-bundle/src/dev/epoch-staging.ts b/packages/agent-bundle/src/dev/epoch-staging.ts deleted file mode 100644 index f847a4ceb..000000000 --- a/packages/agent-bundle/src/dev/epoch-staging.ts +++ /dev/null @@ -1,343 +0,0 @@ -import { randomUUID } from 'node:crypto'; -import { lstat, readdir, realpath, rename, rm, type open } from 'node:fs/promises'; -import { join } from 'node:path'; - -import { mapConcurrent } from '../core/async.ts'; -import { stableJson } from '../core/digest.ts'; -import { readPinnedFile, syncPath, writeNewPinnedFile } from '../core/durable-fs.ts'; -import { CodedError, isErrno } from '../core/errors.ts'; -import { isInside } from '../core/paths.ts'; -import { freezeArtifactEpoch, type ArtifactEpoch } from './types.ts'; - -/** - * Staging publication/recovery path for the epoch store. - * - * The store error vocabulary lives here (epoch-store.ts re-exports it) so the - * dependency between the store and this module stays one-way: the staging - * path constructs these errors, and epoch-store.ts imports this sibling. - */ - -export type EpochStoreErrorCode = - | 'EPOCH_ALREADY_EXISTS' - | 'EPOCH_ID_INVALID' - | 'EPOCH_MANIFEST_INVALID' - | 'EPOCH_METADATA_INVALID' - | 'EPOCH_NOT_FOUND' - | 'EPOCH_STAGING_CLOSED' - | 'EPOCH_STAGING_INVALID' - | 'EPOCH_TARGET_INVALID' - | 'EPOCH_TARGET_SET_INVALID'; - -export class EpochStoreError extends CodedError { - constructor(code: EpochStoreErrorCode, message: string) { - super('EpochStoreError', code, message); - } -} - -export class EpochPostCommitCleanupError extends Error { - readonly committedEpoch: ArtifactEpoch; - - constructor(committedEpoch: ArtifactEpoch, cleanupError: unknown) { - super('Epoch publication committed, but retention cleanup failed.', { cause: cleanupError }); - this.name = 'EpochPostCommitCleanupError'; - this.committedEpoch = freezeArtifactEpoch(committedEpoch); - Object.freeze(this); - } -} - -export interface EpochDurabilityStorage { - readonly open: typeof open; - readonly remove: typeof rm; -} - -export type StagingValidator = (stagingRoot: string) => Promise; - -/** A publication-bound resource is rolled back only by the publisher that created it. */ -export interface EpochPublicationReceipt { - rollback(): Promise; -} - -export type EpochPreActivation = (epoch: ArtifactEpoch) => Promise; - -/** Opaque, store-created staging root that can be published at most once. */ -export interface EpochStaging { - close(): Promise; - publish(validate: StagingValidator, beforeActivate?: EpochPreActivation): Promise; - readonly root: string; -} - -export interface StagingRecord { - readonly epoch: ArtifactEpoch; - readonly markerContents: string; - readonly root: string; - readonly rootDevice: number; - readonly rootInode: number; - readonly targets: readonly string[]; -} - -/** - * Store-private seams the staging path needs, passed explicitly by the store - * instead of exporting its private internals. - */ -export interface EpochStagingContext { - /** Retention pass run under the already-held lease after a committed publish. */ - readonly cleanupUnderLease: () => Promise; - readonly durabilityStorage: EpochDurabilityStorage; - readonly epochsPath: string; - readonly manifestRelativePath: (epoch: ArtifactEpoch) => string; - readonly metadataPathFor: (epochId: string) => string; - /** Invalidates the store's active-epoch cache and marks retention dirty. */ - readonly onCommitted: () => void; - readonly runLeaseTransition: (operation: () => Promise) => Promise; - readonly writeActiveMetadata: (epoch: ArtifactEpoch) => Promise; - readonly writeEpochMetadata: (epoch: ArtifactEpoch) => Promise; -} - -const stagingMarkerFileName = '.agent-bundle-epoch-stage.json'; -const stagingMarkerMaximumBytes = 1024; -export const stagingPrefix = '.stage-'; -const syncTreeConcurrency = 16; - -const pathExists = async (path: string): Promise => { - try { - await lstat(path); - return true; - } catch (error) { - if (isErrno(error, 'ENOENT')) return false; - throw error; - } -}; - -export class EpochStagingHandle implements EpochStaging { - readonly #close: () => Promise; - readonly #publish: (validate: StagingValidator, beforeActivate?: EpochPreActivation) => Promise; - #closed = false; - - constructor( - root: string, - publish: (validate: StagingValidator, beforeActivate?: EpochPreActivation) => Promise, - close: () => Promise, - ) { - this.root = root; - this.#publish = publish; - this.#close = close; - } - - readonly root: string; - - async close(): Promise { - if (this.#closed) return; - this.#closed = true; - await this.#close(); - } - - async publish(validate: StagingValidator, beforeActivate?: EpochPreActivation): Promise { - if (this.#closed) { - throw new EpochStoreError('EPOCH_STAGING_CLOSED', 'Epoch staging is already closed.'); - } - this.#closed = true; - return this.#publish(validate, beforeActivate); - } -} - -export const syncDurablePath = async (storage: EpochDurabilityStorage, path: string, directory = false): Promise => { - await syncPath(path, { directory, open: storage.open }); -}; - -/** Writes the store-owned staging marker and returns its exact contents. */ -export const createStagingMarker = async (root: string): Promise => { - const markerContents = `${stableJson({ token: randomUUID() })}\n`; - await writeNewPinnedFile(join(root, stagingMarkerFileName), markerContents, { - invalid: () => - new EpochStoreError('EPOCH_STAGING_INVALID', 'The store-created staging marker could not be created safely.'), - }); - return markerContents; -}; - -/** Removes leftover staging directories from crashed or interrupted publishes. */ -export const removeStagingRemnants = async (epochsPath: string): Promise => { - let entries; - try { - entries = await readdir(epochsPath, { withFileTypes: true }); - } catch (error) { - if (isErrno(error, 'ENOENT')) return; - throw error; - } - await Promise.all( - entries - .filter((entry) => entry.isDirectory() && entry.name.startsWith(stagingPrefix)) - .map((entry) => rm(join(epochsPath, entry.name), { force: true, recursive: true })), - ); -}; - -const removeStagingMarker = async (storage: EpochDurabilityStorage, record: StagingRecord): Promise => { - const markerPath = join(record.root, stagingMarkerFileName); - await syncDurablePath(storage, markerPath); - await storage.remove(markerPath); - await syncDurablePath(storage, record.root, true); -}; - -const syncStagedTree = async (storage: EpochDurabilityStorage, path: string): Promise => { - const metadata = await lstat(path); - if (metadata.isSymbolicLink()) { - throw new EpochStoreError('EPOCH_STAGING_INVALID', 'Staged epoch contents must not contain symbolic links.'); - } - if (metadata.isFile()) { - await syncDurablePath(storage, path); - return; - } - if (!metadata.isDirectory()) { - throw new EpochStoreError('EPOCH_STAGING_INVALID', 'Staged epoch contents must be regular files or directories.'); - } - const entries = await readdir(path, { withFileTypes: true }); - const directories: string[] = []; - const files: string[] = []; - for (const entry of entries.sort((left, right) => left.name < right.name ? -1 : left.name > right.name ? 1 : 0)) { - const child = join(path, entry.name); - if (entry.isDirectory() && !entry.isSymbolicLink()) directories.push(child); - else files.push(child); - } - await Promise.all([ - mapConcurrent(directories, syncTreeConcurrency, (child) => syncStagedTree(storage, child)), - mapConcurrent(files, syncTreeConcurrency, (child) => syncStagedTree(storage, child)), - ]); - await syncDurablePath(storage, path, true); -}; - -export const verifyStaging = async (context: EpochStagingContext, record: StagingRecord): Promise => { - let rootMetadata; - try { - rootMetadata = await lstat(record.root); - } catch (error) { - if (isErrno(error, 'ENOENT')) { - throw new EpochStoreError('EPOCH_STAGING_INVALID', 'The store-created staging root no longer exists.'); - } - throw error; - } - if ( - !rootMetadata.isDirectory() || - rootMetadata.isSymbolicLink() || - rootMetadata.dev !== record.rootDevice || - rootMetadata.ino !== record.rootInode - ) { - throw new EpochStoreError('EPOCH_STAGING_INVALID', 'The store-created staging root was replaced.'); - } - const markerReplaced = (): EpochStoreError => - new EpochStoreError('EPOCH_STAGING_INVALID', 'The store-created staging marker was replaced.'); - let markerContents: string; - try { - markerContents = await readPinnedFile(join(record.root, stagingMarkerFileName), { - changedWhileOpening: markerReplaced, - changedWhileReading: markerReplaced, - maximumBytes: stagingMarkerMaximumBytes, - unsafe: markerReplaced, - }); - } catch (error) { - if (isErrno(error, 'ENOENT')) { - throw new EpochStoreError('EPOCH_STAGING_INVALID', 'The store-created staging marker is missing.'); - } - throw error; - } - if (markerContents !== record.markerContents) throw markerReplaced(); - - const [epochsRoot, stagingRoot] = await Promise.all([ - realpath(context.epochsPath), - realpath(record.root), - ]); - if (!isInside(epochsRoot, stagingRoot)) { - throw new EpochStoreError('EPOCH_STAGING_INVALID', 'The staging root escapes the epoch store.'); - } - - for (const target of record.targets) { - const targetPath = join(record.root, target); - let targetMetadata; - try { - targetMetadata = await lstat(targetPath); - } catch (error) { - if (isErrno(error, 'ENOENT')) { - throw new EpochStoreError( - 'EPOCH_STAGING_INVALID', - `Staged epoch is missing selected target ${JSON.stringify(target)}.`, - ); - } - throw error; - } - if (!targetMetadata.isDirectory() || targetMetadata.isSymbolicLink()) { - throw new EpochStoreError( - 'EPOCH_STAGING_INVALID', - `Staged epoch target ${JSON.stringify(target)} must be a contained non-symlink directory.`, - ); - } - if (!isInside(stagingRoot, await realpath(targetPath))) { - throw new EpochStoreError( - 'EPOCH_STAGING_INVALID', - `Staged epoch target ${JSON.stringify(target)} escapes the staging root.`, - ); - } - } - - const manifestPath = join(record.root, context.manifestRelativePath(record.epoch)); - let manifestMetadata; - try { - manifestMetadata = await lstat(manifestPath); - } catch (error) { - if (isErrno(error, 'ENOENT')) { - throw new EpochStoreError('EPOCH_MANIFEST_INVALID', 'Staged epoch manifest is missing.'); - } - throw error; - } - if (!manifestMetadata.isFile() || manifestMetadata.isSymbolicLink() || !isInside(stagingRoot, await realpath(manifestPath))) { - throw new EpochStoreError('EPOCH_MANIFEST_INVALID', 'Staged epoch manifest must be a contained non-symlink file.'); - } -}; - -export const publishVerifiedStaging = async ( - context: EpochStagingContext, - record: StagingRecord, - beforeActivate: EpochPreActivation | undefined, -): Promise => { - const epochRoot = join(context.epochsPath, record.epoch.id); - if (await pathExists(epochRoot)) { - throw new EpochStoreError('EPOCH_ALREADY_EXISTS', `Epoch ${JSON.stringify(record.epoch.id)} already exists.`); - } - - let publication: EpochPublicationReceipt | undefined; - if (beforeActivate !== undefined) { - publication = (await beforeActivate(record.epoch)) ?? undefined; - } - return context.runLeaseTransition(async () => { - let moved = false; - try { - if (await pathExists(epochRoot)) { - throw new EpochStoreError('EPOCH_ALREADY_EXISTS', `Epoch ${JSON.stringify(record.epoch.id)} already exists.`); - } - await syncStagedTree(context.durabilityStorage, record.root); - await removeStagingMarker(context.durabilityStorage, record); - await rename(record.root, epochRoot); - moved = true; - await syncDurablePath(context.durabilityStorage, context.epochsPath, true); - await context.writeEpochMetadata(record.epoch); - await context.writeActiveMetadata(record.epoch); - context.onCommitted(); - } catch (error) { - const cleanupResults = await Promise.allSettled([ - ...(moved ? [ - rm(epochRoot, { force: true, recursive: true }), - rm(context.metadataPathFor(record.epoch.id), { force: true }), - ] : []), - ...(publication === undefined ? [] : [publication.rollback()]), - ]); - const cleanupFailures = cleanupResults.flatMap((result) => result.status === 'rejected' ? [result.reason] : []); - if (cleanupFailures.length > 0) { - throw new AggregateError([error, ...cleanupFailures], 'Epoch publication and rollback both failed.', { cause: error }); - } - throw error; - } - try { - await context.cleanupUnderLease(); - } catch (error) { - throw new EpochPostCommitCleanupError(record.epoch, error); - } - return record.epoch; - }); -}; diff --git a/packages/agent-bundle/src/dev/eval/eval-artifact-reader.ts b/packages/agent-bundle/src/dev/eval/eval-artifact-reader.ts deleted file mode 100644 index 6602f18a2..000000000 --- a/packages/agent-bundle/src/dev/eval/eval-artifact-reader.ts +++ /dev/null @@ -1,127 +0,0 @@ -import { constants } from 'node:fs'; -import { lstat, open, realpath } from 'node:fs/promises'; -import { basename, join, relative, resolve } from 'node:path'; -import { Readable } from 'node:stream'; - -import { sha256Hex } from '../../core/digest.ts'; -import { isInsideOrEqual, isSafePathSegment, sameFile } from '../../core/paths.ts'; -import type { EvalArtifactReader } from './eval-service-types.ts'; - -const maximumArtifactBytes = 8 * 1024 * 1024; - -export const artifactSegments = (value: unknown): readonly string[] | undefined => { - if (typeof value !== 'string' || /%(?:2f|5c)/iu.test(value) || value.includes('\\') || value.includes('\0')) { - return undefined; - } - const segments = value.split('/'); - if ( - segments.length < 2 || segments[0] !== 'artifacts' || - segments.some((segment) => !isSafePathSegment(segment)) - ) return undefined; - return Object.freeze(segments); -}; - -export const assertNoSymlinkedArtifactPath = async (projectRoot: string, target: string): Promise => { - const root = resolve(projectRoot); - const resolvedTarget = resolve(target); - if (!isInsideOrEqual(root, resolvedTarget)) throw new Error('Raw evidence path escaped the project.'); - const segments = relative(root, resolvedTarget).split(/[/\\]/u); - let current = root; - for (const [index, segment] of segments.entries()) { - current = join(current, segment); - const entry = await lstat(current); - if (entry.isSymbolicLink() || index < segments.length - 1 && !entry.isDirectory()) { - throw new Error('Raw evidence path must contain only real directories and a real file.'); - } - } -}; - -export class OpenedEvalArtifact implements EvalArtifactReader { - readonly digest: string; - readonly filename: string; - readonly ref: string; - readonly size: number; - readonly #bytes: Buffer; - readonly #onClose: () => void; - #closePromise: Promise | undefined; - - constructor(options: { - readonly bytes: Buffer; - readonly digest: string; - readonly filename: string; - readonly onClose: () => void; - readonly ref: string; - readonly size: number; - }) { - this.digest = options.digest; - this.filename = options.filename; - this.#bytes = options.bytes; - this.#onClose = options.onClose; - this.ref = options.ref; - this.size = options.size; - } - - read(start = 0, end = this.size - 1): Readable { - if (this.#closePromise !== undefined) throw new Error('Raw evidence reader is closed.'); - if (this.size === 0 && start === 0 && end === -1) return Readable.from([]); - if (!Number.isSafeInteger(start) || !Number.isSafeInteger(end) || start < 0 || end < start || end >= this.size) { - throw new RangeError('Raw evidence read range is not valid.'); - } - return Readable.from([Buffer.from(this.#bytes.subarray(start, end + 1))]); - } - - close(): Promise { - if (this.#closePromise !== undefined) return this.#closePromise; - this.#closePromise = Promise.resolve().then(() => { this.#onClose(); }); - return this.#closePromise; - } -} - -export const openEvalArtifactSnapshot = async (options: { - readonly directory: string; - readonly onClose: (reader: OpenedEvalArtifact) => void; - readonly projectRoot: string; - readonly ref: string; - readonly segments: readonly string[]; -}): Promise => { - const artifactRoot = join(options.directory, 'artifacts'); - const target = join(options.directory, ...options.segments); - await assertNoSymlinkedArtifactPath(options.projectRoot, target); - const before = await lstat(target); - if (!before.isFile() || before.isSymbolicLink() || before.nlink !== 1 || before.size > maximumArtifactBytes) { - throw new Error('Raw evidence file metadata is not safe.'); - } - const [physicalRoot, physicalTarget] = await Promise.all([realpath(artifactRoot), realpath(target)]); - if (!isInsideOrEqual(physicalRoot, physicalTarget)) throw new Error('Raw evidence file escaped its run artifacts directory.'); - const file = await open(target, constants.O_RDONLY | constants.O_NOFOLLOW); - try { - const [after, descriptor] = await Promise.all([lstat(target), file.stat()]); - if ( - !after.isFile() || after.isSymbolicLink() || after.nlink !== 1 || after.size > maximumArtifactBytes || - !descriptor.isFile() || descriptor.nlink !== 1 || descriptor.size > maximumArtifactBytes || - !sameFile(before, descriptor) || !sameFile(after, descriptor) - ) { - throw new Error('Raw evidence file changed while opening.'); - } - const bytes = Buffer.alloc(Math.min(descriptor.size, maximumArtifactBytes) + 1); - const { bytesRead } = await file.read(bytes, 0, bytes.length, 0); - const final = await file.stat(); - if (!sameFile(descriptor, final) || final.size !== descriptor.size || bytesRead !== descriptor.size || bytesRead > maximumArtifactBytes) { - throw new Error('Raw evidence file changed while hashing.'); - } - const snapshot = Buffer.from(bytes.subarray(0, bytesRead)); - await file.close(); - const reader = new OpenedEvalArtifact({ - bytes: snapshot, - digest: sha256Hex(snapshot), - filename: basename(options.ref), - onClose: () => options.onClose(reader), - ref: options.ref, - size: descriptor.size, - }); - return reader; - } catch (error) { - await file.close().catch(() => undefined); - throw error; - } -}; diff --git a/packages/agent-bundle/src/dev/eval/eval-event-subscription.ts b/packages/agent-bundle/src/dev/eval/eval-event-subscription.ts deleted file mode 100644 index 6adef7bb8..000000000 --- a/packages/agent-bundle/src/dev/eval/eval-event-subscription.ts +++ /dev/null @@ -1,48 +0,0 @@ -import type { EvalRunEvent } from '../../eval/run-store.ts'; -import type { EvalEventSubscription, EvalRunEventsReplay } from './eval-service-types.ts'; - -export class PendingEvalEventSubscription implements EvalEventSubscription { - #closed = false; - #listener: ((event: EvalRunEvent) => void) | undefined; - readonly #onClose: () => void; - #queued: EvalRunEvent[] = []; - #replay: EvalRunEventsReplay | undefined; - - constructor(onClose: () => void) { - this.#onClose = onClose; - } - - get replay(): EvalRunEventsReplay { - if (this.#replay === undefined) throw new Error('Eval event subscription has not finished replaying.'); - return this.#replay; - } - - bind(replay: EvalRunEventsReplay): void { - this.#replay = replay; - this.#queued = this.#queued.filter((event) => event.sequence > replay.cursor.afterSequence); - } - - publish(event: EvalRunEvent): void { - if (this.#closed) return; - if (this.#replay !== undefined && event.sequence <= this.#replay.cursor.afterSequence) return; - const listener = this.#listener; - if (listener === undefined) this.#queued.push(event); - else listener(event); - } - - activate(listener: (event: EvalRunEvent) => void): void { - if (this.#closed || this.#listener !== undefined) return; - this.#listener = listener; - const queued = this.#queued; - this.#queued = []; - for (const event of queued) listener(event); - } - - close(): void { - if (this.#closed) return; - this.#closed = true; - this.#listener = undefined; - this.#queued = []; - this.#onClose(); - } -} diff --git a/packages/agent-bundle/src/dev/eval/eval-routes.ts b/packages/agent-bundle/src/dev/eval/eval-routes.ts index 7701ba5f9..f77a9aca0 100644 --- a/packages/agent-bundle/src/dev/eval/eval-routes.ts +++ b/packages/agent-bundle/src/dev/eval/eval-routes.ts @@ -3,6 +3,18 @@ import type { IncomingMessage, ServerResponse } from 'node:http'; import { Readable } from 'node:stream'; import { hasOnlyOwnKeys, isRecord as coreIsRecord, parseJsonWithoutDuplicateKeys } from '../../core/strict-json.ts'; +import { + diagnostic, + isJsonRequest, + isRequestDiagnostic, + nonemptyString, + rawPathname, + readBody, + requestError, + responseDiagnostic, + responseJson as writeJsonResponse, + type RequestDiagnostic, +} from '../http.ts'; import { EvalConfigError, EvalDefinitionError, @@ -24,16 +36,9 @@ import { type EvalSuiteListing, } from './eval-service.ts'; -const bodyLimit = 64 * 1024; const maximumTrials = 100; const streamByteLimit = 256 * 1024; -interface RequestDiagnostic { - readonly code: string; - readonly message: string; - readonly status: number; -} - type Route = | Readonly<{ readonly artifactRef: string; readonly kind: 'artifact'; readonly runId: string }> | Readonly<{ readonly kind: 'cancel'; readonly runId: string }> @@ -69,19 +74,6 @@ export interface EvalRoutesOptions { readonly service?: EvalRouteService; } -const diagnostic = (code: string, message: string, status: number): RequestDiagnostic => ({ code, message, status }); - -const requestError = (value: RequestDiagnostic): RequestDiagnostic & Error => Object.assign( - new Error(value.message), - value, -); - -const isRequestDiagnostic = (value: unknown): value is RequestDiagnostic => - typeof value === 'object' && value !== null && - typeof (value as Partial).code === 'string' && - typeof (value as Partial).message === 'string' && - typeof (value as Partial).status === 'number'; - /** Service messages name project paths, so each code keeps one fixed browser-facing sentence. */ const serviceDiagnostics: Readonly> = Object.freeze({ EVAL_ARTIFACT_NOT_FOUND: diagnostic('AB8085', 'Recorded raw evidence was not found.', 404), @@ -113,73 +105,9 @@ const authoringDiagnostic = (error: unknown): RequestDiagnostic | undefined => { return undefined; }; -const responseDiagnostic = (response: ServerResponse, value: RequestDiagnostic): void => { - if (response.headersSent || response.writableEnded) { - response.destroy(); - return; - } - response.writeHead(value.status, { 'content-type': 'application/json; charset=utf-8' }); - response.end(JSON.stringify({ diagnostic: { code: value.code, message: value.message } })); -}; - -const responseJson = (response: ServerResponse, body: unknown, status = 200): void => { - if (response.headersSent || response.writableEnded) { - response.destroy(); - return; - } - response.writeHead(status, { 'content-type': 'application/json; charset=utf-8' }); - response.end(JSON.stringify(body)); -}; - const terminalEvent = (event: EvalRunEventsReplay['events'][number]): boolean => event.kind === 'run.cancelled' || event.kind === 'run.completed' || event.kind === 'run.failed'; -const singleHeader = (value: string | readonly string[] | undefined): string | undefined => - typeof value === 'string' ? value : undefined; - -const unquoteHeaderValue = (value: string): string | undefined => { - if (/^[!#$%&'*+\-.^_`|~0-9A-Za-z]+$/u.test(value)) return value; - if (!/^"(?:[^"\\\r\n]|\\[\t !-~])*"$/u.test(value)) return undefined; - return value.slice(1, -1).replace(/\\([\t !-~])/gu, '$1'); -}; - -const isJsonRequest = (request: IncomingMessage): boolean => { - const contentType = singleHeader(request.headers['content-type']); - if (contentType === undefined) return false; - const parts = contentType.split(';').map((part) => part.trim()); - if (parts.shift()?.toLowerCase() !== 'application/json') return false; - if (parts.length === 0) return true; - if (parts.length !== 1) return false; - const parameter = parts[0] ?? ''; - const equals = parameter.indexOf('='); - if (equals < 1 || parameter.slice(0, equals).trim().toLowerCase() !== 'charset') return false; - return unquoteHeaderValue(parameter.slice(equals + 1).trim())?.toLowerCase() === 'utf-8'; -}; - -const readBody = async (request: IncomingMessage): Promise => new Promise((resolvePromise, rejectPromise) => { - let size = 0; - let tooLarge = false; - const chunks: Buffer[] = []; - request.on('data', (chunk: Buffer) => { - size += chunk.length; - if (size > bodyLimit) { - tooLarge = true; - return; - } - if (!tooLarge) chunks.push(chunk); - }); - request.once('end', () => { - if (tooLarge) { - rejectPromise(requestError(diagnostic('AB8010', 'Request body exceeds 64 KiB.', 413))); - return; - } - resolvePromise(Buffer.concat(chunks).toString('utf8')); - }); - request.once('error', rejectPromise); -}); - -const rawPathname = (requestTarget: string | undefined): string => requestTarget?.split(/[?#]/u, 1)[0] ?? ''; - const pathError = (): never => { throw requestError(diagnostic('AB8070', 'Eval route path is not valid.', 400)); }; @@ -245,9 +173,6 @@ const isRecord = coreIsRecord as (value: unknown) => value is JsonObject; const hasOnly: (value: JsonObject, fields: readonly string[]) => boolean = hasOnlyOwnKeys; -const nonemptyString = (value: unknown): value is string => - typeof value === 'string' && value.trim().length > 0 && value.length <= 4_096 && !value.includes('\0'); - const nameList = (value: unknown): readonly string[] => { if (!Array.isArray(value) || value.length === 0 || !value.every(nonemptyString)) return invalidShape(); return Object.freeze([...value]); @@ -436,7 +361,7 @@ export class EvalRoutes { const selection = runRequest(await jsonBody(request)); if (this.#closePromise !== undefined) throw this.#unavailable(503); const admission = await service.start(selection); - return responseJson(response, { run: admission.run }, 202); + return writeJsonResponse(response, { run: admission.run }, { destroyIfEnded: true, status: 202 }); } finally { finishAdmission(); } @@ -448,7 +373,7 @@ export class EvalRoutes { await cancelRequest(request); if (this.#closePromise !== undefined) throw this.#unavailable(503); const cancelled = await service.cancel(parsed.runId); - return responseJson(response, { cancelled, runId: parsed.runId }, 202); + return writeJsonResponse(response, { cancelled, runId: parsed.runId }, { destroyIfEnded: true, status: 202 }); } finally { finishAdmission(); } @@ -458,10 +383,10 @@ export class EvalRoutes { } if (parsed.kind === 'comparisons') { const query = comparisonQuery(request.url); - return responseJson(response, { comparison: await service.compare(query.base, query.candidate) }); + return writeJsonResponse(response, { comparison: await service.compare(query.base, query.candidate) }, { destroyIfEnded: true }); } if (parsed.kind === 'events') { - return responseJson(response, { replay: await service.events(parsed.runId, eventCursor(request.url)) }); + return writeJsonResponse(response, { replay: await service.events(parsed.runId, eventCursor(request.url)) }, { destroyIfEnded: true }); } if (parsed.kind === 'stream') { return this.#stream(response, service, parsed.runId, eventCursor(request.url)); @@ -470,9 +395,9 @@ export class EvalRoutes { return this.#artifact(request, response, service, parsed); } noQuery(request.url); - if (parsed.kind === 'suites') return responseJson(response, await service.suites()); - if (parsed.kind === 'runs') return responseJson(response, { runs: await service.list() }); - return responseJson(response, { run: await service.read(parsed.runId) }); + if (parsed.kind === 'suites') return writeJsonResponse(response, await service.suites(), { destroyIfEnded: true }); + if (parsed.kind === 'runs') return writeJsonResponse(response, { runs: await service.list() }, { destroyIfEnded: true }); + return writeJsonResponse(response, { run: await service.read(parsed.runId) }, { destroyIfEnded: true }); } #unavailable(status: number): Error { diff --git a/packages/agent-bundle/src/dev/eval/eval-service-error.ts b/packages/agent-bundle/src/dev/eval/eval-service-error.ts deleted file mode 100644 index c71f2b656..000000000 --- a/packages/agent-bundle/src/dev/eval/eval-service-error.ts +++ /dev/null @@ -1,23 +0,0 @@ -import { CodedError } from '../../core/errors.ts'; - -export type EvalServiceErrorCode = - | 'EVAL_ARTIFACT_NOT_FOUND' - | 'EVAL_ARTIFACT_OUTSIDE_PROJECT' - | 'EVAL_ARTIFACT_UNAVAILABLE' - | 'EVAL_EVENTS_CURSOR_INVALID' - | 'EVAL_HARNESS_UNSUPPORTED' - | 'EVAL_RUN_NOT_FOUND' - | 'EVAL_SELECTION_EMPTY' - | 'EVAL_SEMANTIC_GRADER_UNSUPPORTED' - | 'EVAL_TARGET_MISSING' - | 'EVAL_TRIALS_INVALID'; - -/** Every refusal a caller can act on without reading the eval internals. */ -export class EvalServiceError extends CodedError { - constructor(code: EvalServiceErrorCode, message: string) { - super('EvalServiceError', code, message); - } -} - -export const evalServiceError = (code: EvalServiceErrorCode, message: string): EvalServiceError => - new EvalServiceError(code, message); diff --git a/packages/agent-bundle/src/dev/eval/eval-service-types.ts b/packages/agent-bundle/src/dev/eval/eval-service-types.ts deleted file mode 100644 index 474f67c89..000000000 --- a/packages/agent-bundle/src/dev/eval/eval-service-types.ts +++ /dev/null @@ -1,114 +0,0 @@ -import type { Readable } from 'node:stream'; - -import type { TargetRegistry } from '../../adapters/registry.ts'; -import type { Diagnostic } from '../../core/diagnostics.ts'; -import type { EvalCaseAggregate } from '../../eval/aggregate.ts'; -import type { CodexCommandRunner } from '../../eval/codex-harness.ts'; -import type { - EvalRunEvent, - EvalRunRecord, - EvalTrialRecord, -} from '../../eval/run-store.ts'; -import type { EvalAssertionKind, EvalInvocation } from '../../eval/types.ts'; -import type { NativeClaudeProcessRunner } from '../../host-contracts/native-claude-contract.ts'; -import type { DevLogSink } from '../logs/dev-log-service.ts'; - -export interface EvalAssertionSummary { - readonly id: string; - readonly kind: EvalAssertionKind; - /** The Skill an activation assertion references; absent for blanket and non-Skill assertions. */ - readonly skill?: string; -} - -export interface EvalCaseSummary { - readonly assertions: readonly EvalAssertionSummary[]; - readonly digest: string; - readonly hosts: readonly string[]; - readonly id: string; - readonly invocation: EvalInvocation; - readonly prompt: string; - readonly trials: number; -} - -export interface EvalSuiteSummary { - readonly cases: readonly EvalCaseSummary[]; - readonly digest: string; - readonly name: string; - /** Project-relative so no caller, browser included, learns an absolute location. */ - readonly sourcePath: string; -} - -export interface EvalSuiteListing { - readonly diagnostics: readonly Diagnostic[]; - readonly suites: readonly EvalSuiteSummary[]; -} - -export interface EvalRunSelection { - readonly caseIds?: readonly string[]; - readonly suites?: readonly string[]; -} - -export interface EvalRunRequest extends EvalRunSelection { - /** An already-built artifact root. Only the API and CLI may name one; the browser never does. */ - readonly artifact?: string; - readonly harness?: string; - readonly signal?: AbortSignal; - readonly trials?: number; -} - -export interface EvalRunResult { - readonly aggregates: readonly EvalCaseAggregate[]; - readonly diagnostics: readonly Diagnostic[]; - readonly run: EvalRunRecord; - readonly trials: readonly EvalTrialRecord[]; -} - -/** A durable run identity returned before its background trials settle. */ -export interface EvalRunAdmission { - readonly run: EvalRunRecord; -} - -/** A durable event replay never includes an incomplete append record. */ -export interface EvalRunEventsReplay { - readonly cursor: Readonly<{ readonly afterSequence: number }>; - readonly events: readonly EvalRunEvent[]; - /** True when the event file ends with an incomplete record that was not replayed. */ - readonly incompleteTrailingRecord?: true; -} - -/** A run-pinned raw-evidence descriptor. Call close when its response stream ends. */ -export interface EvalArtifactReader { - readonly digest: string; - readonly filename: string; - readonly ref: string; - readonly size: number; - close(): Promise; - read(start?: number, end?: number): Readable; -} - -/** A replay snapshot followed by durable, ordered events published after its cursor. */ -export interface EvalEventSubscription { - readonly replay: EvalRunEventsReplay; - activate(listener: (event: EvalRunEvent) => void): void; - close(): void; -} - -export interface EvalServiceOptions { - readonly configPath?: string; - readonly mode?: string; - /** Optional non-throwing producer-wide diagnostics sink. */ - readonly logger?: DevLogSink; - /** Native CLI injection is deliberately limited to test runners and their child environment. */ - readonly native?: EvalServiceNativeOptions; - /** Injectable only to make run identity deterministic in tests. */ - readonly now?: () => Date; - readonly projectRoot: string; - readonly registry?: TargetRegistry; - readonly targets?: readonly string[]; -} - -export interface EvalServiceNativeOptions { - readonly claudeRun?: NativeClaudeProcessRunner; - readonly codexRun?: CodexCommandRunner; - readonly environment?: Readonly; -} diff --git a/packages/agent-bundle/src/dev/foreground-server.ts b/packages/agent-bundle/src/dev/foreground-server.ts index e319a6ebf..3d48e2e76 100644 --- a/packages/agent-bundle/src/dev/foreground-server.ts +++ b/packages/agent-bundle/src/dev/foreground-server.ts @@ -27,8 +27,19 @@ import { PlaygroundRoutes, type PlaygroundRouteService } from './playground/play import { RouteManifestRoutes, type RouteManifestRouteService } from './routes/route-manifest-routes.ts'; import { SkillDocumentError, type SkillDocumentService } from './skill-document-service.ts'; import type { Invalidation, ProjectEventMessage, ProjectStatus } from './types.ts'; +import { + diagnostic, + isJsonRequest, + isRequestDiagnostic, + rawPathname, + readBody, + requestError, + responseDiagnostic as writeDiagnosticResponse, + responseJson as writeJsonResponse, + singleHeader, + type RequestDiagnostic, +} from './http.ts'; -const bodyLimit = 64 * 1024; const instanceIdLengthLimit = 128; const loopbackHosts = new Set(['127.0.0.1', '::1']); const sseQueueByteLimit = 256 * 1024; @@ -166,12 +177,6 @@ export interface ForegroundServerOptions { readonly testing?: ForegroundServerTesting; } -interface RequestDiagnostic { - readonly code: string; - readonly message: string; - readonly status: number; -} - type SkillRoute = | Readonly<{ readonly kind: 'source-tree' }> | Readonly<{ readonly kind: 'source-document'; readonly skillId: string }> @@ -180,49 +185,18 @@ type SkillRoute = | Readonly<{ readonly epochId: string; readonly kind: 'generated-document'; readonly skillId: string; readonly target: string }> | Readonly<{ readonly epochId: string; readonly kind: 'generated-resource'; readonly resource: readonly string[]; readonly skillId: string; readonly target: string }>; -const diagnostic = (code: string, message: string, status: number): RequestDiagnostic => ({ code, message, status }); - -const requestError = (value: RequestDiagnostic): RequestDiagnostic & Error => Object.assign( - new Error(value.message), - value, -); - -const isRequestDiagnostic = (value: unknown): value is RequestDiagnostic => - typeof value === 'object' && value !== null && - typeof (value as Partial).code === 'string' && - typeof (value as Partial).message === 'string' && - typeof (value as Partial).status === 'number'; - /** Route groups may attach structured diagnostics that are the answer, not an internal detail. */ const attachedDiagnostics = (value: RequestDiagnostic): readonly unknown[] | undefined => { const diagnostics = (value as Partial<{ readonly diagnostics: unknown }>).diagnostics; return Array.isArray(diagnostics) ? diagnostics : undefined; }; -const responseDiagnostic = (response: ServerResponse, value: RequestDiagnostic): void => { - if (response.headersSent || response.writableEnded) { - response.destroy(); - return; - } - const diagnostics = attachedDiagnostics(value); - response.writeHead(value.status, { 'content-type': 'application/json; charset=utf-8' }); - response.end(JSON.stringify({ - diagnostic: { code: value.code, message: value.message }, - ...(diagnostics === undefined ? {} : { diagnostics }), - })); -}; - -const responseJson = (response: ServerResponse, body: unknown): void => { - response.writeHead(200, { 'content-type': 'application/json; charset=utf-8' }); - response.end(JSON.stringify(body)); -}; +const responseDiagnostic = (response: ServerResponse, value: RequestDiagnostic): void => + writeDiagnosticResponse(response, value, attachedDiagnostics(value)); const attachmentHeader = (relativePath: string): string => `attachment; filename*=UTF-8''${encodeURIComponent(basename(relativePath)).replaceAll("'", '%27')}`; -const singleHeader = (value: string | readonly string[] | undefined): string | undefined => - typeof value === 'string' ? value : undefined; - const cookieValue = (request: IncomingMessage, name: string): string | undefined => { const header = singleHeader(request.headers.cookie); if (header === undefined) return undefined; @@ -234,49 +208,6 @@ const cookieValue = (request: IncomingMessage, name: string): string | undefined return undefined; }; -const readBody = async (request: IncomingMessage): Promise => new Promise((resolvePromise, rejectPromise) => { - let size = 0; - let tooLarge = false; - const chunks: Buffer[] = []; - request.on('data', (chunk: Buffer) => { - size += chunk.length; - if (size > bodyLimit) { - tooLarge = true; - return; - } - chunks.push(chunk); - }); - request.once('end', () => { - if (tooLarge) { - rejectPromise(requestError(diagnostic('AB8010', 'Request body exceeds 64 KiB.', 413))); - return; - } - resolvePromise(Buffer.concat(chunks).toString('utf8')); - }); - request.once('error', rejectPromise); -}); - -const isJsonRequest = (request: IncomingMessage): boolean => { - const contentType = singleHeader(request.headers['content-type']); - if (contentType === undefined) return false; - const parts = contentType.split(';').map((part) => part.trim()); - if (parts.shift()?.toLowerCase() !== 'application/json') return false; - if (parts.length === 0) return true; - if (parts.length !== 1) return false; - const parameter = parts[0]!; - const equals = parameter.indexOf('='); - if (equals < 1 || parameter.slice(0, equals).trim().toLowerCase() !== 'charset') return false; - const rawValue = parameter.slice(equals + 1).trim(); - const value = unquoteHeaderValue(rawValue); - return value?.toLowerCase() === 'utf-8'; -}; - -const unquoteHeaderValue = (value: string): string | undefined => { - if (/^[!#$%&'*+\-.^_`|~0-9A-Za-z]+$/u.test(value)) return value; - if (!/^"(?:[^"\\\r\n]|\\[\t !-~])*"$/u.test(value)) return undefined; - return value.slice(1, -1).replace(/\\([\t !-~])/gu, '$1'); -}; - const decodedAssetPath = (requestTarget: string | undefined): string => { const pathname = requestTarget?.split(/[?#]/u, 1)[0]; if (pathname === undefined || !pathname.startsWith('/')) { @@ -302,9 +233,6 @@ const decodedAssetPath = (requestTarget: string | undefined): string => { return parts.join('/'); }; -const rawPathname = (requestTarget: string | undefined): string => - requestTarget?.split(/[?#]/u, 1)[0] ?? ''; - const decodedSkillSegment = (segment: string): string => { let decoded: string; try { @@ -803,7 +731,7 @@ export class ForegroundServer { if (route !== undefined) return this.#serveSkill(route, response, method); if (pathname === '/api/project/status') { if (method !== 'GET') return responseDiagnostic(response, diagnostic('AB8007', 'Route does not accept this method.', 405)); - return responseJson(response, { status: this.#coordinator.status() }); + return writeJsonResponse(response, { status: this.#coordinator.status() }); } if (pathname === '/api/project/session') { if (method !== 'GET') return responseDiagnostic(response, diagnostic('AB8007', 'Route does not accept this method.', 405)); @@ -825,7 +753,7 @@ export class ForegroundServer { return responseDiagnostic(response, diagnostic('AB8009', 'Request body must use application/json.', 415)); } await this.#coordinator.rebuild(manualInvalidation(await readBody(request), this.#now)); - return responseJson(response, { status: this.#coordinator.status() }); + return writeJsonResponse(response, { status: this.#coordinator.status() }); } if (pathname === '/api/project/events') { if (method !== 'GET') return responseDiagnostic(response, diagnostic('AB8007', 'Route does not accept this method.', 405)); @@ -844,13 +772,13 @@ export class ForegroundServer { return responseDiagnostic(response, diagnostic('AB8007', 'Route does not accept this method.', 405)); } try { - if (route.kind === 'source-tree') return responseJson(response, await service.sourceTree()); - if (route.kind === 'source-document') return responseJson(response, { document: await service.source(route.skillId) }); + if (route.kind === 'source-tree') return writeJsonResponse(response, await service.sourceTree()); + if (route.kind === 'source-document') return writeJsonResponse(response, { document: await service.source(route.skillId) }); if (route.kind === 'generated-tree') { - return responseJson(response, await service.generatedTree(route.epochId, route.target)); + return writeJsonResponse(response, await service.generatedTree(route.epochId, route.target)); } if (route.kind === 'generated-document') { - return responseJson(response, { document: await service.generated(route.epochId, route.target, route.skillId) }); + return writeJsonResponse(response, { document: await service.generated(route.epochId, route.target, route.skillId) }); } const value = route.kind === 'source-resource' ? await service.sourceResource(route.skillId, route.resource) diff --git a/packages/agent-bundle/src/dev/logs/dev-log-kinds.ts b/packages/agent-bundle/src/dev/logs/dev-log-kinds.ts index 7110d9b89..d66f540d8 100644 --- a/packages/agent-bundle/src/dev/logs/dev-log-kinds.ts +++ b/packages/agent-bundle/src/dev/logs/dev-log-kinds.ts @@ -47,6 +47,19 @@ export type DevLogKindMap = { readonly [TProducer in DevLogProducer]: (typeof de export type DevLogKindFor = DevLogKindMap[TProducer]; +/** The closed set of context keys a producer may attach; everything else is dropped at the boundary. */ +export const safeContextKeys: ReadonlySet = new Set([ + 'buildId', + 'diagnosticCode', + 'epochId', + 'hookId', + 'projectId', + 'routeId', + 'runId', + 'sessionId', + 'target', +]); + export const hasControlOrSeparators = (value: string): boolean => { for (let index = 0; index < value.length; index += 1) { const code = value.charCodeAt(index); diff --git a/packages/agent-bundle/src/dev/logs/dev-log-service.ts b/packages/agent-bundle/src/dev/logs/dev-log-service.ts index ec6c3b91c..f7137d6db 100644 --- a/packages/agent-bundle/src/dev/logs/dev-log-service.ts +++ b/packages/agent-bundle/src/dev/logs/dev-log-service.ts @@ -8,6 +8,7 @@ import { devLogLevels, devLogProducers, hasControlOrSeparators, + safeContextKeys, type DevLogKindFor, type DevLogLevel, type DevLogProducer, @@ -114,17 +115,6 @@ const defaultSubscriberRecordLimit = 128; const minimumRecordByteLimit = 256; const unavailable = '[UNAVAILABLE]' as const; const redacted = '[REDACTED]'; -const safeContextKeys = new Set([ - 'buildId', - 'diagnosticCode', - 'epochId', - 'hookId', - 'projectId', - 'routeId', - 'runId', - 'sessionId', - 'target', -]); const safeIdentifier = /^[A-Za-z0-9][A-Za-z0-9._:@+-]{0,255}$/u; const maxSummaryLength = 2_048; diff --git a/packages/agent-bundle/src/dev/mcp-app-action-validation.ts b/packages/agent-bundle/src/dev/mcp-app-action-validation.ts index 664213387..e65047599 100644 --- a/packages/agent-bundle/src/dev/mcp-app-action-validation.ts +++ b/packages/agent-bundle/src/dev/mcp-app-action-validation.ts @@ -28,7 +28,7 @@ const jsonRecord = (value: unknown): McpAppJsonRecord | undefined => { : copied as McpAppJsonRecord; }; -const validIcon = (value: unknown): boolean => { +export const validIcon = (value: unknown): boolean => { const icon = jsonRecord(value); return icon !== undefined && nonempty(icon.src) && (icon.mimeType === undefined || nonempty(icon.mimeType)) @@ -36,9 +36,9 @@ const validIcon = (value: unknown): boolean => { && (icon.theme === undefined || icon.theme === 'light' || icon.theme === 'dark'); }; -const validIcons = (value: unknown): boolean => Array.isArray(value) && value.every(validIcon); +export const validIcons = (value: unknown): boolean => Array.isArray(value) && value.every(validIcon); -const validIsoDateTimeWithOffset = (value: unknown): boolean => { +export const validIsoDateTimeWithOffset = (value: unknown): boolean => { if (typeof value !== 'string') return false; const match = /^(\d{4})-(\d{2})-(\d{2})T(\d{2}):(\d{2}):(\d{2})(?:\.\d+)?(?:Z|[+-]\d{2}:\d{2})$/u.exec(value); if (match === null) return false; diff --git a/packages/agent-bundle/src/dev/mcp-apps/mcp-app-bridge.ts b/packages/agent-bundle/src/dev/mcp-apps/mcp-app-bridge.ts index ab2975180..c0b63e702 100644 --- a/packages/agent-bundle/src/dev/mcp-apps/mcp-app-bridge.ts +++ b/packages/agent-bundle/src/dev/mcp-apps/mcp-app-bridge.ts @@ -1,6 +1,8 @@ import { validateMcpAppDownloadRequest, validateMcpAppExternalLink, + validIcons, + validIsoDateTimeWithOffset, type McpAppValidatedDownload, } from '../mcp-app-action-validation.ts'; import { @@ -393,27 +395,6 @@ const validAppCapabilities = (value: unknown): McpAppBridgeJsonRecord | undefine return capabilities; }; -const validIcon = (value: unknown): boolean => { - const icon = jsonRecord(value); - return icon !== undefined && nonempty(icon.src) - && (icon.mimeType === undefined || nonempty(icon.mimeType)) - && (icon.sizes === undefined || (Array.isArray(icon.sizes) && icon.sizes.every(nonempty))) - && (icon.theme === undefined || icon.theme === 'light' || icon.theme === 'dark'); -}; - -const validIcons = (value: unknown): boolean => Array.isArray(value) && value.every(validIcon); - -const validIsoDateTimeWithOffset = (value: unknown): boolean => { - if (typeof value !== 'string') return false; - const match = /^(\d{4})-(\d{2})-(\d{2})T(\d{2}):(\d{2}):(\d{2})(?:\.\d+)?(?:Z|[+-]\d{2}:\d{2})$/u.exec(value); - if (match === null) return false; - const [year, month, day, hour, minute, second] = match.slice(1, 7).map(Number); - if (year === undefined || month === undefined || day === undefined || hour === undefined || minute === undefined || second === undefined - || month < 1 || month > 12 || hour > 23 || minute > 59 || second > 59) return false; - const date = new Date(Date.UTC(year, month - 1, day)); - return date.getUTCFullYear() === year && date.getUTCMonth() === month - 1 && date.getUTCDate() === day && !Number.isNaN(Date.parse(value)); -}; - const validAnnotations = (value: unknown): boolean => { const annotations = jsonRecord(value); if (annotations === undefined) return false; diff --git a/packages/agent-bundle/src/dev/mcp-apps/mcp-app-protocol.ts b/packages/agent-bundle/src/dev/mcp-apps/mcp-app-protocol.ts deleted file mode 100644 index 49cdfbab1..000000000 --- a/packages/agent-bundle/src/dev/mcp-apps/mcp-app-protocol.ts +++ /dev/null @@ -1,497 +0,0 @@ -import { Buffer } from 'node:buffer'; - -import { - cloneMcpAppJson, - snapshotMcpAppJson, - snapshotMcpAppJsonRecord, - type McpAppJsonValue, -} from './mcp-app-json.ts'; -import type { - McpAppSandboxCsp, - McpAppSandboxPermissions, -} from './mcp-app-sandbox.ts'; - -export const MCP_APP_PROTOCOL_VERSION = '2026-01-26'; - -/** The pinned MCP Apps extension identifier the workbench advertises on session initialize. */ -export const MCP_APP_UI_EXTENSION = 'io.modelcontextprotocol/ui'; - -/** The only resource MIME type the workbench renders as an MCP App. */ -export const MCP_APP_MIME_TYPE = 'text/html;profile=mcp-app'; - -export type McpAppBridgeRequestId = string | number | null; -export type McpAppBridgeDisplayMode = 'inline' | 'fullscreen' | 'pip'; -export type McpAppBridgeJsonRecord = { readonly [key: string]: McpAppJsonValue }; - -export interface McpAppBridgeMessage { - readonly error?: McpAppBridgeRpcError; - readonly id?: McpAppBridgeRequestId; - readonly jsonrpc: '2.0'; - readonly method?: string; - readonly params?: McpAppJsonValue; - readonly result?: McpAppJsonValue; -} - -export interface McpAppBridgeRpcError { - readonly code: number; - readonly message: string; -} - -export interface McpAppBridgeToolCall { - readonly arguments?: McpAppJsonValue; - readonly name: string; -} - -export interface McpAppBridgeResourceRead { - readonly uri: string; -} - -export interface McpAppBridgeHostInfo { - readonly name: string; - readonly version: string; -} - -export interface McpAppBridgeLogEvent { - readonly data?: McpAppJsonValue; - readonly level: string; - readonly logger?: string; -} - -export interface McpAppBridgeMessageEvent { - readonly content: readonly McpAppJsonValue[]; - readonly role: 'user'; -} - -export interface McpAppBridgeModelContext { - readonly content?: readonly McpAppJsonValue[]; - readonly structuredContent?: McpAppBridgeJsonRecord; -} - -export interface McpAppBridgeSize { - readonly height?: number; - readonly width?: number; -} - -export interface ParsedMcpAppResource { - readonly csp?: McpAppSandboxCsp; - readonly html: string; - readonly permissions?: McpAppSandboxPermissions; -} - -export interface ValidMcpAppResourceReadResult extends McpAppBridgeJsonRecord { - readonly contents: readonly McpAppJsonValue[]; -} - -const loggingLevels = new Set(['debug', 'info', 'notice', 'warning', 'error', 'critical', 'alert', 'emergency']); -const displayModes = new Set(['inline', 'fullscreen', 'pip']); -const hostStyleVariables = new Set([ - '--color-background-primary', '--color-background-secondary', '--color-background-tertiary', '--color-background-inverse', '--color-background-ghost', '--color-background-info', '--color-background-danger', '--color-background-success', '--color-background-warning', '--color-background-disabled', - '--color-text-primary', '--color-text-secondary', '--color-text-tertiary', '--color-text-inverse', '--color-text-ghost', '--color-text-info', '--color-text-danger', '--color-text-success', '--color-text-warning', '--color-text-disabled', - '--color-border-primary', '--color-border-secondary', '--color-border-tertiary', '--color-border-inverse', '--color-border-ghost', '--color-border-info', '--color-border-danger', '--color-border-success', '--color-border-warning', '--color-border-disabled', - '--color-ring-primary', '--color-ring-secondary', '--color-ring-inverse', '--color-ring-info', '--color-ring-danger', '--color-ring-success', '--color-ring-warning', - '--font-sans', '--font-mono', '--font-weight-normal', '--font-weight-medium', '--font-weight-semibold', '--font-weight-bold', - '--font-text-xs-size', '--font-text-sm-size', '--font-text-md-size', '--font-text-lg-size', '--font-heading-xs-size', '--font-heading-sm-size', '--font-heading-md-size', '--font-heading-lg-size', '--font-heading-xl-size', '--font-heading-2xl-size', '--font-heading-3xl-size', - '--font-text-xs-line-height', '--font-text-sm-line-height', '--font-text-md-line-height', '--font-text-lg-line-height', '--font-heading-xs-line-height', '--font-heading-sm-line-height', '--font-heading-md-line-height', '--font-heading-lg-line-height', '--font-heading-xl-line-height', '--font-heading-2xl-line-height', '--font-heading-3xl-line-height', - '--border-radius-xs', '--border-radius-sm', '--border-radius-md', '--border-radius-lg', '--border-radius-xl', '--border-radius-full', '--border-width-regular', '--shadow-hairline', '--shadow-sm', '--shadow-md', '--shadow-lg', -]); - -export const hasOwn = (value: object, key: string): boolean => Object.hasOwn(value, key); -export const cloneJson = cloneMcpAppJson; -export const jsonRecord = snapshotMcpAppJsonRecord; -export const nonempty = (value: unknown): value is string => typeof value === 'string' && value.trim().length > 0; - -export const isRequestId = (value: unknown): value is McpAppBridgeRequestId => - value === null || typeof value === 'string' || (typeof value === 'number' && Number.isFinite(value)); - -export const messageOf = (value: unknown): McpAppBridgeMessage | undefined => { - const record = jsonRecord(value); - if (record === undefined || record.jsonrpc !== '2.0') return undefined; - const hasMethod = hasOwn(record, 'method'); - const hasResult = hasOwn(record, 'result'); - const hasError = hasOwn(record, 'error'); - if (Number(hasMethod) + Number(hasResult) + Number(hasError) !== 1) return undefined; - if (!hasMethod && !hasOwn(record, 'id')) return undefined; - if (hasOwn(record, 'id') && !isRequestId(record.id)) return undefined; - if (hasOwn(record, 'method') && !nonempty(record.method)) return undefined; - if (!hasMethod && hasOwn(record, 'params')) return undefined; - const error = hasError ? jsonRecord(record.error) : undefined; - if (hasError && (error === undefined || typeof error.code !== 'number' || !Number.isFinite(error.code) || !nonempty(error.message))) return undefined; - return Object.freeze({ - ...(error === undefined ? {} : { error: Object.freeze({ code: error.code as number, message: error.message as string }) }), - ...(hasOwn(record, 'id') ? { id: record.id as McpAppBridgeRequestId } : {}), - jsonrpc: '2.0' as const, - ...(hasOwn(record, 'method') ? { method: record.method as string } : {}), - ...(hasOwn(record, 'params') ? { params: cloneJson(record.params as McpAppJsonValue) } : {}), - ...(hasOwn(record, 'result') ? { result: cloneJson(record.result as McpAppJsonValue) } : {}), - }); -}; - -export const isInitialize = (message: McpAppBridgeMessage): boolean => message.method === 'ui/initialize' && hasOwn(message, 'id'); - -export const initializedNotification = (message: McpAppBridgeMessage): boolean => - message.method === 'ui/notifications/initialized' && !hasOwn(message, 'id') - && (message.params === undefined || jsonRecord(message.params) !== undefined); - -const validExperimentalCapabilities = (value: unknown): boolean => { - const capabilities = jsonRecord(value); - return capabilities !== undefined && Object.values(capabilities).every((capability) => jsonRecord(capability) !== undefined); -}; - -const validListChangedCapability = (value: unknown): boolean => { - const capability = jsonRecord(value); - return capability !== undefined && (capability.listChanged === undefined || typeof capability.listChanged === 'boolean'); -}; - -const validContentModalities = (value: unknown): boolean => { - const modalities = jsonRecord(value); - if (modalities === undefined) return false; - return ['text', 'image', 'audio', 'resource', 'resourceLink', 'structuredContent'] - .every((key) => modalities[key] === undefined || jsonRecord(modalities[key]) !== undefined); -}; - -const validCsp = (value: unknown): boolean => { - const csp = jsonRecord(value); - return csp !== undefined && ['connectDomains', 'resourceDomains', 'frameDomains', 'baseUriDomains'] - .every((key) => csp[key] === undefined || (Array.isArray(csp[key]) && csp[key].every((domain) => typeof domain === 'string'))); -}; - -const validPermissions = (value: unknown): boolean => { - const permissions = jsonRecord(value); - return permissions !== undefined && ['camera', 'microphone', 'geolocation', 'clipboardWrite'] - .every((key) => permissions[key] === undefined || jsonRecord(permissions[key]) !== undefined); -}; - -const validSandbox = (value: unknown): boolean => { - const sandbox = jsonRecord(value); - return sandbox !== undefined - && (sandbox.permissions === undefined || validPermissions(sandbox.permissions)) - && (sandbox.csp === undefined || validCsp(sandbox.csp)); -}; - -const validAppCapabilities = (value: unknown): McpAppBridgeJsonRecord | undefined => { - const capabilities = jsonRecord(value); - if (capabilities === undefined) return undefined; - if (capabilities.experimental !== undefined && !validExperimentalCapabilities(capabilities.experimental)) return undefined; - if (capabilities.tools !== undefined && !validListChangedCapability(capabilities.tools)) return undefined; - if (capabilities.availableDisplayModes !== undefined && validDisplayModeList(capabilities.availableDisplayModes) === undefined) return undefined; - return capabilities; -}; - -const validIcon = (value: unknown): boolean => { - const icon = jsonRecord(value); - return icon !== undefined && nonempty(icon.src) - && (icon.mimeType === undefined || nonempty(icon.mimeType)) - && (icon.sizes === undefined || (Array.isArray(icon.sizes) && icon.sizes.every(nonempty))) - && (icon.theme === undefined || icon.theme === 'light' || icon.theme === 'dark'); -}; - -const validIcons = (value: unknown): boolean => Array.isArray(value) && value.every(validIcon); - -const validIsoDateTimeWithOffset = (value: unknown): boolean => { - if (typeof value !== 'string') return false; - const match = /^(\d{4})-(\d{2})-(\d{2})T(\d{2}):(\d{2}):(\d{2})(?:\.\d+)?(?:Z|[+-]\d{2}:\d{2})$/u.exec(value); - if (match === null) return false; - const [year, month, day, hour, minute, second] = match.slice(1, 7).map(Number); - if (year === undefined || month === undefined || day === undefined || hour === undefined || minute === undefined || second === undefined - || month < 1 || month > 12 || hour > 23 || minute > 59 || second > 59) return false; - const date = new Date(Date.UTC(year, month - 1, day)); - return date.getUTCFullYear() === year && date.getUTCMonth() === month - 1 && date.getUTCDate() === day && !Number.isNaN(Date.parse(value)); -}; - -const validAnnotations = (value: unknown): boolean => { - const annotations = jsonRecord(value); - if (annotations === undefined) return false; - if (annotations.audience !== undefined && (!Array.isArray(annotations.audience) || !annotations.audience.every((role) => role === 'user' || role === 'assistant'))) return false; - if (annotations.priority !== undefined && (typeof annotations.priority !== 'number' || annotations.priority < 0 || annotations.priority > 1)) return false; - if (annotations.lastModified !== undefined && !validIsoDateTimeWithOffset(annotations.lastModified)) return false; - return true; -}; - -const validImplementation = (value: unknown): McpAppBridgeJsonRecord | undefined => { - const implementation = jsonRecord(value); - if (implementation === undefined || !nonempty(implementation.name) || !nonempty(implementation.version)) return undefined; - if (['title', 'websiteUrl', 'description'].some((key) => implementation[key] !== undefined && typeof implementation[key] !== 'string')) return undefined; - if (implementation.icons !== undefined && !validIcons(implementation.icons)) return undefined; - return implementation; -}; - -export const validInitialize = (params: McpAppJsonValue | undefined): boolean => { - const record = jsonRecord(params); - if (record === undefined || record.protocolVersion !== MCP_APP_PROTOCOL_VERSION) return false; - return validImplementation(record.appInfo) !== undefined && validAppCapabilities(record.appCapabilities) !== undefined; -}; - -const contentBlock = (value: McpAppJsonValue): boolean => { - const block = jsonRecord(value); - if (block === undefined || !nonempty(block.type)) return false; - if (block.annotations !== undefined && !validAnnotations(block.annotations)) return false; - switch (block.type) { - case 'text': - return typeof block.text === 'string'; - case 'image': - case 'audio': - return typeof block.data === 'string' && nonempty(block.mimeType); - case 'resource_link': - return nonempty(block.name) && nonempty(block.uri) - && (block.title === undefined || typeof block.title === 'string') - && (block.description === undefined || typeof block.description === 'string') - && (block.mimeType === undefined || typeof block.mimeType === 'string') - && (block.size === undefined || typeof block.size === 'number') - && (block.icons === undefined || validIcons(block.icons)); - case 'resource': - return validResourceContent(block.resource) !== undefined; - default: - return false; - } -}; - -const validContentBlocks = (value: unknown): readonly McpAppJsonValue[] | undefined => { - if (!Array.isArray(value)) return undefined; - const blocks: McpAppJsonValue[] = []; - for (const valueBlock of value) { - const block = snapshotMcpAppJson(valueBlock); - if (block === undefined || !contentBlock(block)) return undefined; - blocks.push(block); - } - return Object.freeze(blocks); -}; - -export const validToolResult = (value: unknown): McpAppJsonValue | undefined => { - const result = jsonRecord(value); - if (result === undefined || validContentBlocks(result.content) === undefined) return undefined; - if (result.structuredContent !== undefined && jsonRecord(result.structuredContent) === undefined) return undefined; - if (result.isError !== undefined && typeof result.isError !== 'boolean') return undefined; - if (result._meta !== undefined && jsonRecord(result._meta) === undefined) return undefined; - return result; -}; - -export const validMessageResult = (value: unknown): McpAppJsonValue | undefined => { - const result = jsonRecord(value); - return result === undefined || (result.isError !== undefined && typeof result.isError !== 'boolean') ? undefined : result; -}; - -export const validDisplayModeList = (value: unknown): readonly McpAppBridgeDisplayMode[] | undefined => - Array.isArray(value) && value.every((mode) => typeof mode === 'string' && displayModes.has(mode as McpAppBridgeDisplayMode)) - ? Object.freeze([...value] as McpAppBridgeDisplayMode[]) - : undefined; - -export const validHostCapabilities = (value: unknown): McpAppBridgeJsonRecord | undefined => { - const capabilities = jsonRecord(value); - if (capabilities === undefined) return undefined; - if (capabilities.experimental !== undefined && !validExperimentalCapabilities(capabilities.experimental)) return undefined; - if (['openLinks', 'downloadFile', 'logging'].some((key) => capabilities[key] !== undefined && jsonRecord(capabilities[key]) === undefined)) return undefined; - if (['serverTools', 'serverResources'].some((key) => capabilities[key] !== undefined && !validListChangedCapability(capabilities[key]))) return undefined; - if (capabilities.sandbox !== undefined && !validSandbox(capabilities.sandbox)) return undefined; - if (['updateModelContext', 'message'].some((key) => capabilities[key] !== undefined && !validContentModalities(capabilities[key]))) return undefined; - if (capabilities.sampling !== undefined) { - const sampling = jsonRecord(capabilities.sampling); - if (sampling === undefined || (sampling.tools !== undefined && jsonRecord(sampling.tools) === undefined)) return undefined; - } - return capabilities; -}; - -const validObjectJsonSchema = (value: unknown): boolean => { - const schema = jsonRecord(value); - if (schema === undefined || schema.type !== 'object') return false; - if (schema.properties !== undefined) { - const properties = jsonRecord(schema.properties); - if (properties === undefined || !Object.values(properties).every((property) => jsonRecord(property) !== undefined)) return false; - } - return schema.required === undefined || (Array.isArray(schema.required) && schema.required.every((required) => typeof required === 'string')); -}; - -const validToolDefinition = (value: unknown): boolean => { - const tool = jsonRecord(value); - if (tool === undefined || !nonempty(tool.name) || !validObjectJsonSchema(tool.inputSchema)) return false; - if (tool.outputSchema !== undefined && !validObjectJsonSchema(tool.outputSchema)) return false; - if (tool.icons !== undefined && !validIcons(tool.icons)) return false; - if (tool.title !== undefined && typeof tool.title !== 'string') return false; - if (tool.description !== undefined && typeof tool.description !== 'string') return false; - if (tool.annotations !== undefined) { - const annotations = jsonRecord(tool.annotations); - if (annotations === undefined - || (annotations.title !== undefined && !nonempty(annotations.title)) - || ['readOnlyHint', 'destructiveHint', 'idempotentHint', 'openWorldHint'].some((key) => annotations[key] !== undefined && typeof annotations[key] !== 'boolean')) return false; - } - if (tool.execution !== undefined) { - const execution = jsonRecord(tool.execution); - if (execution === undefined || (execution.taskSupport !== undefined && execution.taskSupport !== 'required' && execution.taskSupport !== 'optional' && execution.taskSupport !== 'forbidden')) return false; - } - return true; -}; - -export const validHostContext = (value: unknown): McpAppBridgeJsonRecord | undefined => { - const context = jsonRecord(value); - if (context === undefined) return undefined; - if (context.theme !== undefined && context.theme !== 'light' && context.theme !== 'dark') return undefined; - if (context.displayMode !== undefined && (typeof context.displayMode !== 'string' || !displayModes.has(context.displayMode as McpAppBridgeDisplayMode))) return undefined; - if (context.availableDisplayModes !== undefined && validDisplayModeList(context.availableDisplayModes) === undefined) return undefined; - if (context.locale !== undefined && !nonempty(context.locale)) return undefined; - if (context.timeZone !== undefined && !nonempty(context.timeZone)) return undefined; - if (context.userAgent !== undefined && !nonempty(context.userAgent)) return undefined; - if (context.platform !== undefined && context.platform !== 'web' && context.platform !== 'desktop' && context.platform !== 'mobile') return undefined; - if (context.toolInfo !== undefined) { - const toolInfo = jsonRecord(context.toolInfo); - if (toolInfo === undefined || !validToolDefinition(toolInfo.tool) || (toolInfo.id !== undefined && !isRequestId(toolInfo.id))) return undefined; - } - if (context.deviceCapabilities !== undefined) { - const device = jsonRecord(context.deviceCapabilities); - if (device === undefined || (device.touch !== undefined && typeof device.touch !== 'boolean') || (device.hover !== undefined && typeof device.hover !== 'boolean')) return undefined; - } - if (context.styles !== undefined) { - const styles = jsonRecord(context.styles); - const variables = styles === undefined || styles.variables === undefined ? undefined : jsonRecord(styles.variables); - const css = styles === undefined || styles.css === undefined ? undefined : jsonRecord(styles.css); - if (styles === undefined || (styles.variables !== undefined && (variables === undefined || !Object.entries(variables).every(([key, variable]) => hostStyleVariables.has(key) && typeof variable === 'string'))) - || (styles.css !== undefined && (css === undefined || (css.fonts !== undefined && typeof css.fonts !== 'string')))) return undefined; - } - if (context.containerDimensions !== undefined) { - const dimensions = jsonRecord(context.containerDimensions); - if (dimensions === undefined || !['height', 'maxHeight', 'width', 'maxWidth'].every((key) => dimensions[key] === undefined || (typeof dimensions[key] === 'number' && Number.isFinite(dimensions[key]) && dimensions[key] >= 0))) return undefined; - } - if (context.safeAreaInsets !== undefined) { - const insets = jsonRecord(context.safeAreaInsets); - if (insets === undefined || !['top', 'right', 'bottom', 'left'].every((key) => typeof insets[key] === 'number' && Number.isFinite(insets[key]) && insets[key] >= 0)) return undefined; - } - return context; -}; - -const validResourceMetadata = (value: unknown): McpAppBridgeJsonRecord | undefined => { - const metadata = jsonRecord(value); - if (metadata === undefined) return undefined; - if (metadata.ui === undefined) return metadata; - const ui = jsonRecord(metadata.ui); - if (ui === undefined - || (ui.csp !== undefined && !validCsp(ui.csp)) - || (ui.permissions !== undefined && !validPermissions(ui.permissions)) - || (ui.domain !== undefined && !nonempty(ui.domain)) - || (ui.prefersBorder !== undefined && typeof ui.prefersBorder !== 'boolean')) return undefined; - return metadata; -}; - -const validResourceContent = (value: unknown): McpAppBridgeJsonRecord | undefined => { - const content = jsonRecord(value); - if (content === undefined || !nonempty(content.uri) || (content.mimeType !== undefined && !nonempty(content.mimeType))) return undefined; - const hasText = typeof content.text === 'string'; - const hasBlob = typeof content.blob === 'string'; - if (hasText === hasBlob || (content._meta !== undefined && validResourceMetadata(content._meta) === undefined)) return undefined; - return content; -}; - -export const validResourceReadResult = (value: unknown): ValidMcpAppResourceReadResult | undefined => { - const result = jsonRecord(value); - const contents = result?.contents; - if (result === undefined || !Array.isArray(contents) || !contents.every((content) => validResourceContent(content) !== undefined)) return undefined; - return Object.freeze({ ...result, contents: Object.freeze([...contents]) }) as ValidMcpAppResourceReadResult; -}; - -const resourceMetadata = (value: unknown): { readonly csp?: McpAppSandboxCsp; readonly permissions?: McpAppSandboxPermissions } | undefined => { - if (value === undefined) return Object.freeze({}); - const metadata = validResourceMetadata(value); - const ui = metadata === undefined ? undefined : jsonRecord(metadata.ui); - if (metadata === undefined || ui === undefined && metadata.ui !== undefined) return undefined; - if (ui === undefined) return Object.freeze({}); - const csp = ui.csp === undefined ? undefined : jsonRecord(ui.csp); - const permissions = ui.permissions === undefined ? undefined : jsonRecord(ui.permissions); - if ((ui.csp !== undefined && csp === undefined) || (ui.permissions !== undefined && permissions === undefined)) return undefined; - return Object.freeze({ - ...(csp === undefined ? {} : { csp: csp as McpAppSandboxCsp }), - ...(permissions === undefined ? {} : { permissions: permissions as McpAppSandboxPermissions }), - }); -}; - -const htmlFromBlob = (blob: string): string | undefined => { - if (blob.length === 0 || blob.length % 4 !== 0 || !/^[A-Za-z0-9+/]*={0,2}$/u.test(blob)) return undefined; - const bytes = Buffer.from(blob, 'base64'); - if (bytes.toString('base64') !== blob) return undefined; - try { - return new TextDecoder('utf-8', { fatal: true }).decode(bytes); - } catch { - return undefined; - } -}; - -export const parsedResource = (value: McpAppJsonValue, resourceUri: string): ParsedMcpAppResource | undefined => { - const response = validResourceReadResult(value); - if (response === undefined) return undefined; - for (const candidate of response.contents) { - const content = validResourceContent(candidate); - if (content === undefined || content.uri !== resourceUri || content.mimeType !== MCP_APP_MIME_TYPE) continue; - const hasText = typeof content.text === 'string'; - const hasBlob = typeof content.blob === 'string'; - if (hasText === hasBlob) return undefined; - const html = hasText ? content.text as string : htmlFromBlob(content.blob as string); - const metadata = resourceMetadata(content._meta); - if (html === undefined || metadata === undefined) return undefined; - return Object.freeze({ ...metadata, html }); - } - return undefined; -}; - -export const validToolCall = (params: McpAppJsonValue | undefined): McpAppBridgeToolCall | undefined => { - const record = jsonRecord(params); - if (record === undefined || !nonempty(record.name)) return undefined; - if (record.arguments !== undefined && jsonRecord(record.arguments) === undefined) return undefined; - return Object.freeze({ ...(record.arguments === undefined ? {} : { arguments: cloneJson(record.arguments) }), name: record.name }); -}; - -export const validResourceRead = (params: McpAppJsonValue | undefined): McpAppBridgeResourceRead | undefined => { - const record = jsonRecord(params); - return record === undefined || !nonempty(record.uri) ? undefined : Object.freeze({ uri: record.uri }); -}; - -export const validOpenLink = (params: McpAppJsonValue | undefined): string | undefined => { - const record = jsonRecord(params); - if (record === undefined || !nonempty(record.url)) return undefined; - try { - const url = new URL(record.url); - return url.protocol === 'http:' || url.protocol === 'https:' ? url.href : undefined; - } catch { - return undefined; - } -}; - -export const validMessage = (params: McpAppJsonValue | undefined): McpAppBridgeMessageEvent | undefined => { - const record = jsonRecord(params); - const content = record === undefined ? undefined : validContentBlocks(record.content); - if (record === undefined || record.role !== 'user' || content === undefined) return undefined; - return Object.freeze({ content, role: 'user' }); -}; - -export const validDisplayMode = (params: McpAppJsonValue | undefined): McpAppBridgeDisplayMode | undefined => { - const record = jsonRecord(params); - return record === undefined || typeof record.mode !== 'string' || !displayModes.has(record.mode as McpAppBridgeDisplayMode) - ? undefined - : record.mode as McpAppBridgeDisplayMode; -}; - -export const validModelContext = (params: McpAppJsonValue | undefined): McpAppBridgeModelContext | undefined => { - const record = jsonRecord(params); - const content = record === undefined || record.content === undefined ? undefined : validContentBlocks(record.content); - if (record === undefined || (record.content !== undefined && content === undefined)) return undefined; - const structuredContent = record.structuredContent === undefined ? undefined : jsonRecord(record.structuredContent); - if (record.structuredContent !== undefined && structuredContent === undefined) return undefined; - return Object.freeze({ - ...(content === undefined ? {} : { content }), - ...(structuredContent === undefined ? {} : { structuredContent }), - }); -}; - -export const validLog = (params: McpAppJsonValue | undefined): McpAppBridgeLogEvent | undefined => { - const record = jsonRecord(params); - if (record === undefined || record.data === undefined || typeof record.level !== 'string' || !loggingLevels.has(record.level)) return undefined; - if (record.logger !== undefined && !nonempty(record.logger)) return undefined; - return Object.freeze({ - data: cloneJson(record.data), - level: record.level, - ...(record.logger === undefined ? {} : { logger: record.logger }), - }); -}; - -export const validSize = (params: McpAppJsonValue | undefined): McpAppBridgeSize | undefined => { - const record = jsonRecord(params); - if (record === undefined || (record.width === undefined && record.height === undefined)) return undefined; - if ((record.width !== undefined && (typeof record.width !== 'number' || !Number.isFinite(record.width) || record.width < 0)) - || (record.height !== undefined && (typeof record.height !== 'number' || !Number.isFinite(record.height) || record.height < 0))) return undefined; - return Object.freeze({ ...(record.height === undefined ? {} : { height: record.height }), ...(record.width === undefined ? {} : { width: record.width }) }); -}; diff --git a/packages/agent-bundle/src/dev/mcp-apps/mcp-app-routes.ts b/packages/agent-bundle/src/dev/mcp-apps/mcp-app-routes.ts index 5195eaf27..b844e7fa1 100644 --- a/packages/agent-bundle/src/dev/mcp-apps/mcp-app-routes.ts +++ b/packages/agent-bundle/src/dev/mcp-apps/mcp-app-routes.ts @@ -11,24 +11,28 @@ import type { McpAppRuntimeRoutePreviewService, } from '../mcp-app-runtime-preview-service.ts'; import { hasOnlyOwnKeys } from '../../core/strict-json.ts'; +import { + diagnostic, + isJsonRequest, + isRequestDiagnostic, + nonemptyString, + rawPathname, + readBody, + requestError, + responseDiagnostic, + responseJson as writeJsonResponse, +} from '../http.ts'; import { isMcpAppConsentCapability } from './mcp-app-sandbox.ts'; import type { McpAppConsentChallenge } from './mcp-app-sandbox.ts'; import type { McpAppConsentRequest } from './mcp-app-sandbox.ts'; import { runtimeAppMessageLimits } from '../runtime-app-message-limits.ts'; -const bodyLimit = 64 * 1024; // A force-close DELETE that lands after an accepted graceful close must stay // idempotent (200, not 404), so this window has to dominate the frame relay's // force-close budget — clients may fall back as late as their closeTimeoutMs, // which mcp-app-frame.tsx caps at 30s. const gracefulCloseReceiptTimeoutMs = 35_000; -interface RequestDiagnostic { - readonly code: string; - readonly message: string; - readonly status: number; -} - interface CreateRoute { readonly kind: 'create'; readonly sessionId: string; @@ -99,33 +103,6 @@ export interface McpAppRoutesOptions { readonly service?: McpAppRoutePreviewService; } -const diagnostic = (code: string, message: string, status: number): RequestDiagnostic => ({ code, message, status }); - -const requestError = (value: RequestDiagnostic): RequestDiagnostic & Error => Object.assign( - new Error(value.message), - value, -); - -const isRequestDiagnostic = (value: unknown): value is RequestDiagnostic => - typeof value === 'object' && value !== null && - typeof (value as Partial).code === 'string' && - typeof (value as Partial).message === 'string' && - typeof (value as Partial).status === 'number'; - -const responseDiagnostic = (response: ServerResponse, value: RequestDiagnostic): void => { - if (response.headersSent || response.writableEnded) { - response.destroy(); - return; - } - response.writeHead(value.status, { 'content-type': 'application/json; charset=utf-8' }); - response.end(JSON.stringify({ diagnostic: { code: value.code, message: value.message } })); -}; - -const responseJson = (response: ServerResponse, body: unknown): void => { - response.writeHead(200, { 'content-type': 'application/json; charset=utf-8' }); - response.end(JSON.stringify(body)); -}; - /** Runtime App operation results cross the bounded host-to-opaque-App channel. */ const runtimeOperationResponseJson = (response: ServerResponse, body: unknown): void => { let encoded: string; @@ -148,52 +125,6 @@ const runtimeOperationResponseJson = (response: ServerResponse, body: unknown): response.end(encoded); }; -const singleHeader = (value: string | readonly string[] | undefined): string | undefined => - typeof value === 'string' ? value : undefined; - -const unquoteHeaderValue = (value: string): string | undefined => { - if (/^[!#$%&'*+\-.^_`|~0-9A-Za-z]+$/u.test(value)) return value; - if (!/^"(?:[^"\\\r\n]|\\[\t !-~])*"$/u.test(value)) return undefined; - return value.slice(1, -1).replace(/\\([\t !-~])/gu, '$1'); -}; - -const isJsonRequest = (request: IncomingMessage): boolean => { - const contentType = singleHeader(request.headers['content-type']); - if (contentType === undefined) return false; - const parts = contentType.split(';').map((part) => part.trim()); - if (parts.shift()?.toLowerCase() !== 'application/json') return false; - if (parts.length === 0) return true; - if (parts.length !== 1) return false; - const parameter = parts[0]!; - const equals = parameter.indexOf('='); - if (equals < 1 || parameter.slice(0, equals).trim().toLowerCase() !== 'charset') return false; - return unquoteHeaderValue(parameter.slice(equals + 1).trim())?.toLowerCase() === 'utf-8'; -}; - -const readBody = async (request: IncomingMessage): Promise => new Promise((resolvePromise, rejectPromise) => { - let size = 0; - let tooLarge = false; - const chunks: Buffer[] = []; - request.on('data', (chunk: Buffer) => { - size += chunk.length; - if (size > bodyLimit) { - tooLarge = true; - return; - } - if (!tooLarge) chunks.push(chunk); - }); - request.once('end', () => { - if (tooLarge) { - rejectPromise(requestError(diagnostic('AB8010', 'Request body exceeds 64 KiB.', 413))); - return; - } - resolvePromise(Buffer.concat(chunks).toString('utf8')); - }); - request.once('error', rejectPromise); -}); - -const rawPathname = (requestTarget: string | undefined): string => requestTarget?.split(/[?#]/u, 1)[0] ?? ''; - const opaqueSegment = (value: string): string => { let decoded: string; try { @@ -256,9 +187,6 @@ const isRecord = (value: unknown): value is JsonObject => const hasOnly: (value: JsonObject, fields: readonly string[]) => boolean = hasOnlyOwnKeys; -const nonemptyString = (value: unknown): value is string => - typeof value === 'string' && value.trim().length > 0 && value.length <= 4_096 && !value.includes('\0'); - const isJsonValue = (value: unknown): value is McpAppJsonValue => { if (value === null || typeof value === 'boolean' || typeof value === 'string') return true; if (typeof value === 'number') return Number.isFinite(value); @@ -505,7 +433,7 @@ export class McpAppRoutes { if (parsed.kind === 'create') { if (method !== 'POST') return responseDiagnostic(response, diagnostic('AB8007', 'Route does not accept this method.', 405)); const preview = await service.create(createRequest(await jsonBody(request), parsed.sessionId)); - return responseJson(response, { lifecycle: preview.bridge.lifecycle, preview: previewSnapshot(preview) }); + return writeJsonResponse(response, { lifecycle: preview.bridge.lifecycle, preview: previewSnapshot(preview) }); } if (parsed.kind === 'force-close') { if (method !== 'DELETE') return responseDiagnostic(response, diagnostic('AB8007', 'Route does not accept this method.', 405)); @@ -515,14 +443,14 @@ export class McpAppRoutes { if (!closed && !gracefulCloseAccepted) this.#unavailable(); this.#clearTeardown(parsed.bindingId); }); - return responseJson(response, { closed: true, lifecycle: 'closed' }); + return writeJsonResponse(response, { closed: true, lifecycle: 'closed' }); } if (parsed.kind === 'consent') { const preview = this.#preview(service, parsed.bindingId); if (method === 'GET') { const challenges = service.consentChallenges?.(parsed.bindingId); if (challenges === undefined) this.#unavailable(); - return responseJson(response, { challenges, lifecycle: preview.bridge.lifecycle }); + return writeJsonResponse(response, { challenges, lifecycle: preview.bridge.lifecycle }); } if (method !== 'POST') return responseDiagnostic(response, diagnostic('AB8007', 'Route does not accept this method.', 405)); const decision = consentDecision(await jsonBody(request)); @@ -532,7 +460,7 @@ export class McpAppRoutes { // A rejected-but-recognized action decision may carry the bridge's // terminal -32001 response. Forged/replayed decisions drain nothing. const messages = await service.takeOutbound(parsed.bindingId); - return responseJson(response, { approved, lifecycle: refreshed.bridge.lifecycle, messages, preview: previewSnapshot(refreshed) }); + return writeJsonResponse(response, { approved, lifecycle: refreshed.bridge.lifecycle, messages, preview: previewSnapshot(refreshed) }); } if (method !== 'POST') return responseDiagnostic(response, diagnostic('AB8007', 'Route does not accept this method.', 405)); if (parsed.kind === 'close') { @@ -550,7 +478,7 @@ export class McpAppRoutes { started: true, }); }); - return responseJson(response, { + return writeJsonResponse(response, { actions: [], lifecycle: result.lifecycle, ...(result.started ? { message: result.message } : {}), @@ -566,7 +494,7 @@ export class McpAppRoutes { } return Object.freeze({ accepted, actions: Object.freeze([]), lifecycle: preview.bridge.lifecycle, messages }); }); - return responseJson(response, result); + return writeJsonResponse(response, result); } const result = await this.#serialize(parsed.bindingId, async () => { const preview = this.#preview(service, parsed.bindingId); @@ -576,7 +504,7 @@ export class McpAppRoutes { const messages = await service.takeOutbound(parsed.bindingId); return Object.freeze({ accepted, actions: Object.freeze([]), lifecycle: preview.bridge.lifecycle, messages }); }); - return responseJson(response, result); + return writeJsonResponse(response, result); } async #dispatchRuntime( @@ -589,7 +517,7 @@ export class McpAppRoutes { const method = request.method ?? 'GET'; if (parsed.kind === 'runtime-create') { if (method !== 'POST') return responseDiagnostic(response, diagnostic('AB8007', 'Route does not accept this method.', 405)); - return responseJson(response, { preview: await runtime.create(runtimeCreateRequest(await jsonBody(request))) }); + return writeJsonResponse(response, { preview: await runtime.create(runtimeCreateRequest(await jsonBody(request))) }); } if (parsed.kind === 'runtime-get') { if (method === 'DELETE') { @@ -597,7 +525,7 @@ export class McpAppRoutes { this.#runtimeUnavailable(runtime, parsed.bindingId); } await runtime.close(parsed.bindingId); - return responseJson(response, { closed: true }); + return writeJsonResponse(response, { closed: true }); } if (method !== 'GET') return responseDiagnostic(response, diagnostic('AB8007', 'Route does not accept this method.', 405)); const preview = runtime.get(parsed.bindingId); @@ -609,7 +537,7 @@ export class McpAppRoutes { if (parsed.kind === 'runtime-close') { if (method !== 'DELETE') return responseDiagnostic(response, diagnostic('AB8007', 'Route does not accept this method.', 405)); await runtime.close(parsed.bindingId); - return responseJson(response, { closed: true }); + return writeJsonResponse(response, { closed: true }); } if (parsed.kind === 'runtime-operation') { if (method !== 'POST') return responseDiagnostic(response, diagnostic('AB8007', 'Route does not accept this method.', 405)); @@ -624,12 +552,12 @@ export class McpAppRoutes { if (parsed.kind === 'runtime-consent-create') { if (method !== 'POST') return responseDiagnostic(response, diagnostic('AB8007', 'Route does not accept this method.', 405)); if (runtime.get(parsed.bindingId) === undefined) this.#runtimeUnavailable(runtime, parsed.bindingId); - return responseJson(response, await runtime.createConsent(parsed.bindingId, runtimeConsentRequest(await jsonBody(request)))); + return writeJsonResponse(response, await runtime.createConsent(parsed.bindingId, runtimeConsentRequest(await jsonBody(request)))); } if (parsed.kind !== 'runtime-consent-decide') throw new Error('Runtime MCP App route is not valid.'); if (method !== 'POST') return responseDiagnostic(response, diagnostic('AB8007', 'Route does not accept this method.', 405)); if (runtime.get(parsed.bindingId) === undefined) this.#runtimeUnavailable(runtime, parsed.bindingId); - return responseJson(response, await runtime.decideConsent(parsed.bindingId, parsed.consentId, runtimeConsentDecision(await jsonBody(request)))); + return writeJsonResponse(response, await runtime.decideConsent(parsed.bindingId, parsed.consentId, runtimeConsentDecision(await jsonBody(request)))); } #preview(service: McpAppRoutePreviewService, bindingId: string): McpAppRoutePreview { diff --git a/packages/agent-bundle/src/dev/playground/native-playground-catalog.ts b/packages/agent-bundle/src/dev/playground/native-playground-catalog.ts deleted file mode 100644 index a05dcc212..000000000 --- a/packages/agent-bundle/src/dev/playground/native-playground-catalog.ts +++ /dev/null @@ -1,289 +0,0 @@ -import { isAbsolute, relative, resolve } from 'node:path'; - -import { digest } from '../../core/digest.ts'; -import { isInsideOrEqual } from '../../core/paths.ts'; -import { hasExactOwnKeys, isJsonRecord as isRecord, type JsonValue } from '../../core/strict-json.ts'; -import type { EvalFixturePlan } from '../../eval/fixtures.ts'; -import { NATIVE_HOSTS } from '../../host-contracts/native-hosts.ts'; -import { normalizeEvalCase } from '../../eval/suite.ts'; -import type { EvalCase } from '../../eval/types.ts'; -import type { ArtifactEpoch } from '../types.ts'; -import type { - NativePlaygroundCatalogSelection, - NativePlaygroundHost, -} from './native-playground-types.ts'; - -export interface CatalogSelection { - readonly caseId: string; - readonly caseLabel: string; - readonly evalCase: EvalCase; - readonly fixtureId: string; - readonly fixtureLabel: string; - readonly fixturePlan: EvalFixturePlan; - readonly host: NativePlaygroundHost; - readonly modelPinId: string; - readonly modelPinLabel: string; - readonly suiteDir: string; -} - -/** Persisted paths are deliberately relative to their owning project or suite. */ -export interface PersistedFixturePlan { - readonly digest: string; - readonly entries: readonly EvalFixturePlan['entries'][number][]; - readonly git: boolean; - readonly sourcePath: string; -} - -export interface PersistedCatalogSelection { - readonly caseId: string; - readonly caseLabel: string; - readonly evalCase: EvalCase; - readonly fixtureId: string; - readonly fixtureLabel: string; - readonly fixturePlan: PersistedFixturePlan; - readonly host: NativePlaygroundHost; - readonly modelPinId: string; - readonly modelPinLabel: string; - readonly suiteDir: string; - readonly suiteDigest: string; - readonly suiteName: string; -} - -export interface PersistedCatalogSnapshot { - readonly epochId: string; - readonly selections: readonly PersistedCatalogSelection[]; -} - -export const nativePlaygroundHosts = new Set(NATIVE_HOSTS); -export const maximumCatalogSelections = 256; -export const maximumCatalogSnapshotBytes = 8 * 1_024 * 1_024; -const maximumCatalogSnapshotNodes = 65_536; -const maximumFixtureEntries = 4_096; -const maximumSnapshotStringLength = 16_384; -const fixtureSha256Text = /^[a-f0-9]{64}$/u; - -const isSafeRelativePath = (value: JsonValue | undefined, allowCurrentDirectory = false): value is string => - nonemptySnapshotText(value) && - !isAbsolute(value) && - (allowCurrentDirectory && value === '.' || - value !== '.' && - !value.split(/[\\/]/u).some((part) => part.length === 0 || part === '.' || part === '..')); - -export const relativePathInside = (root: string, path: string, allowCurrentDirectory = false): string => { - const normalizedRoot = resolve(root); - const candidate = resolve(path); - if (!isInsideOrEqual(normalizedRoot, candidate)) { - throw new Error('Native Playground catalog snapshot contains an invalid path.'); - } - const value = relative(normalizedRoot, candidate) || '.'; - if (!isSafeRelativePath(value, allowCurrentDirectory)) { - throw new Error('Native Playground catalog snapshot contains an invalid path.'); - } - return value; -}; - -export const resolveContainedPath = (root: string, value: string, allowCurrentDirectory = false): string => { - if (!isSafeRelativePath(value, allowCurrentDirectory)) { - throw new Error('Native Playground catalog snapshot is invalid.'); - } - const candidate = resolve(root, value); - if (!isInsideOrEqual(resolve(root), candidate)) { - throw new Error('Native Playground catalog snapshot is invalid.'); - } - return candidate; -}; - -const opaqueId = (epoch: ArtifactEpoch, kind: string, content: unknown): string => - digest({ - content, - epoch: Object.freeze({ - configDigest: epoch.configDigest, - id: epoch.id, - modelDigest: epoch.modelDigest, - projectRevision: epoch.projectRevision, - targetDigests: epoch.targetDigests, - }), - kind, - }); - -export const catalogSelectionIdentity = (options: Readonly<{ - readonly epoch: ArtifactEpoch; - readonly evalCase: EvalCase; - readonly fixturePlan: Pick; - readonly host: NativePlaygroundHost; - readonly suiteDigest: string; - readonly suiteName: string; -}>): Readonly<{ - readonly caseId: string; - readonly caseLabel: string; - readonly fixtureId: string; - readonly fixtureLabel: string; - readonly modelPinId: string; - readonly modelPinLabel: string; -}> => Object.freeze({ - caseId: opaqueId(options.epoch, 'case', { digest: options.evalCase.digest, suite: options.suiteDigest }), - caseLabel: `${options.suiteName} / ${options.evalCase.id}`, - fixtureId: opaqueId(options.epoch, 'fixture', { case: options.evalCase.digest, digest: options.fixturePlan.digest }), - fixtureLabel: `${options.suiteName} / ${options.evalCase.id}`, - modelPinId: opaqueId(options.epoch, 'model', { case: options.evalCase.digest, host: options.host, model: options.evalCase.hosts[options.host]!.model }), - modelPinLabel: `${options.host} pinned model`, -}); - - -export const selectionKey = (selection: NativePlaygroundCatalogSelection): string => - `${selection.caseId}\u0000${selection.fixtureId}\u0000${selection.host}\u0000${selection.modelPinId}`; - -const nonemptySnapshotText = (value: JsonValue | undefined): value is string => - typeof value === 'string' && value.length > 0 && value.length <= maximumSnapshotStringLength; - -export const hasExactKeys = (value: Readonly>, keys: readonly string[]): boolean => - hasExactOwnKeys(value, keys); - -export const withinCatalogSnapshotNodeBudget = (root: unknown): boolean => { - let remaining = maximumCatalogSnapshotNodes; - const pending: unknown[] = [root]; - while (pending.length > 0) { - remaining -= 1; - if (remaining < 0) return false; - const value = pending.pop(); - if (Array.isArray(value)) { - for (const entry of value) pending.push(entry); - } else if (typeof value === 'object' && value !== null) { - for (const entry of Object.values(value)) pending.push(entry); - } - } - return true; -}; - -const activationEvidence = (value: JsonValue | undefined): value is 'inferred' | 'observed' | 'unavailable' => - value === 'inferred' || value === 'observed' || value === 'unavailable'; - -const safeInteger = (value: JsonValue | undefined): value is number => - typeof value === 'number' && Number.isSafeInteger(value); - -const persistedAssertion = (value: JsonValue): EvalCase['assertions'][number] | undefined => { - if (!isRecord(value) || !nonemptySnapshotText(value.id) || !activationEvidence(value.minimumEvidence) || typeof value.kind !== 'string') return undefined; - if (value.kind === 'exit-code' && hasExactKeys(value, ['expected', 'id', 'kind', 'minimumEvidence']) && safeInteger(value.expected)) { - return Object.freeze({ expected: value.expected, id: value.id, kind: value.kind, minimumEvidence: value.minimumEvidence }); - } - if (value.kind === 'mcp-call' && hasExactKeys(value, ['atLeast', 'id', 'kind', 'minimumEvidence', 'server', 'tool']) && - safeInteger(value.atLeast) && value.atLeast >= 0 && nonemptySnapshotText(value.server) && nonemptySnapshotText(value.tool)) { - return Object.freeze({ atLeast: value.atLeast, id: value.id, kind: value.kind, minimumEvidence: value.minimumEvidence, server: value.server, tool: value.tool }); - } - if (value.kind === 'no-mcp-call' && nonemptySnapshotText(value.server) && - ((hasExactKeys(value, ['id', 'kind', 'minimumEvidence', 'server'])) || - (hasExactKeys(value, ['id', 'kind', 'minimumEvidence', 'server', 'tool']) && nonemptySnapshotText(value.tool)))) { - return Object.freeze({ - ...(value.tool === undefined ? {} : { tool: value.tool as string }), - id: value.id, - kind: value.kind, - minimumEvidence: value.minimumEvidence, - server: value.server, - }); - } - if (value.kind === 'no-skill-activation' && - ((hasExactKeys(value, ['id', 'kind', 'minimumEvidence'])) || (hasExactKeys(value, ['id', 'kind', 'minimumEvidence', 'skill']) && nonemptySnapshotText(value.skill)))) { - return Object.freeze({ ...(value.skill === undefined ? {} : { skill: value.skill as string }), id: value.id, kind: value.kind, minimumEvidence: value.minimumEvidence }); - } - if (value.kind === 'outcome' && hasExactKeys(value, ['id', 'kind', 'minimumEvidence', 'script']) && nonemptySnapshotText(value.script)) { - return Object.freeze({ id: value.id, kind: value.kind, minimumEvidence: value.minimumEvidence, script: value.script }); - } - if (value.kind === 'skill-activation' && hasExactKeys(value, ['id', 'kind', 'minimumEvidence', 'skill']) && nonemptySnapshotText(value.skill)) { - return Object.freeze({ id: value.id, kind: value.kind, minimumEvidence: value.minimumEvidence, skill: value.skill }); - } - return undefined; -}; - -const persistedEvalCase = (value: JsonValue): EvalCase | undefined => { - if (!isRecord(value) || !hasExactKeys(value, ['assertions', 'digest', 'fixture', 'hosts', 'id', 'invocation', 'prompt', 'trials']) || - !Array.isArray(value.assertions) || value.assertions.length > maximumFixtureEntries || - !nonemptySnapshotText(value.digest) || !nonemptySnapshotText(value.id) || !nonemptySnapshotText(value.prompt) || - !safeInteger(value.trials) || value.trials < 1 || !isRecord(value.fixture) || !isRecord(value.hosts) || !isRecord(value.invocation)) return undefined; - if (!hasExactKeys(value.fixture, ['git', 'include', 'path']) || typeof value.fixture.git !== 'boolean' || - !Array.isArray(value.fixture.include) || value.fixture.include.length > maximumFixtureEntries || - !value.fixture.include.every(nonemptySnapshotText) || !isSafeRelativePath(value.fixture.path, true)) return undefined; - const hostEntries = Object.entries(value.hosts); - if (hostEntries.length > maximumFixtureEntries || hostEntries.some(([host, binding]) => !nonemptySnapshotText(host) || - !isRecord(binding) || !hasExactKeys(binding, ['model']) || !nonemptySnapshotText(binding.model))) return undefined; - const invocation = value.invocation; - if (!((invocation.mode === 'automatic' || invocation.mode === 'none') && hasExactKeys(invocation, ['mode'])) && - !(invocation.mode === 'explicit' && hasExactKeys(invocation, ['mode', 'skill']) && nonemptySnapshotText(invocation.skill))) return undefined; - const assertions = value.assertions.map(persistedAssertion); - if (assertions.some((assertion) => assertion === undefined)) return undefined; - try { - const normalized = normalizeEvalCase({ - assertions: assertions as EvalCase['assertions'][number][], - fixture: Object.freeze({ git: value.fixture.git, include: Object.freeze([...value.fixture.include] as string[]), path: value.fixture.path }), - hosts: Object.freeze(Object.fromEntries(hostEntries.map(([host, binding]) => [host, Object.freeze({ model: (binding as Readonly>).model })]))), - id: value.id, - invocation: Object.freeze(invocation.mode === 'explicit' ? { mode: 'explicit' as const, skill: invocation.skill as string } : { mode: invocation.mode }), - prompt: value.prompt, - trials: value.trials, - }); - return normalized.digest === value.digest ? normalized : undefined; - } catch { - return undefined; - } -}; - -const persistedFixturePlan = (value: JsonValue): PersistedFixturePlan | undefined => { - if (!isRecord(value) || !hasExactKeys(value, ['digest', 'entries', 'git', 'sourcePath']) || - !nonemptySnapshotText(value.digest) || typeof value.git !== 'boolean' || !isSafeRelativePath(value.sourcePath, true) || - !Array.isArray(value.entries) || value.entries.length > maximumFixtureEntries) return undefined; - const entries = value.entries.map((entry) => { - if (!isRecord(entry) || !hasExactKeys(entry, ['executable', 'path', 'sha256']) || typeof entry.executable !== 'boolean' || - !isSafeRelativePath(entry.path) || typeof entry.sha256 !== 'string' || !fixtureSha256Text.test(entry.sha256)) return undefined; - return Object.freeze({ executable: entry.executable, path: entry.path, sha256: entry.sha256 }); - }); - if (entries.some((entry) => entry === undefined) || new Set(entries.map((entry) => entry?.path)).size !== entries.length) return undefined; - const normalizedEntries = Object.freeze(entries as EvalFixturePlan['entries'][number][]); - return digest({ entries: normalizedEntries, git: value.git }) === value.digest - ? Object.freeze({ digest: value.digest, entries: normalizedEntries, git: value.git, sourcePath: value.sourcePath }) - : undefined; -}; - -export const persistedSelection = (value: JsonValue): PersistedCatalogSelection | undefined => { - if (!isRecord(value) || !hasExactKeys(value, [ - 'caseId', 'caseLabel', 'evalCase', 'fixtureId', 'fixtureLabel', 'fixturePlan', 'host', 'modelPinId', 'modelPinLabel', 'suiteDigest', 'suiteDir', 'suiteName', - ]) || - !nonemptySnapshotText(value.caseId) || !nonemptySnapshotText(value.caseLabel) || - !nonemptySnapshotText(value.fixtureId) || !nonemptySnapshotText(value.fixtureLabel) || - !nonemptySnapshotText(value.modelPinId) || !nonemptySnapshotText(value.modelPinLabel) || - !nonemptySnapshotText(value.suiteDigest) || !nonemptySnapshotText(value.suiteName) || - !isSafeRelativePath(value.suiteDir, true) || - (value.host !== 'claude' && value.host !== 'codex')) return undefined; - const evalCase = persistedEvalCase(value.evalCase); - const fixturePlan = persistedFixturePlan(value.fixturePlan); - if (evalCase === undefined || fixturePlan === undefined || evalCase.hosts[value.host] === undefined) return undefined; - return Object.freeze({ - caseId: value.caseId, - caseLabel: value.caseLabel, - evalCase, - fixtureId: value.fixtureId, - fixtureLabel: value.fixtureLabel, - fixturePlan, - host: value.host, - modelPinId: value.modelPinId, - modelPinLabel: value.modelPinLabel, - suiteDir: value.suiteDir, - suiteDigest: value.suiteDigest, - suiteName: value.suiteName, - }); -}; - -export const hasCanonicalSelectionIdentity = (epoch: ArtifactEpoch, selection: PersistedCatalogSelection): boolean => { - const expected = catalogSelectionIdentity({ - epoch, - evalCase: selection.evalCase, - fixturePlan: selection.fixturePlan, - host: selection.host, - suiteDigest: selection.suiteDigest, - suiteName: selection.suiteName, - }); - return selection.caseId === expected.caseId && - selection.caseLabel === expected.caseLabel && - selection.fixtureId === expected.fixtureId && - selection.fixtureLabel === expected.fixtureLabel && - selection.modelPinId === expected.modelPinId && - selection.modelPinLabel === expected.modelPinLabel; -}; diff --git a/packages/agent-bundle/src/dev/playground/native-playground-evidence.ts b/packages/agent-bundle/src/dev/playground/native-playground-evidence.ts deleted file mode 100644 index 923584779..000000000 --- a/packages/agent-bundle/src/dev/playground/native-playground-evidence.ts +++ /dev/null @@ -1,177 +0,0 @@ -import { provenanceIdentifierPattern } from '../../eval/provenance.ts'; -import { redactEvalCredentialText } from '../../eval/credentials.ts'; -import type { EvalTrialRecord, EvalTrialWriter } from '../../eval/run-store.ts'; -import type { WorkspaceDiff } from '../../eval/workspace-diff.ts'; -import type { PlaygroundEventInput, PlaygroundJsonObject } from './playground-store.ts'; -import { safeDevWireText } from '../logs/dev-log-service.ts'; -import type { NativePlaygroundProgress } from './native-playground-types.ts'; - -/** Native raw trial artifacts have no durable backing store; durable Playground events are the only exposed evidence. */ -export class DiscardingTrialWriter implements EvalTrialWriter { - async writeArtifactFile(relativePath: string, _contents: string): Promise { - return relativePath; - } - - async writeTrial(trial: EvalTrialRecord): Promise { - return Object.freeze({ ...trial }); - } -} - -export const hardcodedProgress = (phase: NativePlaygroundProgress): PlaygroundEventInput => Object.freeze({ - kind: `native.${phase}`, - raw: Object.freeze({ phase }), - source: 'host-preflight', - summary: phase === 'preflight' - ? 'Native host preflight completed.' - : phase === 'fixture.materialized' - ? 'Native fixture materialized.' - : phase === 'codex.setup' - ? 'Codex temporary environment setup started.' - : 'Native host process started.', -}); - -const safeNativeProvenanceText = (value: string, projectRoot: string): string => { - const redacted = safeDevWireText(redactEvalCredentialText(value), projectRoot); - return provenanceIdentifierPattern.test(redacted) ? redacted : '[REDACTED]'; -}; - -const nativeTrialProvenance = (trial: EvalTrialRecord, projectRoot: string): PlaygroundJsonObject => { - const provenance = trial.provenance; - const semanticGrader = provenance?.semanticGrader; - return Object.freeze({ - ...(provenance?.hostCliVersion === undefined - ? {} - : { hostCliVersion: safeNativeProvenanceText(provenance.hostCliVersion, projectRoot) }), - ...(provenance === undefined - ? {} - : { - invocation: Object.freeze({ - mode: provenance.invocation.mode, - ...(provenance.invocation.skill === undefined - ? {} - : { skill: safeNativeProvenanceText(provenance.invocation.skill, projectRoot) }), - }), - ...(semanticGrader === undefined - ? {} - : { semanticGrader: semanticGrader === null - ? null - : 'state' in semanticGrader - ? Object.freeze({ state: 'unrecorded' }) - : Object.freeze({ - id: safeNativeProvenanceText(semanticGrader.id, projectRoot), - model: safeNativeProvenanceText(semanticGrader.model, projectRoot), - }) }), - }), - model: safeNativeProvenanceText(trial.model, projectRoot), - }); -}; - -/** A completed response is user-facing evidence, but never a raw host stream. */ -export const safeResponse = (value: string): string => redactEvalCredentialText(value) - .replace(/(?:[A-Za-z]:)?(?:[/\\][^\s`'"<>|]*)+/gu, '[path]') - .replaceAll('\0', ''); - -export const workspaceEvidence = (diff: WorkspaceDiff): PlaygroundJsonObject => Object.freeze({ - changes: Object.freeze(diff.changes.map((change) => Object.freeze({ - digest: change.digest, - id: change.id, - kind: change.kind, - }))), - ...(diff.truncated === true ? { truncated: true } : {}), -}); - -export const normalizedTrialEvents = ( - trial: EvalTrialRecord, - diff: WorkspaceDiff | undefined, - hookEvents: readonly string[], - projectRoot: string, - response: string | undefined, -): readonly PlaygroundEventInput[] => Object.freeze([ - Object.freeze({ - kind: 'native.provenance', - raw: nativeTrialProvenance(trial, projectRoot), - source: 'host-preflight', - summary: 'Recorded safe native model and host provenance.', - }), - Object.freeze({ - kind: 'native.activation', - raw: Object.freeze({ - activated: Object.freeze(trial.evidence.skillActivation.activated.map((name) => safeDevWireText(name, projectRoot))), - level: trial.evidence.skillActivation.level, - }), - source: 'skill-evidence', - summary: 'Recorded normalized native Skill activation evidence.', - }), - Object.freeze({ - kind: 'native.mcp', - raw: Object.freeze({ - calls: Object.freeze(trial.evidence.mcp.calls.map((call) => Object.freeze({ - server: safeDevWireText(call.server, projectRoot), - tool: safeDevWireText(call.tool, projectRoot), - }))), - level: trial.evidence.mcp.level, - }), - source: 'mcp', - summary: 'Recorded normalized native MCP evidence.', - }), - Object.freeze({ - kind: 'native.assertions', - raw: Object.freeze({ - assertions: Object.freeze(trial.assertions.map((assertion) => Object.freeze({ - evidence: assertion.evidence, - id: safeDevWireText(assertion.assertionId, projectRoot), - kind: safeDevWireText(assertion.kind, projectRoot), - outcome: assertion.outcome, - }))), - }), - source: 'diagnostics', - summary: 'Recorded normalized native assertion evidence.', - }), - ...(hookEvents.length === 0 - ? [] - : [Object.freeze({ - kind: 'native.hooks', - raw: Object.freeze({ events: Object.freeze(hookEvents.map((event) => safeDevWireText(event, projectRoot))) }), - source: 'hook' as const, - summary: 'Recorded normalized native Hook evidence.', - })]), - ...(Object.keys(trial.evidence.scripts.results).length === 0 - ? [] - : [Object.freeze({ - kind: 'native.scripts', - raw: Object.freeze({ - level: trial.evidence.scripts.level, - results: Object.freeze(Object.entries(trial.evidence.scripts.results).map(([id, result]) => Object.freeze({ - detail: safeDevWireText(result.detail, projectRoot), - id: safeDevWireText(id, projectRoot), - outcome: result.outcome, - }))), - }), - source: 'script' as const, - summary: 'Recorded normalized native script evidence.', - })]), - ...(response === undefined - ? [] - : [Object.freeze({ - kind: 'native.response', - raw: Object.freeze({ text: response }), - source: 'response' as const, - summary: 'Recorded normalized native host response.', - })]), - ...(trial.harnessFailure === undefined - ? [] - : [Object.freeze({ - kind: 'native.harness.failed', - raw: Object.freeze({ code: trial.harnessFailure.code, stage: trial.harnessFailure.stage }), - source: 'host-preflight' as const, - summary: 'Native host could not complete the requested run.', - })]), - ...(diff === undefined - ? [] - : [Object.freeze({ - kind: 'native.workspace', - raw: workspaceEvidence(diff), - source: 'workspace-change' as const, - summary: 'Recorded bounded native workspace changes.', - })]), -]); diff --git a/packages/agent-bundle/src/dev/playground/playground-close-errors.ts b/packages/agent-bundle/src/dev/playground/playground-close-errors.ts deleted file mode 100644 index e0f3a9281..000000000 --- a/packages/agent-bundle/src/dev/playground/playground-close-errors.ts +++ /dev/null @@ -1,30 +0,0 @@ -import type { PlaygroundCleanupFailure } from './playground-protocol.ts'; - -/** Cleanup-failure errors raised while closing playground sessions or the whole service. */ - -export class PlaygroundSessionCloseError extends Error { - readonly failures: readonly PlaygroundCleanupFailure[]; - readonly sessionId: string; - - constructor(sessionId: string, failures: readonly PlaygroundCleanupFailure[]) { - super(`Playground session ${JSON.stringify(sessionId)} closed with cleanup failures.`); - this.name = 'PlaygroundSessionCloseError'; - this.sessionId = sessionId; - this.failures = failures; - } -} - -export interface PlaygroundServiceCloseFailure { - readonly error: unknown; - readonly sessionId: string; -} - -export class PlaygroundServiceCloseError extends Error { - readonly failures: readonly PlaygroundServiceCloseFailure[]; - - constructor(failures: readonly PlaygroundServiceCloseFailure[]) { - super('Playground service closed with session cleanup failures.'); - this.name = 'PlaygroundServiceCloseError'; - this.failures = failures; - } -} diff --git a/packages/agent-bundle/src/dev/playground/playground-durability.ts b/packages/agent-bundle/src/dev/playground/playground-durability.ts deleted file mode 100644 index 501db2721..000000000 --- a/packages/agent-bundle/src/dev/playground/playground-durability.ts +++ /dev/null @@ -1,52 +0,0 @@ -import { testModeGlobalValue } from '../../core/durability-test-hook.ts'; - -/** - * The playground store's durability test seam: the phase vocabulary its - * durability-injection tests target and the symbol-keyed hook readers. The - * `Symbol.for` key strings are part of the test contract - * (tests/support/durability.ts installs hooks under them) and must not change. - */ - -export type DirectorySyncReason = - | 'final-index-publication' - | 'layout-index-entry' - | 'layout-object-entry' - | 'layout-pending-index-entry' - | 'layout-project-entry' - | 'layout-storage-entry' - | 'new-file' - | 'object-created' - | 'owner-lock-create' - | 'owner-lock-create-recovery' - | 'owner-lock-recovery' - | 'owner-lock-release' - | 'pending-index-publication' - | 'session-metadata-rename'; -export type DurableFilePhase = 'event' | 'owner' | 'pending-index' | 'session-metadata'; -export type OwnerMutationReason = 'create-recovery' | 'recovery' | 'release'; -export type DurabilityTestPhase = - | 'after-final-index-link' - | 'before-owner-lock-recovery' - | 'before-final-index-link' - | `before-directory-fsync:${DirectorySyncReason}` - | `before-directory-open:${DirectorySyncReason}` - | `before-directory-sync:${DirectorySyncReason}` - | `before-file-fsync:${DurableFilePhase}` - | `before-file-write:${DurableFilePhase}` - | `before-owner-lock-unlink:${OwnerMutationReason}`; -export type DurabilityTestHook = (phase: DurabilityTestPhase, path: string) => Promise | void; - -/** Non-API test seam, unavailable unless the process explicitly runs in test mode. */ -const durabilityTestHookKey = Symbol.for('agent-bundle.playground-service.durability-test-hook'); -const durabilityTestPlatformKey = Symbol.for('agent-bundle.playground-service.durability-test-platform'); - -export const runDurabilityTestHook = (phase: DurabilityTestPhase, path: string): void => { - void testModeGlobalValue(durabilityTestHookKey)?.(phase, path); -}; - -export const runAsyncDurabilityTestHook = async (phase: DurabilityTestPhase, path: string): Promise => { - await testModeGlobalValue(durabilityTestHookKey)?.(phase, path); -}; - -export const durabilityPlatform = (): NodeJS.Platform => - testModeGlobalValue(durabilityTestPlatformKey) ?? process.platform; diff --git a/packages/agent-bundle/src/dev/playground/playground-protocol.ts b/packages/agent-bundle/src/dev/playground/playground-protocol.ts deleted file mode 100644 index 4ac82a528..000000000 --- a/packages/agent-bundle/src/dev/playground/playground-protocol.ts +++ /dev/null @@ -1,171 +0,0 @@ -import { CodedError } from '../../core/errors.ts'; -import type { DevLogSink } from '../logs/dev-log-service.ts'; - -export type PlaygroundJsonPrimitive = boolean | null | number | string; -export type PlaygroundJsonArray = readonly PlaygroundJsonValue[]; -export interface PlaygroundJsonObject { - readonly [key: string]: PlaygroundJsonValue; -} -export type PlaygroundJsonValue = PlaygroundJsonPrimitive | PlaygroundJsonArray | PlaygroundJsonObject; - -export type PlaygroundTraceSource = - | 'build' - | 'diagnostics' - | 'hook' - | 'host-preflight' - | 'mcp' - | 'project' - | 'response' - | 'script' - | 'skill-evidence' - | 'workspace-change'; - -export interface PlaygroundEpochIdentity { - readonly digest: string; - readonly id: string; -} - -export interface PlaygroundFixtureIdentity { - readonly digest: string; - readonly id: string; -} - -export interface PlaygroundTask { - readonly id: string; - readonly text: string; -} - -export interface PlaygroundTarget { - readonly digest?: string; - readonly name: string; -} - -export interface PlaygroundInvocation { - readonly intent: PlaygroundJsonObject; - readonly kind: string; -} - -export interface PlaygroundSessionIdentity { - readonly epoch: PlaygroundEpochIdentity; - readonly fixture: PlaygroundFixtureIdentity; - readonly invocation: PlaygroundInvocation; - readonly target: PlaygroundTarget; - readonly task: PlaygroundTask; -} - -export interface PlaygroundSessionInput extends PlaygroundSessionIdentity { - readonly sessionId?: string; -} - -export interface PlaygroundDurableOutcome { - readonly response?: string; - readonly status: string; - readonly workspace?: PlaygroundJsonObject; -} - -export interface PlaygroundEventInput { - readonly kind: string; - readonly raw: PlaygroundJsonValue; - readonly source: PlaygroundTraceSource; - readonly summary: string; -} - -export interface PlaygroundTraceEvent extends PlaygroundEventInput { - readonly rawEventRef: string; - readonly sequence: number; - readonly timestamp: string; -} - -export interface PlaygroundCleanupFailure { - readonly message: string; - readonly operation: 'admission' | 'subscriber'; -} - -export interface PlaygroundSession { - readonly cleanupFailures: readonly PlaygroundCleanupFailure[]; - readonly createdAt: string; - readonly id: string; - readonly identity: PlaygroundSessionIdentity; - readonly outcome?: PlaygroundDurableOutcome; - readonly state: 'closed' | 'finalized' | 'open'; -} - -export interface PlaygroundReplayCursor { - readonly afterSequence: number; -} - -export interface PlaygroundReplay { - readonly cursor: PlaygroundReplayCursor; - readonly events: readonly PlaygroundTraceEvent[]; - readonly session: PlaygroundSession; -} - -export interface PlaygroundExport { - readonly events: readonly PlaygroundTraceEvent[]; - readonly session: PlaygroundSession; -} - -export interface PlaygroundSelectedAssertion { - readonly evidence: PlaygroundJsonValue; - readonly expectation: PlaygroundJsonValue; - readonly id: string; - readonly kind: string; -} - -/** A deliberately narrow payload that converts unchanged into authored eval DSL. */ -export interface DraftEvalCase { - readonly assertions: readonly PlaygroundSelectedAssertion[]; - readonly epoch: PlaygroundEpochIdentity; - readonly fixture: PlaygroundFixtureIdentity; - readonly invocation: PlaygroundInvocation; - readonly outcome: PlaygroundDurableOutcome; - readonly target: PlaygroundTarget; - readonly task: PlaygroundTask; -} - -export interface PlaygroundSubscription { - close(): Promise; - readonly closed: boolean; -} - -export interface PlaygroundSubscribeOptions { - readonly afterSequence?: number; - readonly onEvent: (event: PlaygroundTraceEvent) => void | Promise; -} - -export interface PlaygroundServiceOptions { - /** Emits only durable append metadata after the event file fsync succeeds. */ - readonly logger?: DevLogSink; - readonly maxSubscriberQueue?: number; - readonly now?: () => Date; - readonly projectId: string; - readonly projectRoot: string; - readonly storageRoot: string; -} - -export type PlaygroundServiceErrorCode = - | 'PLAYGROUND_CURSOR_AHEAD' - | 'PLAYGROUND_CURSOR_INVALID' - | 'PLAYGROUND_CREDENTIAL_REJECTED' - | 'PLAYGROUND_OUTCOME_REQUIRED' - | 'PLAYGROUND_PROJECT_MISMATCH' - | 'PLAYGROUND_ROOT_INVALID' - | 'PLAYGROUND_SERVICE_CLOSED' - | 'PLAYGROUND_SESSION_CONFLICT' - | 'PLAYGROUND_SESSION_FINALIZED' - | 'PLAYGROUND_SESSION_ID_INVALID' - | 'PLAYGROUND_SESSION_NOT_FOUND' - | 'PLAYGROUND_SESSION_OWNED' - | 'PLAYGROUND_STORE_CORRUPT' - | 'PLAYGROUND_VALUE_INVALID'; - -export class PlaygroundServiceError extends CodedError { - constructor(code: PlaygroundServiceErrorCode, message: string) { - super('PlaygroundServiceError', code, message); - } -} - -export const playgroundServiceError = ( - code: PlaygroundServiceErrorCode, - message: string, -): PlaygroundServiceError => new PlaygroundServiceError(code, message); diff --git a/packages/agent-bundle/src/dev/playground/playground-store-codec.ts b/packages/agent-bundle/src/dev/playground/playground-store-codec.ts deleted file mode 100644 index fb1be5bf8..000000000 --- a/packages/agent-bundle/src/dev/playground/playground-store-codec.ts +++ /dev/null @@ -1,257 +0,0 @@ -import { readTornTailJsonl } from '../../core/durable-fs.ts'; -import { hasExactOwnKeys, isRecord, parseJsonWithoutDuplicateKeys } from '../../core/strict-json.ts'; -import { - PlaygroundServiceError, - playgroundServiceError as serviceError, - type PlaygroundCleanupFailure, - type PlaygroundDurableOutcome, - type PlaygroundJsonValue, - type PlaygroundSession, - type PlaygroundSessionIdentity, - type PlaygroundTraceEvent, -} from './playground-protocol.ts'; -import { - assertNoProviderCredentials, - normalizeEventInput, - normalizeIdentity, - normalizeOutcome, - safeSessionId, -} from './playground-values.ts'; - -/** - * Codecs for the playground store's persisted documents (session metadata, - * the session index, the event journal, and the owner lock), following the - * run-store / run-store-codec sibling convention: everything here is pure - * decoding and validation over already-read bytes, while the service owns - * every filesystem interaction. - */ - -export const sessionDocumentName = 'session.json'; -export const eventDocumentName = 'events.jsonl'; -export const ownerLockName = '.owner.lock'; - -export interface OwnerLock { - readonly pid: number; - readonly token: string; -} - -export interface PersistedSessionBase { - readonly cleanupFailures: readonly PlaygroundCleanupFailure[]; - readonly createdAt: string; - readonly identity: PlaygroundSessionIdentity; - readonly kind: 'agent-bundle-playground-session'; - readonly outcome?: PlaygroundDurableOutcome; - readonly projectId: string; - readonly sessionId: string; - readonly state: PlaygroundSession['state']; -} - -export interface PersistedSession extends PersistedSessionBase { - readonly storageObjectId: string; -} - -export interface PersistedSessionIndex { - readonly kind: 'agent-bundle-playground-session-index'; - readonly objectId: string; - readonly projectId: string; - readonly sessionId: string; -} - -export interface PersistedSessionContext { - readonly expectedId: string; - readonly expectedObjectId: string; - readonly projectId: string; -} - -export interface SessionIndexContext { - readonly projectId: string; - readonly sessionId: string; -} - -const canonicalOwnerToken = /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/u; - -const hasOptionalOwnKey = (value: Record, required: readonly string[], optional: string): boolean => - hasExactOwnKeys(value, required) || hasExactOwnKeys(value, [...required, optional]); - -const hasPersistedIdentitySchema = (value: unknown): boolean => { - if (!isRecord(value) - || !hasExactOwnKeys(value, ['epoch', 'fixture', 'invocation', 'target', 'task']) - || !isRecord(value.epoch) - || !hasExactOwnKeys(value.epoch, ['digest', 'id']) - || !isRecord(value.fixture) - || !hasExactOwnKeys(value.fixture, ['digest', 'id']) - || !isRecord(value.invocation) - || !hasExactOwnKeys(value.invocation, ['intent', 'kind']) - || !isRecord(value.invocation.intent) - || !isRecord(value.target) - || !hasOptionalOwnKey(value.target, ['name'], 'digest') - || !isRecord(value.task) - || !hasExactOwnKeys(value.task, ['id', 'text'])) { - return false; - } - return true; -}; - -const hasPersistedOutcomeSchema = (value: unknown): boolean => { - if (!isRecord(value)) return false; - return hasExactOwnKeys(value, ['status']) - || hasExactOwnKeys(value, ['status', 'response']) - || hasExactOwnKeys(value, ['status', 'workspace']) - || hasExactOwnKeys(value, ['status', 'response', 'workspace']); -}; - -export const parseOwnerLockDocument = (document: string): OwnerLock => { - let parsed: unknown; - try { - parsed = parseJsonWithoutDuplicateKeys(document); - } catch { - throw serviceError('PLAYGROUND_STORE_CORRUPT', 'Playground owner lock is malformed.'); - } - if (!isRecord(parsed) - || !hasExactOwnKeys(parsed, ['pid', 'token']) - || typeof parsed.pid !== 'number' - || !Number.isSafeInteger(parsed.pid) - || parsed.pid < 1 - || typeof parsed.token !== 'string' - || !canonicalOwnerToken.test(parsed.token)) { - throw serviceError('PLAYGROUND_STORE_CORRUPT', 'Playground owner lock is invalid.'); - } - return Object.freeze({ pid: parsed.pid, token: parsed.token }); -}; - -export const sameOwner = (left: OwnerLock, right: OwnerLock): boolean => - left.pid === right.pid && left.token === right.token; - -export const decodeSessionIndexDocument = ( - contents: string, - context: SessionIndexContext, -): PersistedSessionIndex => { - const { projectId, sessionId } = context; - let parsed: unknown; - try { - parsed = parseJsonWithoutDuplicateKeys(contents); - } catch { - throw serviceError('PLAYGROUND_STORE_CORRUPT', `Playground session ${JSON.stringify(sessionId)} has malformed index metadata.`); - } - if (!isRecord(parsed) - || !hasExactOwnKeys(parsed, ['kind', 'objectId', 'projectId', 'sessionId']) - || parsed.kind !== 'agent-bundle-playground-session-index' - || parsed.projectId !== projectId - || parsed.sessionId !== sessionId - || typeof parsed.objectId !== 'string') { - throw serviceError('PLAYGROUND_STORE_CORRUPT', `Playground session ${JSON.stringify(sessionId)} has invalid index metadata.`); - } - try { - safeSessionId(parsed.objectId); - } catch { - throw serviceError('PLAYGROUND_STORE_CORRUPT', `Playground session ${JSON.stringify(sessionId)} has invalid index metadata.`); - } - return Object.freeze({ - kind: 'agent-bundle-playground-session-index', - objectId: parsed.objectId, - projectId, - sessionId, - }); -}; - -export const decodePersistedSession = ( - document: string, - context: PersistedSessionContext, -): PersistedSession => { - const { expectedId, expectedObjectId, projectId } = context; - let parsed: unknown; - try { - parsed = parseJsonWithoutDuplicateKeys(document); - } catch { - throw serviceError('PLAYGROUND_STORE_CORRUPT', `Playground session ${JSON.stringify(expectedId)} has malformed metadata.`); - } - if (!isRecord(parsed)) { - throw serviceError('PLAYGROUND_STORE_CORRUPT', `Playground session ${JSON.stringify(expectedId)} has unsupported metadata.`); - } - try { - assertNoProviderCredentials(parsed as PlaygroundJsonValue); - } catch { - throw serviceError('PLAYGROUND_STORE_CORRUPT', `Playground session ${JSON.stringify(expectedId)} has invalid persisted values.`); - } - const expectedKeys = ['cleanupFailures', 'createdAt', 'identity', 'kind', 'outcome', 'projectId', 'sessionId', 'state', 'storageObjectId']; - const optionalOutcomeKeys = expectedKeys.filter((key) => key !== 'outcome'); - if ((!hasExactOwnKeys(parsed, expectedKeys) && !hasExactOwnKeys(parsed, optionalOutcomeKeys)) - || parsed.kind !== 'agent-bundle-playground-session') { - throw serviceError('PLAYGROUND_STORE_CORRUPT', `Playground session ${JSON.stringify(expectedId)} has unsupported metadata.`); - } - if (parsed.projectId !== projectId) { - throw serviceError('PLAYGROUND_PROJECT_MISMATCH', `Playground session ${JSON.stringify(expectedId)} belongs to a different project.`); - } - if (parsed.sessionId !== expectedId || typeof parsed.createdAt !== 'string') { - throw serviceError('PLAYGROUND_STORE_CORRUPT', `Playground session ${JSON.stringify(expectedId)} has invalid identity metadata.`); - } - if (typeof parsed.storageObjectId !== 'string' || parsed.storageObjectId !== expectedObjectId) { - throw serviceError('PLAYGROUND_STORE_CORRUPT', `Playground session ${JSON.stringify(expectedId)} has invalid object metadata.`); - } - if (parsed.state !== 'open' && parsed.state !== 'finalized' && parsed.state !== 'closed') { - throw serviceError('PLAYGROUND_STORE_CORRUPT', `Playground session ${JSON.stringify(expectedId)} has invalid state metadata.`); - } - if (!hasPersistedIdentitySchema(parsed.identity) - || (parsed.outcome !== undefined && !hasPersistedOutcomeSchema(parsed.outcome))) { - throw serviceError('PLAYGROUND_STORE_CORRUPT', `Playground session ${JSON.stringify(expectedId)} has invalid persisted values.`); - } - let identity: PlaygroundSessionIdentity; - let outcome: PlaygroundDurableOutcome | undefined; - try { - identity = normalizeIdentity(parsed.identity as PlaygroundSessionIdentity); - outcome = parsed.outcome === undefined ? undefined : normalizeOutcome(parsed.outcome as PlaygroundDurableOutcome); - } catch { - throw serviceError('PLAYGROUND_STORE_CORRUPT', `Playground session ${JSON.stringify(expectedId)} has invalid persisted values.`); - } - if ((parsed.state === 'finalized' || parsed.state === 'closed') && outcome === undefined) { - throw serviceError('PLAYGROUND_STORE_CORRUPT', `Playground session ${JSON.stringify(expectedId)} is missing a durable outcome.`); - } - const cleanupFailures = Array.isArray(parsed.cleanupFailures) - ? parsed.cleanupFailures.map((value): PlaygroundCleanupFailure => { - if (!isRecord(value) || !hasExactOwnKeys(value, ['message', 'operation']) - || value.operation !== 'subscriber' || typeof value.message !== 'string') { - throw serviceError('PLAYGROUND_STORE_CORRUPT', `Playground session ${JSON.stringify(expectedId)} has invalid cleanup failures.`); - } - return Object.freeze({ message: value.message, operation: 'subscriber' }); - }) - : (() => { throw serviceError('PLAYGROUND_STORE_CORRUPT', `Playground session ${JSON.stringify(expectedId)} has missing cleanup failures.`); })(); - const documentBase = { - cleanupFailures: Object.freeze(cleanupFailures), - createdAt: parsed.createdAt, - identity, - kind: 'agent-bundle-playground-session', - ...(outcome === undefined ? {} : { outcome }), - projectId, - sessionId: expectedId, - state: parsed.state, - } as const; - return Object.freeze({ - ...documentBase, - storageObjectId: expectedObjectId, - }); -}; - -export const decodeEventLog = (contents: string): readonly PlaygroundTraceEvent[] => - readTornTailJsonl(contents, { - decode: (parsed, index) => { - if (!isRecord(parsed) || !hasExactOwnKeys(parsed, ['kind', 'raw', 'rawEventRef', 'sequence', 'source', 'summary', 'timestamp'])) { - throw serviceError('PLAYGROUND_STORE_CORRUPT', 'Playground event log contains an invalid record envelope.'); - } - try { - assertNoProviderCredentials(parsed as PlaygroundJsonValue); - const input = normalizeEventInput(parsed); - const sequence = parsed.sequence; - if (!Number.isSafeInteger(sequence) || sequence !== index + 1 || typeof parsed.timestamp !== 'string' - || parsed.rawEventRef !== `${eventDocumentName}#${sequence}`) { - throw serviceError('PLAYGROUND_STORE_CORRUPT', 'Playground event log contains an invalid sequence record.'); - } - return Object.freeze({ ...input, rawEventRef: parsed.rawEventRef, sequence, timestamp: parsed.timestamp }); - } catch (error) { - if (error instanceof PlaygroundServiceError && error.code === 'PLAYGROUND_STORE_CORRUPT') throw error; - throw serviceError('PLAYGROUND_STORE_CORRUPT', 'Playground event log contains invalid persisted values.'); - } - }, - emptyRecord: () => serviceError('PLAYGROUND_STORE_CORRUPT', 'Playground event log contains an empty completed record.'), - malformedRecord: () => serviceError('PLAYGROUND_STORE_CORRUPT', 'Playground event log contains a malformed completed record.'), - sequenceViolation: () => serviceError('PLAYGROUND_STORE_CORRUPT', 'Playground event log contains an invalid sequence record.'), - }).records; diff --git a/packages/agent-bundle/src/dev/playground/playground-store-layout.ts b/packages/agent-bundle/src/dev/playground/playground-store-layout.ts deleted file mode 100644 index cff96d04e..000000000 --- a/packages/agent-bundle/src/dev/playground/playground-store-layout.ts +++ /dev/null @@ -1,116 +0,0 @@ -import { lstat, mkdir, realpath } from 'node:fs/promises'; -import { isAbsolute, join, relative, resolve, sep } from 'node:path'; - -import { isErrno } from '../../core/errors.ts'; -import { isInsideOrEqual } from '../../core/paths.ts'; -import type { DirectorySyncReason } from './playground-durability.ts'; -import { playgroundServiceError as serviceError } from './playground-protocol.ts'; - -/** - * The playground store's on-disk layout: the directory names under the - * storage root and the validated, durably created bootstrap that resolves - * them to real paths. - */ - -export const objectDirectoryName = 'session-objects'; -export const indexDirectoryName = 'session-index'; -export const pendingIndexDirectoryName = '.pending'; - -/** Resolved real-path roots of a validated playground storage layout. */ -export interface PlaygroundStorageLayout { - readonly indexRoot: string; - readonly objectRoot: string; - readonly pendingIndexRoot: string; -} - -type SyncDirectory = (path: string, reason: DirectorySyncReason) => void; - -const createLayoutDirectory = async ( - path: string, - parent: string, - reason: DirectorySyncReason, - syncDirectory: SyncDirectory, -): Promise => { - let created = false; - try { - await mkdir(path); - created = true; - } catch (error) { - if (!isErrno(error, 'EEXIST')) throw error; - } - const stat = await lstat(path); - if (!stat.isDirectory() && !stat.isSymbolicLink()) { - throw serviceError('PLAYGROUND_ROOT_INVALID', 'Playground storage layout must contain real directories.'); - } - if (created) syncDirectory(parent, reason); -}; - -const createStorageRoot = async ( - projectRoot: string, - storageRoot: string, - syncDirectory: SyncDirectory, -): Promise => { - const storageRelativePath = relative(projectRoot, storageRoot); - if (storageRelativePath === '') return; - const segments = storageRelativePath.split(sep); - if (segments.some((segment) => segment === '' || segment === '.' || segment === '..')) { - throw serviceError('PLAYGROUND_ROOT_INVALID', 'Playground storage root must be a contained directory path.'); - } - let parent = projectRoot; - for (let index = 0; index < segments.length; index += 1) { - const path = join(parent, segments[index]!); - await createLayoutDirectory( - path, - parent, - index === segments.length - 1 ? 'layout-storage-entry' : 'layout-project-entry', - syncDirectory, - ); - parent = path; - } -}; - -/** Validates containment, durably creates the layout, and resolves its real-path roots. */ -export const initializePlaygroundStorageLayout = async ( - projectRoot: string, - storageRoot: string, - syncDirectory: SyncDirectory, -): Promise => { - if (!isAbsolute(projectRoot) || !isAbsolute(storageRoot)) { - throw serviceError('PLAYGROUND_ROOT_INVALID', 'Playground storage root must be an absolute project-owned path.'); - } - const requestedProjectRoot = resolve(projectRoot); - const requestedStorageRoot = resolve(storageRoot); - if (!isInsideOrEqual(requestedProjectRoot, requestedStorageRoot)) { - throw serviceError('PLAYGROUND_ROOT_INVALID', 'Playground storage root must be contained by the configured project root.'); - } - const resolvedProjectRoot = await realpath(requestedProjectRoot); - await createStorageRoot(requestedProjectRoot, requestedStorageRoot, syncDirectory); - const storageStat = await lstat(requestedStorageRoot); - if (storageStat.isSymbolicLink()) { - throw serviceError('PLAYGROUND_ROOT_INVALID', 'Playground storage root must not be a symbolic link.'); - } - const resolvedStorageRoot = await realpath(requestedStorageRoot); - if (!isInsideOrEqual(resolvedProjectRoot, resolvedStorageRoot)) { - throw serviceError('PLAYGROUND_ROOT_INVALID', 'Playground storage root resolves outside the configured project root.'); - } - const objectRoot = join(requestedStorageRoot, objectDirectoryName); - await createLayoutDirectory(objectRoot, requestedStorageRoot, 'layout-object-entry', syncDirectory); - const resolvedObjectRoot = await realpath(objectRoot); - if (!isInsideOrEqual(resolvedStorageRoot, resolvedObjectRoot)) { - throw serviceError('PLAYGROUND_ROOT_INVALID', 'Playground session object root resolves outside configured storage.'); - } - const indexRoot = join(requestedStorageRoot, indexDirectoryName); - const pendingIndexRoot = join(indexRoot, pendingIndexDirectoryName); - await createLayoutDirectory(indexRoot, requestedStorageRoot, 'layout-index-entry', syncDirectory); - await createLayoutDirectory(pendingIndexRoot, indexRoot, 'layout-pending-index-entry', syncDirectory); - const resolvedIndexRoot = await realpath(indexRoot); - const resolvedPendingIndexRoot = await realpath(pendingIndexRoot); - if (!isInsideOrEqual(resolvedStorageRoot, resolvedIndexRoot) || !isInsideOrEqual(resolvedIndexRoot, resolvedPendingIndexRoot)) { - throw serviceError('PLAYGROUND_ROOT_INVALID', 'Playground session index root resolves outside configured storage.'); - } - return Object.freeze({ - indexRoot: resolvedIndexRoot, - objectRoot: resolvedObjectRoot, - pendingIndexRoot: resolvedPendingIndexRoot, - }); -}; diff --git a/packages/agent-bundle/src/dev/playground/playground-store.ts b/packages/agent-bundle/src/dev/playground/playground-store.ts index 9c559c8c4..5883663be 100644 --- a/packages/agent-bundle/src/dev/playground/playground-store.ts +++ b/packages/agent-bundle/src/dev/playground/playground-store.ts @@ -4,6 +4,7 @@ import { lstat, mkdir, open, realpath, rename, rm, unlink } from 'node:fs/promis import { basename, dirname, isAbsolute, join, relative, resolve, sep } from 'node:path'; import { serialQueue, type SerialQueue } from '../../core/async.ts'; +import { containsProviderCredential, isCredentialKey } from '../../core/credentials.ts'; import { isErrno } from '../../core/errors.ts'; import { isInsideOrEqual } from '../../core/paths.ts'; import { hasExactOwnKeys, isRecord, parseJsonWithoutDuplicateKeys } from '../../core/strict-json.ts'; @@ -336,23 +337,6 @@ const traceSources: ReadonlySet = new Set([ const isTraceSource = (value: unknown): value is PlaygroundTraceSource => typeof value === 'string' && traceSources.has(value); -const providerCredentialPatterns = Object.freeze([ - /\bsk-(?:proj-|ant-|live-)?[a-z0-9_-]{16,}\b/iu, - /\b(?:gh[pousr]_[a-z0-9]{20,}|github_pat_[a-z0-9_]{20,}|xox[baprs]-[a-z0-9-]{16,}|akia[a-z0-9]{16})\b/iu, - /\bbearer[ \t]+[a-z0-9._~+/=-]{20,}\b/iu, -]); - -const sensitiveKey = (key: string): boolean => { - const segments = key - .replace(/([a-z0-9])([A-Z])/gu, '$1 $2') - .toLocaleLowerCase('en-US') - .split(/[^a-z0-9]+/u) - .filter((segment) => segment.length > 0); - const compact = segments.join(''); - return segments.some((segment) => ['authorization', 'credential', 'credentials', 'password', 'secret', 'token'].includes(segment)) - || /(?:apikey|apitoken|authtoken|accesstoken)$/u.test(compact); -}; - const hasOptionalOwnKey = (value: Record, required: readonly string[], optional: string): boolean => hasExactOwnKeys(value, required) || hasExactOwnKeys(value, [...required, optional]); @@ -531,7 +515,7 @@ const clone = (value: T, label = 'Playground value'): T => json(value, label) const assertNoProviderCredentials = (value: PlaygroundJsonValue): void => { if (typeof value === 'string') { - if (providerCredentialPatterns.some((pattern) => pattern.test(value))) { + if (containsProviderCredential(value)) { throw serviceError('PLAYGROUND_CREDENTIAL_REJECTED', 'Playground records must not contain provider credential material.'); } return; @@ -542,7 +526,7 @@ const assertNoProviderCredentials = (value: PlaygroundJsonValue): void => { } if (isRecord(value)) { for (const [key, item] of Object.entries(value)) { - if (sensitiveKey(key)) { + if (isCredentialKey(key)) { throw serviceError('PLAYGROUND_CREDENTIAL_REJECTED', 'Playground records must not contain provider credential material.'); } assertNoProviderCredentials(item); diff --git a/packages/agent-bundle/src/dev/playground/playground-subscriptions.ts b/packages/agent-bundle/src/dev/playground/playground-subscriptions.ts deleted file mode 100644 index c1200c4c7..000000000 --- a/packages/agent-bundle/src/dev/playground/playground-subscriptions.ts +++ /dev/null @@ -1,112 +0,0 @@ -import { errorMessage } from '../../core/errors.ts'; -import type { - PlaygroundCleanupFailure, - PlaygroundSubscribeOptions, - PlaygroundTraceEvent, -} from './playground-protocol.ts'; -import { snapshotEvent } from './playground-values.ts'; - -export interface PlaygroundSubscriptionEntry { - active: boolean; - closed: boolean; - delivery: Promise; - draining: boolean; - readonly onEvent: PlaygroundSubscribeOptions['onEvent']; - readonly queue: PlaygroundTraceEvent[]; -} - -/** Owns bounded subscriber queues and isolates listener failures from durable session state. */ -export class PlaygroundSubscriptionSet { - readonly #cleanupFailures: PlaygroundCleanupFailure[]; - readonly #entries = new Set(); - readonly #maximumQueue: number; - - constructor(maximumQueue: number, cleanupFailures: PlaygroundCleanupFailure[]) { - this.#maximumQueue = maximumQueue; - this.#cleanupFailures = cleanupFailures; - } - - add( - backlog: readonly PlaygroundTraceEvent[], - onEvent: PlaygroundSubscribeOptions['onEvent'], - ): PlaygroundSubscriptionEntry { - const admitted = backlog.length <= this.#maximumQueue; - const subscription: PlaygroundSubscriptionEntry = { - active: admitted, - closed: !admitted, - delivery: Promise.resolve(), - draining: false, - onEvent, - queue: admitted ? [...backlog] : [], - }; - if (admitted) { - this.#entries.add(subscription); - this.#drain(subscription); - } - return subscription; - } - - entries(): readonly PlaygroundSubscriptionEntry[] { - return Object.freeze([...this.#entries]); - } - - publish(event: PlaygroundTraceEvent): void { - for (const subscription of [...this.#entries]) { - if (!subscription.active) continue; - if (subscription.queue.length >= this.#maximumQueue) { - this.deactivate(subscription); - continue; - } - subscription.queue.push(event); - this.#drain(subscription); - } - } - - async drain(subscriptions: readonly PlaygroundSubscriptionEntry[] = this.entries()): Promise { - for (const subscription of subscriptions) this.#drain(subscription); - const settled = await Promise.allSettled(subscriptions.map((subscription) => subscription.delivery)); - for (const result of settled) { - if (result.status === 'rejected') this.#recordFailure(result.reason); - } - } - - deactivate(subscription: PlaygroundSubscriptionEntry): void { - subscription.active = false; - subscription.closed = true; - subscription.queue.length = 0; - this.#entries.delete(subscription); - } - - async waitFor(subscription: PlaygroundSubscriptionEntry): Promise { - await subscription.delivery; - } - - #drain(subscription: PlaygroundSubscriptionEntry): void { - if (!subscription.active || subscription.draining) return; - subscription.draining = true; - subscription.delivery = (async () => { - try { - while (subscription.active && subscription.queue.length > 0) { - const event = subscription.queue.shift()!; - try { - await subscription.onEvent(snapshotEvent(event)); - } catch (error) { - this.#recordFailure(error); - this.deactivate(subscription); - } - } - } finally { - subscription.draining = false; - if (subscription.active && subscription.queue.length > 0) this.#drain(subscription); - } - })(); - void subscription.delivery.catch((error: unknown) => { - this.#recordFailure(error); - this.deactivate(subscription); - }); - } - - #recordFailure(error: unknown): void { - this.#cleanupFailures.push(Object.freeze({ message: errorMessage(error), operation: 'subscriber' })); - } -} diff --git a/packages/agent-bundle/src/dev/playground/playground-values.ts b/packages/agent-bundle/src/dev/playground/playground-values.ts deleted file mode 100644 index 41908ec9d..000000000 --- a/packages/agent-bundle/src/dev/playground/playground-values.ts +++ /dev/null @@ -1,194 +0,0 @@ -import { containsProviderCredential, isCredentialKey } from '../../core/credentials.ts'; -import { isRecord } from '../../core/strict-json.ts'; -import { - playgroundServiceError, - type PlaygroundDurableOutcome, - type PlaygroundEventInput, - type PlaygroundJsonObject, - type PlaygroundJsonValue, - type PlaygroundReplayCursor, - type PlaygroundSelectedAssertion, - type PlaygroundSessionIdentity, - type PlaygroundTraceEvent, - type PlaygroundTraceSource, -} from './playground-protocol.ts'; -import { deepFreeze } from '../../core/freeze.ts'; - - -const pathSegment = /^[a-z0-9][a-z0-9._-]*$/iu; -const traceSources: ReadonlySet = new Set([ - 'build', - 'diagnostics', - 'hook', - 'host-preflight', - 'mcp', - 'project', - 'response', - 'script', - 'skill-evidence', - 'workspace-change', -]); -const isTraceSource = (value: unknown): value is PlaygroundTraceSource => - typeof value === 'string' && traceSources.has(value); - -export const nonempty = (value: unknown, label: string): string => { - if (typeof value !== 'string' || value.length === 0) { - throw playgroundServiceError('PLAYGROUND_VALUE_INVALID', `${label} must be a nonempty string.`); - } - return value; -}; - -export const safeSessionId = (value: string): string => { - if (!pathSegment.test(value) || value === '.' || value === '..') { - throw playgroundServiceError('PLAYGROUND_SESSION_ID_INVALID', 'Playground session id must be a path-safe identifier.'); - } - return value; -}; - -const json = (value: unknown, label: string, seen = new WeakSet()): PlaygroundJsonValue => { - if (value === null || typeof value === 'boolean' || typeof value === 'string') return value; - if (typeof value === 'number') { - if (!Number.isFinite(value)) throw playgroundServiceError('PLAYGROUND_VALUE_INVALID', `${label} must be JSON-compatible.`); - return value; - } - if (typeof value !== 'object') throw playgroundServiceError('PLAYGROUND_VALUE_INVALID', `${label} must be JSON-compatible.`); - if (seen.has(value)) throw playgroundServiceError('PLAYGROUND_VALUE_INVALID', `${label} must not contain cycles.`); - seen.add(value); - if (Array.isArray(value)) { - const copied = Object.freeze(value.map((item) => json(item, label, seen))); - seen.delete(value); - return copied; - } - const prototype = Object.getPrototypeOf(value); - if (prototype !== Object.prototype && prototype !== null) { - seen.delete(value); - throw playgroundServiceError('PLAYGROUND_VALUE_INVALID', `${label} must be JSON-compatible.`); - } - const copied = Object.create(null) as Record; - for (const key of Object.keys(value).sort()) { - const descriptor = Object.getOwnPropertyDescriptor(value, key); - if (descriptor === undefined || descriptor.get !== undefined || descriptor.set !== undefined) { - seen.delete(value); - throw playgroundServiceError('PLAYGROUND_VALUE_INVALID', `${label} must not contain accessors.`); - } - copied[key] = json(descriptor.value, label, seen); - } - seen.delete(value); - return Object.freeze(copied); -}; - -const jsonObject = (value: unknown, label: string): PlaygroundJsonObject => { - const copied = json(value, label); - if (!isRecord(copied)) throw playgroundServiceError('PLAYGROUND_VALUE_INVALID', `${label} must be a JSON object.`); - return copied; -}; - -export const clone = (value: T, label = 'Playground value'): T => json(value, label) as T; - -export const assertNoProviderCredentials = (value: PlaygroundJsonValue): void => { - if (typeof value === 'string') { - if (containsProviderCredential(value)) { - throw playgroundServiceError('PLAYGROUND_CREDENTIAL_REJECTED', 'Playground records must not contain provider credential material.'); - } - return; - } - if (Array.isArray(value)) { - for (const item of value) assertNoProviderCredentials(item); - return; - } - if (isRecord(value)) { - for (const [key, item] of Object.entries(value)) { - if (isCredentialKey(key)) { - throw playgroundServiceError('PLAYGROUND_CREDENTIAL_REJECTED', 'Playground records must not contain provider credential material.'); - } - assertNoProviderCredentials(item); - } - } -}; - -export const normalizeIdentity = (value: PlaygroundSessionIdentity): PlaygroundSessionIdentity => { - const identity = deepFreeze({ - epoch: { - digest: nonempty(value?.epoch?.digest, 'Playground epoch digest'), - id: nonempty(value?.epoch?.id, 'Playground epoch id'), - }, - fixture: { - digest: nonempty(value?.fixture?.digest, 'Playground fixture digest'), - id: nonempty(value?.fixture?.id, 'Playground fixture id'), - }, - invocation: { - intent: jsonObject(value?.invocation?.intent, 'Playground invocation intent'), - kind: nonempty(value?.invocation?.kind, 'Playground invocation kind'), - }, - target: { - ...(value?.target?.digest === undefined ? {} : { digest: nonempty(value.target.digest, 'Playground target digest') }), - name: nonempty(value?.target?.name, 'Playground target name'), - }, - task: { - id: nonempty(value?.task?.id, 'Playground task id'), - text: nonempty(value?.task?.text, 'Playground task text'), - }, - }); - assertNoProviderCredentials(identity as PlaygroundJsonValue); - return identity; -}; - -export const normalizeOutcome = (value: PlaygroundDurableOutcome): PlaygroundDurableOutcome => { - const outcome = Object.freeze({ - ...(value?.response === undefined ? {} : { response: nonempty(value.response, 'Playground outcome response') }), - status: nonempty(value?.status, 'Playground outcome status'), - ...(value?.workspace === undefined ? {} : { workspace: jsonObject(value.workspace, 'Playground outcome workspace') }), - }); - assertNoProviderCredentials(outcome as PlaygroundJsonValue); - return outcome; -}; - -export const sameOutcome = (left: PlaygroundDurableOutcome, right: PlaygroundDurableOutcome): boolean => - JSON.stringify(left) === JSON.stringify(right); - -export const normalizeEventInput = (value: unknown): PlaygroundEventInput => { - if (!isRecord(value)) { - throw playgroundServiceError('PLAYGROUND_VALUE_INVALID', 'Playground event must be a JSON object.'); - } - const source = value.source; - if (!isTraceSource(source)) { - throw playgroundServiceError('PLAYGROUND_VALUE_INVALID', 'Playground event source is unsupported.'); - } - const event = Object.freeze({ - kind: nonempty(value.kind, 'Playground event kind'), - raw: json(value.raw, 'Playground event raw value'), - source, - summary: nonempty(value.summary, 'Playground event summary'), - }); - assertNoProviderCredentials(event as PlaygroundJsonValue); - return event; -}; - -export const snapshotEvent = (value: PlaygroundTraceEvent): PlaygroundTraceEvent => Object.freeze({ - kind: value.kind, - raw: clone(value.raw, 'Playground event raw value'), - rawEventRef: value.rawEventRef, - sequence: value.sequence, - source: value.source, - summary: value.summary, - timestamp: value.timestamp, -}); - -export const normalizeCursor = (value: PlaygroundReplayCursor | undefined): number => { - const sequence = value?.afterSequence ?? 0; - if (!Number.isSafeInteger(sequence) || sequence < 0) { - throw playgroundServiceError('PLAYGROUND_CURSOR_INVALID', 'Playground replay cursor must be a non-negative safe integer.'); - } - return sequence; -}; - -export const normalizeAssertion = (value: PlaygroundSelectedAssertion): PlaygroundSelectedAssertion => { - const assertion = Object.freeze({ - evidence: json(value?.evidence, 'Playground assertion evidence'), - expectation: json(value?.expectation, 'Playground assertion expectation'), - id: nonempty(value?.id, 'Playground assertion id'), - kind: nonempty(value?.kind, 'Playground assertion kind'), - }); - assertNoProviderCredentials(assertion as PlaygroundJsonValue); - return assertion; -}; diff --git a/packages/agent-bundle/src/dev/runtime-routes.ts b/packages/agent-bundle/src/dev/runtime-routes.ts index 380a40ee5..2ffc86b6a 100644 --- a/packages/agent-bundle/src/dev/runtime-routes.ts +++ b/packages/agent-bundle/src/dev/runtime-routes.ts @@ -2,6 +2,17 @@ import type { IncomingMessage, ServerResponse } from 'node:http'; import { hasOnlyOwnKeys, isPlainRecord } from '../core/strict-json.ts'; +import { + diagnostic, + isJsonRequest, + isRequestDiagnostic, + rawPathname, + readBody, + requestError, + responseDiagnostic, + responseJson as writeJsonResponse, +} from './http.ts'; + import { DevRuntimeGenerationConflictError, DevRuntimeUnavailableError, @@ -17,17 +28,10 @@ import type { } from './runtime-protocol.ts'; import { freezeJsonValue, type JsonValue } from './types.ts'; -const bodyLimit = 64 * 1024; const assetLimit = 4 * 1024 * 1024; // 16 MiB leaves room for rich timelines while bounding retained replacement snapshots. const agentDocumentResponseLimit = 16 * 1024 * 1024; -interface RequestDiagnostic { - readonly code: string; - readonly message: string; - readonly status: number; -} - type Route = | Readonly<{ readonly kind: 'status' | 'surfaces' | 'runs' | 'state-reset' }> | Readonly<{ readonly id: string; readonly kind: 'run' | 'document' | 'flight' | 'replay' }> @@ -72,80 +76,6 @@ const loadAgentDocumentRuntime = async (): Promise = return agentDocumentRuntimePromise; }; -const diagnostic = (code: string, message: string, status: number): RequestDiagnostic => ({ code, message, status }); - -const requestError = (value: RequestDiagnostic): RequestDiagnostic & Error => Object.assign( - new Error(value.message), - value, -); - -const isRequestDiagnostic = (value: unknown): value is RequestDiagnostic => - typeof value === 'object' && value !== null && - typeof (value as Partial).code === 'string' && - typeof (value as Partial).message === 'string' && - typeof (value as Partial).status === 'number'; - -const responseDiagnostic = (response: ServerResponse, value: RequestDiagnostic): void => { - if (response.headersSent || response.writableEnded) { - response.destroy(); - return; - } - response.writeHead(value.status, { 'content-type': 'application/json; charset=utf-8' }); - response.end(JSON.stringify({ diagnostic: { code: value.code, message: value.message } })); -}; - -const responseJson = (response: ServerResponse, body: unknown): void => { - response.writeHead(200, { 'content-type': 'application/json; charset=utf-8' }); - response.end(JSON.stringify(body)); -}; - -const singleHeader = (value: string | readonly string[] | undefined): string | undefined => - typeof value === 'string' ? value : undefined; - -const unquoteHeaderValue = (value: string): string | undefined => { - if (/^[!#$%&'*+\-.^_`|~0-9A-Za-z]+$/u.test(value)) return value; - if (!/^"(?:[^"\\\r\n]|\\[\t !-~])*"$/u.test(value)) return undefined; - return value.slice(1, -1).replace(/\\([\t !-~])/gu, '$1'); -}; - -const isJsonRequest = (request: IncomingMessage): boolean => { - const contentType = singleHeader(request.headers['content-type']); - if (contentType === undefined) return false; - const parts = contentType.split(';').map((part) => part.trim()); - if (parts.shift()?.toLowerCase() !== 'application/json') return false; - if (parts.length === 0) return true; - if (parts.length !== 1) return false; - const parameter = parts[0]!; - const equals = parameter.indexOf('='); - if (equals < 1 || parameter.slice(0, equals).trim().toLowerCase() !== 'charset') return false; - return unquoteHeaderValue(parameter.slice(equals + 1).trim())?.toLowerCase() === 'utf-8'; -}; - -const readBody = async (request: IncomingMessage): Promise => new Promise((resolvePromise, rejectPromise) => { - let size = 0; - let tooLarge = false; - const chunks: Buffer[] = []; - request.on('data', (chunk: Buffer) => { - size += chunk.length; - if (size > bodyLimit) { - tooLarge = true; - return; - } - if (!tooLarge) chunks.push(chunk); - }); - request.once('end', () => { - if (tooLarge) { - rejectPromise(requestError(diagnostic('AB8010', 'Request body exceeds 64 KiB.', 413))); - return; - } - resolvePromise(Buffer.concat(chunks).toString('utf8')); - }); - request.once('error', rejectPromise); -}); - -const rawPathname = (requestTarget: string | undefined): string => - requestTarget?.split(/[?#]/u, 1)[0] ?? ''; - const hasQuery = (requestTarget: string | undefined): boolean => requestTarget?.includes('?') ?? false; const runtimePathError = (): never => { @@ -366,18 +296,18 @@ export class RuntimeRoutes { if (parsed.kind === 'status') { if (method !== 'GET') return responseDiagnostic(response, diagnostic('AB8007', 'Route does not accept this method.', 405)); onlyQuery(request.url, undefined); - return responseJson(response, { status: this.#runtime?.status() ?? null }); + return writeJsonResponse(response, { status: this.#runtime?.status() ?? null }); } if (parsed.kind === 'surfaces') { if (method !== 'GET') return responseDiagnostic(response, diagnostic('AB8007', 'Route does not accept this method.', 405)); onlyQuery(request.url, undefined); - return responseJson(response, { surfaces: this.#runtime?.surfaces() ?? [] }); + return writeJsonResponse(response, { surfaces: this.#runtime?.surfaces() ?? [] }); } const session = this.#session(); if (parsed.kind === 'runs') { if (method === 'POST') { onlyQuery(request.url, undefined); - return responseJson(response, { + return writeJsonResponse(response, { run: assertRunOwned(session, await session.invoke(invocation(await jsonBody(request), session.surfaces()))), }); } @@ -389,12 +319,12 @@ export class RuntimeRoutes { if (runs.some((run) => run.vector.providerSessionId !== provider)) { throw requestError(diagnostic('AB8205', 'Runtime request could not be completed.', 500)); } - return responseJson(response, { providerSessionId: provider, runs }); + return writeJsonResponse(response, { providerSessionId: provider, runs }); } if (parsed.kind === 'run') { onlyQuery(request.url, undefined); if (method !== 'GET') return responseDiagnostic(response, diagnostic('AB8007', 'Route does not accept this method.', 405)); - return responseJson(response, { run: assertRunOwned(session, session.run(parsed.id)) }); + return writeJsonResponse(response, { run: assertRunOwned(session, session.run(parsed.id)) }); } if (parsed.kind === 'flight') { onlyQuery(request.url, undefined); @@ -446,7 +376,7 @@ export class RuntimeRoutes { responseBytes += separatorBytes + eventBytes; events.push(event); } - return responseJson(response, { events }); + return writeJsonResponse(response, { events }); } catch (error) { if (isRequestDiagnostic(error)) throw error; throw requestError(diagnostic('AB8208', 'Stored Flight could not be decoded as an Agent Document.', 409)); @@ -457,12 +387,12 @@ export class RuntimeRoutes { if (method !== 'POST') return responseDiagnostic(response, diagnostic('AB8007', 'Route does not accept this method.', 405)); const replayRequest = replay(await jsonBody(request)); if (replayRequest.runId !== parsed.id) return invalidShape(); - return responseJson(response, { run: assertRunOwned(session, await session.replay(replayRequest)) }); + return writeJsonResponse(response, { run: assertRunOwned(session, await session.replay(replayRequest)) }); } if (parsed.kind === 'state-reset') { onlyQuery(request.url, undefined); if (method !== 'POST') return responseDiagnostic(response, diagnostic('AB8007', 'Route does not accept this method.', 405)); - return responseJson(response, { state: await session.resetState(reset(await jsonBody(request))) }); + return writeJsonResponse(response, { state: await session.resetState(reset(await jsonBody(request))) }); } if (parsed.kind !== 'asset') return; if (method !== 'GET') return responseDiagnostic(response, diagnostic('AB8007', 'Route does not accept this method.', 405)); diff --git a/packages/agent-bundle/src/eval/run-store-codec.ts b/packages/agent-bundle/src/eval/run-store-codec.ts deleted file mode 100644 index 758f455b1..000000000 --- a/packages/agent-bundle/src/eval/run-store-codec.ts +++ /dev/null @@ -1,629 +0,0 @@ -import { randomUUID } from 'node:crypto'; -import { isAbsolute, win32 } from 'node:path'; - -import { isRecord, snapshotStrictJsonValue, type JsonValue } from '../core/strict-json.ts'; -import { findCredentialConfiguration } from './credentials.ts'; -import { storeError } from './errors.ts'; -import { provenanceIdentifierPattern } from './provenance.ts'; -import type { - EvalAssertionOutcome, - EvalAssertionResult, - EvalHarnessFailure, - EvalPluginFailure, - EvalTrialEvidence, -} from './types.ts'; -import type { - EvalArtifactBinding, - EvalRunEvent, - EvalRunEventInput, - EvalRunOwner, - EvalRunProvenance, - EvalRunRecord, - EvalRunSummary, - EvalTrialInvocationProvenance, - EvalTrialProvenance, - EvalTrialRecord, - EvalTrialSemanticGraderProvenance, - EvalTrialUsage, - ListEvalRunsOptions, -} from './run-store-types.ts'; - -export const safeSegment = /^[a-z0-9][a-z0-9._-]*$/iu; -export const maximumTrialRecordBytes = 1024 * 1024; -const maximumProvenanceTextLength = 256; - -export const requireSafeSegment = (value: unknown, label: string): string => { - if (typeof value !== 'string' || !safeSegment.test(value)) { - throw storeError('EVAL_RUN_RECORD_INVALID', `${label} must be a path-safe identifier.`); - } - return value; -}; - -export const requireSafeRelativePath = (value: string, label: string): string => { - const segments = value.split('/'); - if ( - value.length === 0 || - value.includes('\\') || - segments.some((segment) => segment !== '.agent-bundle' && !safeSegment.test(segment)) - ) { - throw storeError('EVAL_RUN_RECORD_INVALID', `${label} must be a path-safe relative path.`); - } - return value; -}; - -export const requireRunsDir = (value: unknown): string => { - if ( - typeof value !== 'string' || - value.length === 0 || - value.includes('\\') || - isAbsolute(value) || - win32.isAbsolute(value) || - value.split('/').some((segment) => segment.length === 0 || segment === '.' || segment === '..') - ) { - throw storeError('EVAL_RUN_RECORD_INVALID', 'Eval run storage must be a contained relative path.'); - } - return value; -}; - -type RunStoreValidationCode = 'EVAL_RUN_CORRUPT' | 'EVAL_RUN_RECORD_INVALID'; -type JsonRecord = Readonly>; - -const validationError = (code: RunStoreValidationCode, message: string): never => { - throw storeError(code, message); -}; - -const strictJson = (value: unknown, code: RunStoreValidationCode, label: string): JsonValue => { - try { - return snapshotStrictJsonValue(value); - } catch { - return validationError(code, `${label} must contain only detached strict JSON data.`); - } -}; - -const strictRecord = (value: unknown, code: RunStoreValidationCode, label: string): JsonRecord => { - const snapshot = strictJson(value, code, label); - if (!isRecord(snapshot)) { - return validationError(code, `${label} must be a JSON object.`); - } - return snapshot as JsonRecord; -}; - -const requireKeys = ( - value: JsonRecord, - keys: readonly string[], - code: RunStoreValidationCode, - label: string, -): void => { - const allowed = new Set(keys); - const actual = Object.keys(value); - if (actual.length !== allowed.size || actual.some((key) => !allowed.has(key))) { - validationError(code, `${label} has an invalid schema.`); - } -}; - -const requireOptionalKeys = ( - value: JsonRecord, - required: readonly string[], - optional: readonly string[], - code: RunStoreValidationCode, - label: string, -): void => { - const allowed = new Set([...required, ...optional]); - if (required.some((key) => !Object.hasOwn(value, key)) || Object.keys(value).some((key) => !allowed.has(key))) { - validationError(code, `${label} has an invalid schema.`); - } -}; - -const property = (value: JsonRecord, key: string, code: RunStoreValidationCode, label: string): JsonValue => { - if (!Object.hasOwn(value, key)) { - return validationError(code, `${label} is missing ${JSON.stringify(key)}.`); - } - return value[key]!; -}; - -const requireString = (value: JsonValue, code: RunStoreValidationCode, label: string): string => { - if (typeof value !== 'string' || value.length === 0) { - return validationError(code, `${label} must be a non-empty string.`); - } - return value; -}; - -const requireBoundedText = (value: JsonValue, code: RunStoreValidationCode, label: string): string => { - const text = requireString(value, code, label); - if (text.length > maximumProvenanceTextLength) { - return validationError(code, `${label} must be at most ${maximumProvenanceTextLength} characters.`); - } - return text; -}; - -/** Durable comparison values are labels, never paths, commands, or credential material. */ -const requireProvenanceIdentifier = (value: JsonValue, code: RunStoreValidationCode, label: string): string => { - const text = requireBoundedText(value, code, label); - if (!provenanceIdentifierPattern.test(text) || findCredentialConfiguration(text) !== undefined) { - return validationError(code, 'Eval trial provenance contains an unsafe identifier.'); - } - return text; -}; - -const requireTimestamp = (value: JsonValue, code: RunStoreValidationCode, label: string): string => { - const timestamp = requireString(value, code, label); - if (!Number.isFinite(Date.parse(timestamp))) { - return validationError(code, `${label} must be a valid timestamp.`); - } - return timestamp; -}; - -const requireInteger = (value: JsonValue, code: RunStoreValidationCode, label: string, minimum = 0): number => { - if (typeof value !== 'number' || !Number.isSafeInteger(value) || value < minimum) { - return validationError(code, `${label} must be a safe integer no smaller than ${minimum}.`); - } - return value; -}; - -const requireBoolean = (value: JsonValue, code: RunStoreValidationCode, label: string): boolean => { - if (typeof value !== 'boolean') { - return validationError(code, `${label} must be a boolean.`); - } - return value; -}; - -const requireArray = (value: JsonValue, code: RunStoreValidationCode, label: string): readonly JsonValue[] => { - if (!Array.isArray(value)) { - return validationError(code, `${label} must be a JSON array.`); - } - return value; -}; - -const assertionOutcomes = new Set(['fail', 'inconclusive', 'pass']); -const evidenceLevels = new Set(['inferred', 'observed', 'unavailable']); -const assertionKinds = new Set(['exit-code', 'mcp-call', 'no-mcp-call', 'no-skill-activation', 'outcome', 'skill-activation']); -const harnessFailureCodes = new Set(['EVAL_ARTIFACT_UNAVAILABLE', 'EVAL_FIXTURE_UNAVAILABLE', 'EVAL_GRADER_FAILED', 'EVAL_PROCESS_UNAVAILABLE', 'EVAL_TRACE_UNAVAILABLE']); -const harnessFailureStages = new Set(['artifact', 'fixture', 'grader', 'preflight', 'trace']); -const pluginFailureCodes = new Set(['EVAL_PLUGIN_ASSERTION_FAILED', 'EVAL_PLUGIN_PROCESS_FAILED', 'EVAL_PLUGIN_TIMED_OUT']); - -const requireOutcome = (value: JsonValue, code: RunStoreValidationCode, label: string): EvalAssertionOutcome => { - if (typeof value !== 'string' || !assertionOutcomes.has(value as EvalAssertionOutcome)) { - return validationError(code, `${label} must be an eval assertion outcome.`); - } - return value as EvalAssertionOutcome; -}; - -const requireEvidenceLevel = (value: JsonValue, code: RunStoreValidationCode, label: string): 'inferred' | 'observed' | 'unavailable' => { - if (typeof value !== 'string' || !evidenceLevels.has(value)) { - return validationError(code, `${label} must be an evidence level.`); - } - return value as 'inferred' | 'observed' | 'unavailable'; -}; - -/** The store validates the ids it mints, so every minting caller must share this format. */ -export const mintRunId = (createdAt: Date): string => - `${createdAt.toISOString().replace(/[-:.]/gu, '').replace('T', 't').toLowerCase()}-${randomUUID().slice(0, 8)}`; - -const ownerDocumentKeys = ['createdAt', 'nonce', 'pid']; - -export const parseOwner = (value: unknown): EvalRunOwner => { - const owner = strictRecord(value, 'EVAL_RUN_CORRUPT', 'Eval run owner metadata'); - requireKeys(owner, ownerDocumentKeys, 'EVAL_RUN_CORRUPT', 'Eval run owner metadata'); - const pid = owner.pid; - if ( - typeof owner.createdAt !== 'string' || - typeof owner.nonce !== 'string' || - typeof pid !== 'number' || - !Number.isSafeInteger(pid) || - pid <= 0 - ) { - return validationError('EVAL_RUN_CORRUPT', 'Eval run owner metadata has an invalid schema.'); - } - return Object.freeze({ createdAt: owner.createdAt, nonce: owner.nonce, pid }); -}; - -const parseArtifact = (value: unknown, code: RunStoreValidationCode): EvalArtifactBinding => { - const record = strictRecord(value, code, 'Eval run artifact'); - requireKeys(record, ['manifestPath', 'source', 'targetDigests'], code, 'Eval run artifact'); - const manifestPath = requireSafeRelativePath(requireString(property(record, 'manifestPath', code, 'Eval run artifact'), code, 'Eval run artifact manifest path'), 'Eval run artifact manifest path'); - const source = property(record, 'source', code, 'Eval run artifact'); - if (source !== 'explicit' && source !== 'run-owned') { - return validationError(code, 'Eval run artifact source must be "explicit" or "run-owned".'); - } - const targetDigests = strictRecord(property(record, 'targetDigests', code, 'Eval run artifact'), code, 'Eval run artifact target digests'); - const targets = Object.entries(targetDigests).sort(([left], [right]) => left.localeCompare(right)); - if (targets.length === 0) { - return validationError(code, 'Eval run artifact must record at least one target digest.'); - } - const normalizedTargets: [string, string][] = []; - for (const [target, targetDigest] of targets) { - requireSafeSegment(target, 'Eval run artifact target name'); - normalizedTargets.push([target, requireString(targetDigest, code, `Eval run artifact target ${JSON.stringify(target)} digest`)]); - } - return Object.freeze({ - manifestPath, - source, - targetDigests: Object.freeze(Object.fromEntries(normalizedTargets)), - }); -}; - -const parseProvenance = (value: unknown, code: RunStoreValidationCode): EvalRunProvenance => { - const record = strictRecord(value, code, 'Eval run provenance'); - requireKeys(record, ['agentBundleVersion', 'harness', 'projectRevision'], code, 'Eval run provenance'); - return Object.freeze({ - agentBundleVersion: requireString(property(record, 'agentBundleVersion', code, 'Eval run provenance'), code, 'Eval run agent bundle version'), - harness: requireString(property(record, 'harness', code, 'Eval run provenance'), code, 'Eval run harness'), - projectRevision: requireString(property(record, 'projectRevision', code, 'Eval run provenance'), code, 'Eval run project revision'), - }); -}; - -const optionRecord = ( - value: unknown, - required: readonly string[], - optional: readonly string[], - label: string, -): Readonly> => { - if (typeof value !== 'object' || value === null || Array.isArray(value)) { - return validationError('EVAL_RUN_RECORD_INVALID', `${label} must be a plain object.`); - } - let descriptors: Record; - try { - if (Object.getPrototypeOf(value) !== Object.prototype && Object.getPrototypeOf(value) !== null) { - return validationError('EVAL_RUN_RECORD_INVALID', `${label} must be a plain object.`); - } - descriptors = Object.getOwnPropertyDescriptors(value); - } catch { - return validationError('EVAL_RUN_RECORD_INVALID', `${label} must not be a proxy or inaccessible object.`); - } - const keys = Reflect.ownKeys(descriptors); - const allowed = new Set([...required, ...optional]); - if ( - keys.some((key) => typeof key !== 'string' || !allowed.has(key)) || - required.some((key) => !Object.hasOwn(descriptors, key)) || - keys.some((key) => { - const descriptor = descriptors[key]!; - return !descriptor.enumerable || !('value' in descriptor); - }) - ) { - return validationError('EVAL_RUN_RECORD_INVALID', `${label} must use only enumerable data properties.`); - } - return Object.freeze(Object.fromEntries(keys.map((key) => [key, descriptors[key]!.value]))); -}; - -export interface ParsedCreateEvalRunOptions { - readonly artifact: EvalArtifactBinding; - readonly now?: () => Date; - readonly probeProcess?: (pid: number) => boolean; - readonly projectRoot: string; - readonly provenance: EvalRunProvenance; - readonly runId?: string; - readonly runsDir?: string; -} - -export const parseCreateOptions = (value: unknown): ParsedCreateEvalRunOptions => { - const options = optionRecord(value, ['artifact', 'projectRoot', 'provenance'], ['now', 'probeProcess', 'runId', 'runsDir'], 'Eval run options'); - const now = options.now; - const probeProcess = options.probeProcess; - const runId = options.runId; - const runsDir = options.runsDir; - if (now !== undefined && typeof now !== 'function') { - return validationError('EVAL_RUN_RECORD_INVALID', 'Eval run now must be a function.'); - } - if (probeProcess !== undefined && typeof probeProcess !== 'function') { - return validationError('EVAL_RUN_RECORD_INVALID', 'Eval run probeProcess must be a function.'); - } - if (runId !== undefined && typeof runId !== 'string') { - return validationError('EVAL_RUN_RECORD_INVALID', 'Eval run id must be a string.'); - } - if (runsDir !== undefined && typeof runsDir !== 'string') { - return validationError('EVAL_RUN_RECORD_INVALID', 'Eval run storage must be a string.'); - } - return Object.freeze({ - artifact: parseArtifact(options.artifact, 'EVAL_RUN_RECORD_INVALID'), - ...(now === undefined ? {} : { now: now as () => Date }), - ...(probeProcess === undefined ? {} : { probeProcess: probeProcess as (pid: number) => boolean }), - projectRoot: requireString(options.projectRoot as JsonValue, 'EVAL_RUN_RECORD_INVALID', 'Eval run project root'), - provenance: parseProvenance(options.provenance, 'EVAL_RUN_RECORD_INVALID'), - ...(runId === undefined ? {} : { runId }), - ...(runsDir === undefined ? {} : { runsDir }), - }); -}; - -export const parseListOptions = (value: unknown): ListEvalRunsOptions => { - const options = optionRecord(value, ['projectRoot'], ['runsDir'], 'Eval run list options'); - if (options.runsDir !== undefined && typeof options.runsDir !== 'string') { - return validationError('EVAL_RUN_RECORD_INVALID', 'Eval run storage must be a string.'); - } - return Object.freeze({ - projectRoot: requireString(options.projectRoot as JsonValue, 'EVAL_RUN_RECORD_INVALID', 'Eval run project root'), - ...(options.runsDir === undefined ? {} : { runsDir: options.runsDir }), - }); -}; - -const parseSummary = (value: unknown, code: RunStoreValidationCode): EvalRunSummary => { - const record = strictRecord(value, code, 'Eval run summary'); - requireKeys(record, ['cases', 'fail', 'inconclusive', 'pass', 'trials'], code, 'Eval run summary'); - return Object.freeze({ - cases: requireInteger(property(record, 'cases', code, 'Eval run summary'), code, 'Eval run summary cases'), - fail: requireInteger(property(record, 'fail', code, 'Eval run summary'), code, 'Eval run summary fail'), - inconclusive: requireInteger(property(record, 'inconclusive', code, 'Eval run summary'), code, 'Eval run summary inconclusive'), - pass: requireInteger(property(record, 'pass', code, 'Eval run summary'), code, 'Eval run summary pass'), - trials: requireInteger(property(record, 'trials', code, 'Eval run summary'), code, 'Eval run summary trials'), - }); -}; - -export const parseRunSummaryInput = (value: unknown): EvalRunSummary => - parseSummary(value, 'EVAL_RUN_RECORD_INVALID'); - -const parseRunRecordValue = (value: unknown, code: RunStoreValidationCode): EvalRunRecord => { - const record = strictRecord(value, code, 'Eval run document'); - requireOptionalKeys(record, - ['agentBundleVersion', 'artifact', 'createdAt', 'harness', 'id', 'projectRevision'], - ['completedAt', 'summary'], - code, - 'Eval run document'); - const completedAt = Object.hasOwn(record, 'completedAt') - ? requireTimestamp(property(record, 'completedAt', code, 'Eval run document'), code, 'Eval run completedAt') - : undefined; - const summary = Object.hasOwn(record, 'summary') - ? parseSummary(property(record, 'summary', code, 'Eval run document'), code) - : undefined; - if ((completedAt === undefined) !== (summary === undefined)) { - return validationError(code, 'Eval run document must record completion time and summary together.'); - } - return Object.freeze({ - agentBundleVersion: requireString(property(record, 'agentBundleVersion', code, 'Eval run document'), code, 'Eval run agent bundle version'), - artifact: parseArtifact(property(record, 'artifact', code, 'Eval run document'), code), - ...(completedAt === undefined ? {} : { completedAt }), - createdAt: requireTimestamp(property(record, 'createdAt', code, 'Eval run document'), code, 'Eval run createdAt'), - harness: requireString(property(record, 'harness', code, 'Eval run document'), code, 'Eval run harness'), - id: requireSafeSegment(requireString(property(record, 'id', code, 'Eval run document'), code, 'Eval run id'), 'Eval run id'), - projectRevision: requireString(property(record, 'projectRevision', code, 'Eval run document'), code, 'Eval run project revision'), - ...(summary === undefined ? {} : { summary }), - }); -}; - -export const parseRunRecord = (value: unknown): EvalRunRecord | undefined => { - try { - return parseRunRecordValue(value, 'EVAL_RUN_CORRUPT'); - } catch { - return undefined; - } -}; - -const parseEventRecordValue = (value: unknown, code: RunStoreValidationCode): EvalRunEvent => { - const record = strictRecord(value, code, 'Eval run event'); - requireKeys(record, ['kind', 'payload', 'sequence', 'timestamp'], code, 'Eval run event'); - return Object.freeze({ - kind: requireString(property(record, 'kind', code, 'Eval run event'), code, 'Eval run event kind'), - payload: property(record, 'payload', code, 'Eval run event'), - sequence: requireInteger(property(record, 'sequence', code, 'Eval run event'), code, 'Eval run event sequence', 1), - timestamp: requireTimestamp(property(record, 'timestamp', code, 'Eval run event'), code, 'Eval run event timestamp'), - }); -}; - -export const parseEventInput = (value: unknown): EvalRunEventInput & Pick => { - const record = strictRecord(value, 'EVAL_RUN_RECORD_INVALID', 'Eval run event input'); - requireKeys(record, ['kind', 'payload'], 'EVAL_RUN_RECORD_INVALID', 'Eval run event input'); - return Object.freeze({ - kind: requireString(property(record, 'kind', 'EVAL_RUN_RECORD_INVALID', 'Eval run event input'), 'EVAL_RUN_RECORD_INVALID', 'Eval run event kind'), - payload: property(record, 'payload', 'EVAL_RUN_RECORD_INVALID', 'Eval run event input'), - }); -}; - -export const parseEventRecord = (value: unknown): EvalRunEvent | undefined => { - try { - return parseEventRecordValue(value, 'EVAL_RUN_CORRUPT'); - } catch { - return undefined; - } -}; - -const parseAssertion = (value: JsonValue, code: RunStoreValidationCode): EvalAssertionResult => { - const record = strictRecord(value, code, 'Eval trial assertion'); - requireKeys(record, ['assertionId', 'detail', 'evidence', 'kind', 'outcome'], code, 'Eval trial assertion'); - const kind = requireString(property(record, 'kind', code, 'Eval trial assertion'), code, 'Eval trial assertion kind'); - if (!assertionKinds.has(kind)) { - return validationError(code, 'Eval trial assertion kind is invalid.'); - } - return Object.freeze({ - assertionId: requireString(property(record, 'assertionId', code, 'Eval trial assertion'), code, 'Eval trial assertion id'), - detail: requireString(property(record, 'detail', code, 'Eval trial assertion'), code, 'Eval trial assertion detail'), - evidence: requireEvidenceLevel(property(record, 'evidence', code, 'Eval trial assertion'), code, 'Eval trial assertion evidence'), - kind: kind as EvalAssertionResult['kind'], - outcome: requireOutcome(property(record, 'outcome', code, 'Eval trial assertion'), code, 'Eval trial assertion outcome'), - }); -}; - -const parseEvidence = (value: JsonValue, code: RunStoreValidationCode): EvalTrialEvidence => { - const evidence = strictRecord(value, code, 'Eval trial evidence'); - requireKeys(evidence, ['mcp', 'process', 'scripts', 'skillActivation'], code, 'Eval trial evidence'); - const mcp = strictRecord(property(evidence, 'mcp', code, 'Eval trial evidence'), code, 'Eval trial MCP evidence'); - requireKeys(mcp, ['calls', 'level'], code, 'Eval trial MCP evidence'); - const calls = requireArray(property(mcp, 'calls', code, 'Eval trial MCP evidence'), code, 'Eval trial MCP calls').map((call) => { - const record = strictRecord(call, code, 'Eval trial MCP call'); - requireKeys(record, ['server', 'tool'], code, 'Eval trial MCP call'); - return Object.freeze({ - server: requireString(property(record, 'server', code, 'Eval trial MCP call'), code, 'Eval trial MCP server'), - tool: requireString(property(record, 'tool', code, 'Eval trial MCP call'), code, 'Eval trial MCP tool'), - }); - }); - const process = strictRecord(property(evidence, 'process', code, 'Eval trial evidence'), code, 'Eval trial process evidence'); - requireOptionalKeys(process, ['level', 'timedOut'], ['exitCode'], code, 'Eval trial process evidence'); - const scripts = strictRecord(property(evidence, 'scripts', code, 'Eval trial evidence'), code, 'Eval trial script evidence'); - requireKeys(scripts, ['level', 'results'], code, 'Eval trial script evidence'); - const scriptResults = strictRecord(property(scripts, 'results', code, 'Eval trial script evidence'), code, 'Eval trial script results'); - const results = Object.freeze(Object.fromEntries(Object.entries(scriptResults).map(([name, result]) => { - const record = strictRecord(result, code, `Eval trial script result ${JSON.stringify(name)}`); - requireKeys(record, ['detail', 'outcome'], code, `Eval trial script result ${JSON.stringify(name)}`); - return [name, Object.freeze({ - detail: requireString(property(record, 'detail', code, `Eval trial script result ${JSON.stringify(name)}`), code, `Eval trial script result ${JSON.stringify(name)} detail`), - outcome: requireOutcome(property(record, 'outcome', code, `Eval trial script result ${JSON.stringify(name)}`), code, `Eval trial script result ${JSON.stringify(name)} outcome`), - })]; - }))); - const skillActivation = strictRecord(property(evidence, 'skillActivation', code, 'Eval trial evidence'), code, 'Eval trial skill evidence'); - requireKeys(skillActivation, ['activated', 'level'], code, 'Eval trial skill evidence'); - const activated = requireArray(property(skillActivation, 'activated', code, 'Eval trial skill evidence'), code, 'Eval trial activated skills') - .map((skill) => requireString(skill, code, 'Eval trial activated skill')); - return Object.freeze({ - mcp: Object.freeze({ calls: Object.freeze(calls), level: requireEvidenceLevel(property(mcp, 'level', code, 'Eval trial MCP evidence'), code, 'Eval trial MCP evidence level') }), - process: Object.freeze({ - ...(Object.hasOwn(process, 'exitCode') ? { exitCode: requireInteger(property(process, 'exitCode', code, 'Eval trial process evidence'), code, 'Eval trial process exit code') } : {}), - level: requireEvidenceLevel(property(process, 'level', code, 'Eval trial process evidence'), code, 'Eval trial process evidence level'), - timedOut: requireBoolean(property(process, 'timedOut', code, 'Eval trial process evidence'), code, 'Eval trial process timedOut'), - }), - scripts: Object.freeze({ level: requireEvidenceLevel(property(scripts, 'level', code, 'Eval trial script evidence'), code, 'Eval trial script evidence level'), results }), - skillActivation: Object.freeze({ activated: Object.freeze(activated), level: requireEvidenceLevel(property(skillActivation, 'level', code, 'Eval trial skill evidence'), code, 'Eval trial skill evidence level') }), - }); -}; - -const parseHarnessFailure = (value: JsonValue, code: RunStoreValidationCode): EvalHarnessFailure => { - const record = strictRecord(value, code, 'Eval trial harness failure'); - requireKeys(record, ['code', 'message', 'stage'], code, 'Eval trial harness failure'); - const failureCode = requireString(property(record, 'code', code, 'Eval trial harness failure'), code, 'Eval trial harness failure code'); - const stage = requireString(property(record, 'stage', code, 'Eval trial harness failure'), code, 'Eval trial harness failure stage'); - if (!harnessFailureCodes.has(failureCode) || !harnessFailureStages.has(stage)) { - return validationError(code, 'Eval trial harness failure is invalid.'); - } - return Object.freeze({ - code: failureCode as EvalHarnessFailure['code'], - message: requireString(property(record, 'message', code, 'Eval trial harness failure'), code, 'Eval trial harness failure message'), - stage: stage as EvalHarnessFailure['stage'], - }); -}; - -const parsePluginFailure = (value: JsonValue, code: RunStoreValidationCode): EvalPluginFailure => { - const record = strictRecord(value, code, 'Eval trial plugin failure'); - requireKeys(record, ['code', 'message'], code, 'Eval trial plugin failure'); - const failureCode = requireString(property(record, 'code', code, 'Eval trial plugin failure'), code, 'Eval trial plugin failure code'); - if (!pluginFailureCodes.has(failureCode)) { - return validationError(code, 'Eval trial plugin failure is invalid.'); - } - return Object.freeze({ - code: failureCode as EvalPluginFailure['code'], - message: requireString(property(record, 'message', code, 'Eval trial plugin failure'), code, 'Eval trial plugin failure message'), - }); -}; - -const parseInvocationProvenance = ( - value: JsonValue, - code: RunStoreValidationCode, -): EvalTrialInvocationProvenance => { - const record = strictRecord(value, code, 'Eval trial invocation provenance'); - const mode = property(record, 'mode', code, 'Eval trial invocation provenance'); - if (mode === 'automatic') { - requireOptionalKeys(record, ['mode'], ['skill'], code, 'Eval trial invocation provenance'); - return Object.freeze({ - mode, - ...(Object.hasOwn(record, 'skill') - ? { skill: requireProvenanceIdentifier(property(record, 'skill', code, 'Eval trial invocation provenance'), code, 'Eval trial invocation Skill') } - : {}), - }); - } - if (mode === 'none') { - requireKeys(record, ['mode'], code, 'Eval trial invocation provenance'); - return Object.freeze({ mode }); - } - if (mode === 'explicit') { - requireKeys(record, ['mode', 'skill'], code, 'Eval trial invocation provenance'); - return Object.freeze({ - mode, - skill: requireProvenanceIdentifier(property(record, 'skill', code, 'Eval trial invocation provenance'), code, 'Eval trial invocation Skill'), - }); - } - return validationError(code, 'Eval trial invocation provenance mode is invalid.'); -}; - -const parseSemanticGraderProvenance = ( - value: JsonValue, - code: RunStoreValidationCode, -): Exclude => { - const record = strictRecord(value, code, 'Eval trial semantic grader provenance'); - if (Object.hasOwn(record, 'state')) { - requireKeys(record, ['state'], code, 'Eval trial semantic grader provenance'); - if (record.state !== 'unrecorded') { - return validationError(code, 'Eval trial semantic grader provenance state is invalid.'); - } - return Object.freeze({ state: 'unrecorded' }); - } - requireKeys(record, ['contractRevision', 'id', 'model'], code, 'Eval trial semantic grader provenance'); - return Object.freeze({ - contractRevision: requireProvenanceIdentifier(property(record, 'contractRevision', code, 'Eval trial semantic grader provenance'), code, 'Eval semantic grader contract revision'), - id: requireProvenanceIdentifier(property(record, 'id', code, 'Eval trial semantic grader provenance'), code, 'Eval semantic grader id'), - model: requireProvenanceIdentifier(property(record, 'model', code, 'Eval trial semantic grader provenance'), code, 'Eval semantic grader model'), - }); -}; - -const parseTrialProvenance = (value: JsonValue, code: RunStoreValidationCode): EvalTrialProvenance => { - const record = strictRecord(value, code, 'Eval trial provenance'); - requireOptionalKeys(record, ['invocation', 'semanticGrader'], ['hostCliVersion'], code, 'Eval trial provenance'); - const semanticGrader = property(record, 'semanticGrader', code, 'Eval trial provenance'); - return Object.freeze({ - ...(Object.hasOwn(record, 'hostCliVersion') - ? { hostCliVersion: requireProvenanceIdentifier(property(record, 'hostCliVersion', code, 'Eval trial provenance'), code, 'Eval host CLI version') } - : {}), - invocation: parseInvocationProvenance(property(record, 'invocation', code, 'Eval trial provenance'), code), - semanticGrader: semanticGrader === null ? null : parseSemanticGraderProvenance(semanticGrader, code), - }); -}; - -const parseTrialUsage = (value: JsonValue, code: RunStoreValidationCode): EvalTrialUsage => { - const record = strictRecord(value, code, 'Eval trial usage'); - requireKeys(record, ['inputTokens', 'outputTokens'], code, 'Eval trial usage'); - return Object.freeze({ - inputTokens: requireInteger(property(record, 'inputTokens', code, 'Eval trial usage'), code, 'Eval input tokens'), - outputTokens: requireInteger(property(record, 'outputTokens', code, 'Eval trial usage'), code, 'Eval output tokens'), - }); -}; - -const trialInputKeys = ['assertions', 'caseDigest', 'caseId', 'completedAt', 'durationMs', 'evidence', 'fixtureDigest', 'host', 'id', 'model', 'outcome', 'prompt', 'provenance', 'rawArtifacts', 'startedAt', 'targetDigest', 'trialIndex']; - -const parseTrialRecordValue = (value: unknown, code: RunStoreValidationCode): EvalTrialRecord => { - const record = strictRecord(value, code, 'Eval trial record'); - requireOptionalKeys(record, - trialInputKeys, - ['harnessFailure', 'pluginFailure', 'usage'], - code, - 'Eval trial record'); - const harnessFailure = Object.hasOwn(record, 'harnessFailure') - ? parseHarnessFailure(property(record, 'harnessFailure', code, 'Eval trial record'), code) - : undefined; - const pluginFailure = Object.hasOwn(record, 'pluginFailure') - ? parsePluginFailure(property(record, 'pluginFailure', code, 'Eval trial record'), code) - : undefined; - const provenance = parseTrialProvenance(property(record, 'provenance', code, 'Eval trial record'), code); - const usage = Object.hasOwn(record, 'usage') - ? parseTrialUsage(property(record, 'usage', code, 'Eval trial record'), code) - : undefined; - if (harnessFailure !== undefined && pluginFailure !== undefined) { - return validationError(code, 'A trial records either a harness failure or a plugin failure, never both.'); - } - return Object.freeze({ - assertions: Object.freeze(requireArray(property(record, 'assertions', code, 'Eval trial record'), code, 'Eval trial assertions').map((assertion) => parseAssertion(assertion, code))), - caseDigest: requireString(property(record, 'caseDigest', code, 'Eval trial record'), code, 'Eval trial case digest'), - caseId: requireSafeSegment(requireString(property(record, 'caseId', code, 'Eval trial record'), code, 'Eval trial case id'), 'Eval trial caseId'), - completedAt: requireTimestamp(property(record, 'completedAt', code, 'Eval trial record'), code, 'Eval trial completedAt'), - durationMs: requireInteger(property(record, 'durationMs', code, 'Eval trial record'), code, 'Eval trial duration'), - evidence: parseEvidence(property(record, 'evidence', code, 'Eval trial record'), code), - fixtureDigest: requireString(property(record, 'fixtureDigest', code, 'Eval trial record'), code, 'Eval trial fixture digest'), - ...(harnessFailure === undefined ? {} : { harnessFailure }), - host: requireString(property(record, 'host', code, 'Eval trial record'), code, 'Eval trial host'), - id: requireSafeSegment(requireString(property(record, 'id', code, 'Eval trial record'), code, 'Eval trial id'), 'Eval trial id'), - model: requireString(property(record, 'model', code, 'Eval trial record'), code, 'Eval trial model'), - outcome: requireOutcome(property(record, 'outcome', code, 'Eval trial record'), code, 'Eval trial outcome'), - ...(pluginFailure === undefined ? {} : { pluginFailure }), - prompt: requireString(property(record, 'prompt', code, 'Eval trial record'), code, 'Eval trial prompt'), - provenance, - rawArtifacts: Object.freeze(requireArray(property(record, 'rawArtifacts', code, 'Eval trial record'), code, 'Eval trial raw artifacts') - .map((rawArtifact) => requireSafeRelativePath(requireString(rawArtifact, code, 'Eval trial raw artifact'), 'Eval trial raw artifact'))), - startedAt: requireTimestamp(property(record, 'startedAt', code, 'Eval trial record'), code, 'Eval trial startedAt'), - targetDigest: requireString(property(record, 'targetDigest', code, 'Eval trial record'), code, 'Eval trial target digest'), - trialIndex: requireInteger(property(record, 'trialIndex', code, 'Eval trial record'), code, 'Eval trial index'), - ...(usage === undefined ? {} : { usage }), - }); -}; - -export const parseTrialInput = (value: unknown): EvalTrialRecord => parseTrialRecordValue(value, 'EVAL_RUN_RECORD_INVALID'); - -export const parseTrialRecord = (value: unknown, sourcePath: string): EvalTrialRecord => { - try { - return parseTrialRecordValue(value, 'EVAL_RUN_CORRUPT'); - } catch { - throw storeError('EVAL_RUN_CORRUPT', `Eval trial record ${JSON.stringify(sourcePath)} does not match the trial schema.`); - } -}; diff --git a/packages/agent-bundle/src/eval/run-store-types.ts b/packages/agent-bundle/src/eval/run-store-types.ts deleted file mode 100644 index e96fc845c..000000000 --- a/packages/agent-bundle/src/eval/run-store-types.ts +++ /dev/null @@ -1,145 +0,0 @@ -import type { JsonValue } from '../core/strict-json.ts'; -import type { - EvalAssertionOutcome, - EvalAssertionResult, - EvalHarnessFailure, - EvalPluginFailure, - EvalTrialEvidence, -} from './types.ts'; - -export interface EvalArtifactBinding { - readonly manifestPath: string; - readonly source: 'explicit' | 'run-owned'; - readonly targetDigests: Readonly>; -} - -export interface EvalRunProvenance { - readonly agentBundleVersion: string; - readonly harness: string; - readonly projectRevision: string; -} - -export interface EvalRunSummary { - readonly cases: number; - readonly fail: number; - readonly inconclusive: number; - readonly pass: number; - readonly trials: number; -} - -export interface EvalRunRecord { - readonly agentBundleVersion: string; - readonly artifact: EvalArtifactBinding; - readonly completedAt?: string; - readonly createdAt: string; - readonly harness: string; - readonly id: string; - readonly projectRevision: string; - readonly summary?: EvalRunSummary; -} - -/** A safe, authored invocation identity; it deliberately never carries a command or path. */ -export interface EvalTrialInvocationProvenance { - readonly mode: 'automatic' | 'explicit' | 'none'; - readonly skill?: string; -} - -/** The server-owned semantic grader identity used to produce a semantic outcome. */ -export interface EvalSemanticGraderProvenance { - readonly contractRevision: string; - readonly id: string; - readonly model: string; -} - -/** A semantic grader ran but its identity was not safe or complete enough to compare. */ -export interface EvalUnrecordedSemanticGraderProvenance { - readonly state: 'unrecorded'; -} - -export type EvalTrialSemanticGraderProvenance = - | EvalSemanticGraderProvenance - | EvalUnrecordedSemanticGraderProvenance - | null; - -/** Alignment data observed by a trial, limited to safe model, version, and authored identity values. */ -export interface EvalTrialProvenance { - readonly hostCliVersion?: string; - readonly invocation: EvalTrialInvocationProvenance; - readonly semanticGrader: EvalTrialSemanticGraderProvenance; -} - -/** Normalized token counts reported by a host; absent means that host reported no usable usage. */ -export interface EvalTrialUsage { - readonly inputTokens: number; - readonly outputTokens: number; -} - -export interface EvalTrialRecord { - readonly assertions: readonly EvalAssertionResult[]; - readonly caseDigest: string; - readonly caseId: string; - readonly completedAt: string; - readonly durationMs: number; - readonly evidence: EvalTrialEvidence; - readonly fixtureDigest: string; - readonly harnessFailure?: EvalHarnessFailure; - readonly host: string; - readonly id: string; - readonly model: string; - readonly outcome: EvalAssertionOutcome; - readonly pluginFailure?: EvalPluginFailure; - readonly prompt: string; - readonly provenance: EvalTrialProvenance; - readonly rawArtifacts: readonly string[]; - readonly startedAt: string; - readonly targetDigest: string; - readonly trialIndex: number; - /** Omitted when the host did not emit normalized token counts. */ - readonly usage?: EvalTrialUsage; -} - -export type EvalTrialRecordInput = EvalTrialRecord; - -/** The harness only needs durable artifact and normalized trial writes. */ -export interface EvalTrialWriter { - writeArtifactFile(relativePath: string, contents: string): Promise; - writeTrial(trial: EvalTrialRecordInput): Promise; -} - -export interface EvalRunEventInput { - readonly kind: string; - readonly payload: unknown; -} - -export interface EvalRunEvent { - readonly kind: string; - readonly payload: JsonValue; - readonly sequence: number; - readonly timestamp: string; -} - -export interface EvalRunEventsRead { - readonly events: readonly EvalRunEvent[]; - readonly incompleteTrailingRecord?: string; -} - -export interface EvalRunOwner { - readonly createdAt: string; - readonly nonce: string; - readonly pid: number; -} - -export interface CreateEvalRunOptions { - readonly artifact: EvalArtifactBinding; - readonly now?: () => Date; - readonly probeProcess?: (pid: number) => boolean; - readonly projectRoot: string; - readonly provenance: EvalRunProvenance; - readonly runId?: string; - readonly runsDir?: string; -} - -export interface ListEvalRunsOptions { - readonly projectRoot: string; - readonly runsDir?: string; -} diff --git a/packages/agent-bundle/src/routes/graph.ts b/packages/agent-bundle/src/routes/graph.ts index 1d7f114b9..4b1635274 100644 --- a/packages/agent-bundle/src/routes/graph.ts +++ b/packages/agent-bundle/src/routes/graph.ts @@ -1,9 +1,9 @@ -import { existsSync, statSync } from 'node:fs'; import { readFile } from 'node:fs/promises'; import { extname, relative, resolve } from 'node:path'; import fastGlob from 'fast-glob'; +import { conventionalEntryAt } from '../config/conventional-entry.ts'; import { isProjectPathIgnored, readProjectIgnoreRules, toPosixPath } from '../config/ignore.ts'; import { resolveAppRouteTemplate } from './app-template.ts'; import { @@ -97,27 +97,6 @@ const isNonGeneratedServerOverride = (override: CompiledServerMode | undefined): } }; -const conventionalEntryExtensions = ['.ts', '.tsx'] as const; - -/** - * Local copy of the conventional-entry probe from config/normalize.ts. - * Importing it would close the cycle discover.ts -> routes/graph.ts -> - * normalize.ts -> discover.ts, so the probe is duplicated here with the - * same .ts/.tsx rule. - */ -const conventionalEntryAt = (root: string, ...segments: string[]): string | undefined => { - const stem = resolve(root, ...segments); - for (const extension of conventionalEntryExtensions) { - const candidate = `${stem}${extension}`; - try { - if (existsSync(candidate) && statSync(candidate).isFile()) return candidate; - } catch { - // A racing deletion means the convention does not apply. - } - } - return undefined; -}; - const routeError = (code: string, message: string, recovery: string, sourcePath?: string): Diagnostic => ({ code, message, diff --git a/packages/rsc-runtime/src/state/contract.ts b/packages/rsc-runtime/src/state/contract.ts index 5603c969c..5859c64e6 100644 --- a/packages/rsc-runtime/src/state/contract.ts +++ b/packages/rsc-runtime/src/state/contract.ts @@ -458,13 +458,6 @@ export const expectIdempotencyKey = (key: string): string => { return key; }; -export const expectCanonicalPayload = (value: unknown, label: string): string => { - if (!isJsonSafe(value)) { - throw new AgentStateError('invalid-event', `${label} must be JSON-safe`); - } - return canonicalJson(value); -}; - export const expectRevisionShape = (revision: number | undefined, label: string): void => { if (revision !== undefined && (!Number.isInteger(revision) || revision < 0)) { throw new AgentStateError('invalid-input', `${label} must be an integer >= 0`); diff --git a/packages/workbench/src/discovery/discovery-model.ts b/packages/workbench/src/discovery/discovery-model.ts index b7236aa2d..ee3ded01f 100644 --- a/packages/workbench/src/discovery/discovery-model.ts +++ b/packages/workbench/src/discovery/discovery-model.ts @@ -181,7 +181,7 @@ const endpointPresentationFor = (report: DiscoveryEndpointReport): DiscoveryPres } }; -const hostLabelFor = (host: DiscoveryHost): string => { +export const hostLabelFor = (host: DiscoveryHost): string => { switch (host) { case 'claude': return 'Claude'; diff --git a/packages/workbench/src/discovery/discovery-page.tsx b/packages/workbench/src/discovery/discovery-page.tsx index 39dce44ee..42ba2de1f 100644 --- a/packages/workbench/src/discovery/discovery-page.tsx +++ b/packages/workbench/src/discovery/discovery-page.tsx @@ -30,6 +30,7 @@ import { type DiscoveryFindingView, type DiscoveryHostView, type DiscoveryPresentation, + hostLabelFor, } from './discovery-model.ts'; import './discovery-page.css'; @@ -42,21 +43,6 @@ export interface DiscoveryPageProps { const valueOrDash = (value: string | undefined): string => value ?? '—'; -const hostLabelFor = (host: DiscoveryHost): string => { - switch (host) { - case 'claude': - return 'Claude'; - case 'codex': - return 'Codex'; - case 'cursor': - return 'Cursor'; - default: { - const exhaustive: never = host; - return exhaustive; - } - } -}; - const errorDetails = (reason: unknown): Readonly<{ readonly code: string; readonly message: string }> => { if (reason instanceof Error) { const code = 'code' in reason && typeof reason.code === 'string' ? reason.code : 'AB8234'; diff --git a/packages/workbench/src/logs/log-client.ts b/packages/workbench/src/logs/log-client.ts index c5af0aa1f..65f75546f 100644 --- a/packages/workbench/src/logs/log-client.ts +++ b/packages/workbench/src/logs/log-client.ts @@ -1,8 +1,13 @@ -import type { - DevLogMessage, - DevLogRecord, - DevLogReplay, - DevLogReplayGap, +import { + devLogKinds, + devLogLevels, + devLogProducers, + hasControlOrSeparators, + safeContextKeys, + type DevLogMessage, + type DevLogRecord, + type DevLogReplay, + type DevLogReplayGap, } from '../../../agent-bundle/src/contracts/dev-logs.ts'; import { parseJsonWithoutDuplicateKeys, @@ -47,32 +52,10 @@ export class LogClientError extends Error { const maximumLogFrameBytes = 64 * 1024; const maximumSummaryLength = 2_048; -const safeContextKeys = new Set(['buildId', 'diagnosticCode', 'epochId', 'hookId', 'projectId', 'runId', 'sessionId', 'target']); -const devLogProducers = Object.freeze(['project', 'build', 'diagnostic', 'mcp', 'hook', 'eval', 'playground'] as const); -const devLogLevels = Object.freeze(['debug', 'info', 'warning', 'error'] as const); -const devLogKinds = deepFreeze({ - build: ['artifact.available', 'build.failed', 'build.started'], - diagnostic: [ - 'artifact.available.diagnostic', 'artifact.status.diagnostic', 'build.failed.diagnostic', 'build.started.diagnostic', - 'dev.contract.status.diagnostic', 'dev.host.sync.diagnostic', 'invalidation.diagnostic', 'runtime.event.diagnostic', - 'source.changed.diagnostic', 'source.status.diagnostic', - ], - eval: ['eval.run.completed', 'eval.run.failed', 'eval.run.started'], - hook: ['hook.simulate.completed', 'hook.simulate.failed', 'hook.simulate.started'], - mcp: ['mcp.logging', 'mcp.stderr', 'mcp.operation.failed', 'mcp.operation.started', 'mcp.operation.succeeded'], - playground: ['playground.event.appended'], - project: [ - 'artifact.status', 'dev.contract.status', 'dev.host.sync', 'dev.shutdown.completed', 'dev.shutdown.started', 'invalidation', - 'project.events.replay-gap', 'project.invalid-source', 'project.load', 'project.prepared', 'runtime.event', - 'source.changed', 'source.status', - ], -}); const safeIdentifier = /^[A-Za-z0-9][A-Za-z0-9._:@+-]{0,255}$/u; const safeInteger = (value: unknown, minimum = 0): value is number => typeof value === 'number' && Number.isSafeInteger(value) && value >= minimum; const isDate = (value: unknown): value is string => typeof value === 'string' && !Number.isNaN(Date.parse(value)) && new Date(value).toISOString() === value; -const hasControlOrSeparators = (value: string): boolean => [...value].some((character) => - character === '/' || character === '\\' || character <= '\u001F' || character === '\u007F'); const safeProjectRelativePath = /(?:\/[A-Za-z0-9._@+-]+)*/gu; const isSafeWireText = (value: unknown, maximum = maximumLogFrameBytes): value is string => { if (typeof value !== 'string' || value.length === 0 || value.length > maximum || redactEvalCredentialText(value) !== value) return false; diff --git a/packages/workbench/src/main.tsx b/packages/workbench/src/main.tsx index e65178f09..77599a9d0 100644 --- a/packages/workbench/src/main.tsx +++ b/packages/workbench/src/main.tsx @@ -63,7 +63,7 @@ import { import { RoutesPage } from './routes/routes-page.tsx'; import { overviewFor } from './overview-model.ts'; import { downloadBlob, errorMessage as messageFrom } from './client-helpers.ts'; -import { BundleWorkflow, HostAdoptionSection } from './overview-page.tsx'; +import { BundleWorkflow, HostAdoptionSection, StateMark } from './overview-page.tsx'; import { ProjectClient, type ProjectConnectionState } from './project-client.ts'; import { SkillClient } from './skill-client.ts'; import { SkillsPage } from './skills-page.tsx'; @@ -295,14 +295,6 @@ const RuntimeMcpHandoffButton = ({ authority, host }: { readonly authority: Runt const downloadMcpFile = ({ blob, filename }: McpDownload): void => downloadBlob(blob, filename); -const StateMark = ({ state }: { readonly state: string }) => ( - -); - type WorkbenchPage = GeneralWorkbenchPage | 'runtime'; type RuntimeCapability = 'available' | 'unavailable' | 'unknown'; type CapabilityState = diff --git a/packages/workbench/src/mcp/mcp-session-trace-client.ts b/packages/workbench/src/mcp/mcp-session-trace-client.ts deleted file mode 100644 index dd1983387..000000000 --- a/packages/workbench/src/mcp/mcp-session-trace-client.ts +++ /dev/null @@ -1,205 +0,0 @@ -import type { - McpSessionOperation, - McpSessionTraceEntry, - McpSessionTraceReplayGap, -} from '../../../agent-bundle/src/contracts/mcp-session.ts'; -import { isRecord, parseStrictResponseJson } from '../client-helpers.ts'; -import { abortableNdjsonStream, readNdjsonResponseFrames, type NdjsonStream } from '../ndjson.ts'; -import type { McpRouteTrace } from './mcp-route-client.ts'; - -export type McpSessionTraceMessage = McpSessionTraceEntry | McpSessionTraceReplayGap; - -/** Live frames buffered while a replay snapshot request for the same generation is in flight. */ -interface TraceRefresh { - readonly generation: number; - readonly live: McpSessionTraceMessage[]; -} - -export interface McpSessionTraceRefresh { - /** Stops buffering if this refresh still owns the buffer; for early-exit and failure paths. */ - end(): void; - /** Publishes the buffered live entries, then stops buffering unconditionally. */ - flush(): void; -} - -export interface McpSessionTraceClientOptions { - createError(message: string): Error; - isCurrent(generation: number): boolean; - lastSequence(): number; - publishEntry(entry: McpSessionTraceMessage): void; - publishFailure(code: string, reason: unknown): void; - stream(sessionId: string, after: number, signal?: AbortSignal): Promise; -} - -const validCursor = (value: unknown): value is number => - typeof value === 'number' && Number.isSafeInteger(value) && value > 0; - -const traceOperations = new Set([ - 'callTool', 'cancel', 'close', 'getPrompt', 'initialize', 'listPrompts', 'listResources', 'listResourceTemplates', 'listTools', 'readResource', 'restart', -]); -const tracePhases = new Set(['started', 'succeeded', 'failed']); -const isTraceOperation = (value: unknown): value is McpSessionOperation => - typeof value === 'string' && traceOperations.has(value); -const isTracePhase = (value: unknown): value is 'failed' | 'started' | 'succeeded' => - typeof value === 'string' && tracePhases.has(value); - -const traceEntry = (value: unknown, invalid: () => Error): McpSessionTraceMessage => { - if (!isRecord(value)) throw invalid(); - if (value.type === 'replay.gap') { - if ( - !validCursor(value.earliestAvailableSequence) || !validCursor(value.latestDroppedSequence) || - typeof value.requestedAfterSequence !== 'number' || !Number.isSafeInteger(value.requestedAfterSequence) || - value.requestedAfterSequence < 0 - ) throw invalid(); - return { - earliestAvailableSequence: value.earliestAvailableSequence, - latestDroppedSequence: value.latestDroppedSequence, - requestedAfterSequence: value.requestedAfterSequence, - type: 'replay.gap', - }; - } - if (!validCursor(value.sequence) || typeof value.occurredAt !== 'number' || !Number.isFinite(value.occurredAt)) throw invalid(); - if (value.kind === 'frame' && (value.direction === 'client' || value.direction === 'server')) { - return { direction: value.direction, kind: 'frame', message: value.message, occurredAt: value.occurredAt, sequence: value.sequence }; - } - if (value.kind === 'stderr' && typeof value.text === 'string') { - return { kind: 'stderr', occurredAt: value.occurredAt, sequence: value.sequence, text: value.text }; - } - if (value.kind === 'logging' || value.kind === 'progress') { - return { kind: value.kind, occurredAt: value.occurredAt, payload: value.payload, sequence: value.sequence }; - } - if (value.kind === 'operation' && isTraceOperation(value.operation) && isTracePhase(value.phase)) return { - kind: 'operation', - occurredAt: value.occurredAt, - operation: value.operation, - phase: value.phase, - sequence: value.sequence, - }; - throw invalid(); -}; - -const traceOverflow = (value: unknown, invalid: () => Error): McpSessionTraceReplayGap | undefined => { - if (value === undefined) return undefined; - if ( - !isRecord(value) || - typeof value.afterSequence !== 'number' || !Number.isSafeInteger(value.afterSequence) || - typeof value.droppedThroughSequence !== 'number' || !Number.isSafeInteger(value.droppedThroughSequence) - ) { - throw invalid(); - } - if (value.afterSequence < 0 || value.droppedThroughSequence < value.afterSequence) throw invalid(); - return { - earliestAvailableSequence: value.droppedThroughSequence + 1, - latestDroppedSequence: value.droppedThroughSequence, - requestedAfterSequence: value.afterSequence, - type: 'replay.gap', - }; -}; - -const isReplayGap = (entry: McpSessionTraceMessage): entry is McpSessionTraceReplayGap => - 'type' in entry && entry.type === 'replay.gap'; - -const traceCursor = (entry: McpSessionTraceMessage): number => - isReplayGap(entry) ? entry.latestDroppedSequence : entry.sequence; - -/** Owns the NDJSON trace subscription, replay-gap validation, and cursor-ordered publishing for one controller. */ -export class McpSessionTraceClient { - readonly #options: McpSessionTraceClientOptions; - #refresh: TraceRefresh | undefined; - #stream: NdjsonStream | undefined; - #task: Promise | undefined; - - constructor(options: McpSessionTraceClientOptions) { - this.#options = options; - } - - get task(): Promise | undefined { - return this.#task; - } - - abort(): void { - this.#stream?.close(); - } - - beginRefresh(generation: number): McpSessionTraceRefresh { - const refresh: TraceRefresh = { generation, live: [] }; - this.#refresh = refresh; - return { - end: () => { - if (this.#refresh === refresh) this.#refresh = undefined; - }, - flush: () => { - this.publish(refresh.live); - this.#refresh = undefined; - }, - }; - } - - publish(entries: readonly McpSessionTraceMessage[]): void { - const ordered = entries.length === 1 - ? entries - : [...entries].sort((left, right) => traceCursor(left) - traceCursor(right)); - for (const entry of ordered) { - const cursor = traceCursor(entry); - if (cursor <= this.#options.lastSequence()) continue; - this.#options.publishEntry(entry); - } - } - - replayMessages(trace: McpRouteTrace): readonly McpSessionTraceMessage[] { - const overflow = traceOverflow(trace.overflow, this.#invalid); - return Object.freeze([ - ...(overflow === undefined ? [] : [overflow]), - ...trace.entries.map((entry) => traceEntry(entry, this.#invalid)), - ]); - } - - reset(): void { - this.#stream = undefined; - this.#task = undefined; - } - - subscribe(sessionId: string, generation: number): void { - if (this.#stream !== undefined) return; - const stream = abortableNdjsonStream(undefined, (signal) => this.#subscribe(sessionId, generation, signal)); - this.#stream = stream; - const task = stream.done; - this.#task = task; - void task.finally(() => { - if (this.#task === task) this.#task = undefined; - }); - } - - readonly #invalid = (): Error => this.#options.createError('Foreground MCP trace stream contained an invalid entry.'); - - #receive(entry: McpSessionTraceMessage, generation: number): void { - if (!this.#options.isCurrent(generation)) return; - if (this.#refresh?.generation === generation) { - this.#refresh.live.push(entry); - return; - } - this.publish([entry]); - } - - async #subscribe(sessionId: string, generation: number, signal: AbortSignal): Promise { - try { - const response = await this.#options.stream(sessionId, this.#options.lastSequence(), signal); - const receiveLine = (bytes: Uint8Array): void => { - if (bytes.byteLength === 0) return; - this.#receive(traceEntry(parseStrictResponseJson(bytes, this.#invalid), this.#invalid), generation); - }; - await readNdjsonResponseFrames(response, receiveLine, { - invalidFrameError: this.#invalid, - missingBodyError: () => this.#options.createError('Foreground MCP trace stream did not include a body.'), - signal, - }); - if (!signal.aborted && this.#options.isCurrent(generation)) { - this.#options.publishFailure('mcp.trace.stream.closed', 'Foreground MCP trace stream closed unexpectedly.'); - } - } catch (reason) { - if (!signal.aborted && this.#options.isCurrent(generation)) { - this.#options.publishFailure('mcp.trace.stream.error', reason); - } - } - } -} diff --git a/packages/workbench/src/ndjson.ts b/packages/workbench/src/ndjson.ts index 8c29c81cb..a010bb6fc 100644 --- a/packages/workbench/src/ndjson.ts +++ b/packages/workbench/src/ndjson.ts @@ -103,26 +103,3 @@ export const readNdjsonResponseFrames = async ( } } }; - -export interface NdjsonStream { - close(): void; - readonly done: Promise; -} - -/** - * Owns the per-stream AbortController, forwards an optional caller signal into - * it, and detaches that forwarding once the stream settles. - */ -export const abortableNdjsonStream = ( - signal: AbortSignal | undefined, - run: (signal: AbortSignal) => Promise, -): NdjsonStream => { - const controller = new AbortController(); - const forwardAbort = (): void => controller.abort(); - signal?.addEventListener('abort', forwardAbort, { once: true }); - if (signal?.aborted) controller.abort(); - return Object.freeze({ - close: () => controller.abort(), - done: run(controller.signal).finally(() => signal?.removeEventListener('abort', forwardAbort)), - }); -}; diff --git a/packages/workbench/src/overview-page.tsx b/packages/workbench/src/overview-page.tsx index b032dc6a8..d45687b02 100644 --- a/packages/workbench/src/overview-page.tsx +++ b/packages/workbench/src/overview-page.tsx @@ -4,7 +4,7 @@ import { bundleSummaryFor, type OverviewHostAdoption } from './overview-model.ts import type { WorkbenchCapabilities } from './workbench-capabilities.ts'; import type { WorkbenchPage } from './workbench-screen.tsx'; -const StateMark = ({ state }: { readonly state: string }) => ( +export const StateMark = ({ state }: { readonly state: string }) => (