diff --git a/CHANGELOG.md b/CHANGELOG.md index ec74c2f..78a81ae 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,17 @@ # Changelog +## 4.0.1 + +### Changed + +- Split benchmark, memory experiment, and memory improvement internals into focused modules while preserving every public export and signature. +- Split memory and benchmark tests by behavior, with shared controller and adapter fixtures kept in one test-support module. +- Moved retrieval holdout contracts into the memory type layer to remove the holdout/types import cycle. + +### Fixed + +- Made the Mem0 deletion-convergence test deterministic under file-level parallel execution. + ## 4.0.0 ### Breaking Changes diff --git a/package.json b/package.json index 548577d..c4c2bc0 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@tangle-network/agent-knowledge", - "version": "4.0.0", + "version": "4.0.1", "description": "Source-grounded, eval-gated knowledge growth primitives for agents.", "homepage": "https://github.com/tangle-network/agent-knowledge#readme", "repository": { diff --git a/src/benchmarks/adapters.ts b/src/benchmarks/adapters.ts new file mode 100644 index 0000000..9154351 --- /dev/null +++ b/src/benchmarks/adapters.ts @@ -0,0 +1,194 @@ +import { memoryHitToSourceRecord, memoryWriteResultToSourceRecord } from '../memory/source-record' + +import type { + AgentMemoryAdapter, + AgentMemoryHit, + AgentMemoryScope, + AgentMemoryWriteInput, +} from '../memory/types' + +export function createNoopMemoryBenchmarkAdapter(id = 'no-memory'): AgentMemoryAdapter { + return { + id, + branchIsolation: { mode: 'scoped' }, + async search() { + return [] + }, + async getContext(query) { + return { query, text: '', hits: [], sourceRecords: [] } + }, + async write(input) { + return { + accepted: false, + id: input.id ?? `${id}:ignored`, + uri: `memory://${id}/ignored`, + kind: input.kind, + } + }, + async clear() {}, + async flush() {}, + } +} + +export function createInMemoryBenchmarkAdapter(options: { id?: string } = {}): AgentMemoryAdapter { + const id = options.id ?? 'in-memory' + const rows: Array<{ + seq: number + input: AgentMemoryWriteInput + hit: AgentMemoryHit + }> = [] + let seq = 0 + const adapter: AgentMemoryAdapter = { + id, + branchIsolation: { mode: 'scoped' }, + async search(query, searchOptions = {}) { + const scored = rows + .filter((row) => memoryScopeMatches(row.input.scope, searchOptions.scope)) + .filter((row) => !searchOptions.kinds?.length || searchOptions.kinds.includes(row.hit.kind)) + .map((row) => { + const lexical = tokenOverlap(query, row.hit.text) + const recency = row.seq / Math.max(1, seq) + return { + ...row.hit, + score: lexical + recency * 0.01, + normalizedScore: lexical, + } + }) + .filter((hit) => + searchOptions.minScore === undefined ? true : hit.score! >= searchOptions.minScore, + ) + .sort((a, b) => (b.score ?? 0) - (a.score ?? 0)) + return scored.slice(0, searchOptions.limit ?? 5) + }, + async getContext(query, searchOptions = {}) { + const hits = await adapter.search(query, searchOptions) + return { + query, + hits, + sourceRecords: hits.map((hit) => + memoryHitToSourceRecord(hit, { scope: searchOptions.scope }), + ), + text: renderMemoryHits(hits), + } + }, + async write(input) { + seq += 1 + const memoryId = input.id ?? `${id}:${seq}` + const hit: AgentMemoryHit = { + id: memoryId, + uri: `memory://${id}/${encodeURIComponent(memoryId)}`, + kind: input.kind, + text: input.text, + title: input.title, + score: 1, + normalizedScore: 1, + createdAt: input.metadata?.timestamp as string | undefined, + metadata: { + ...(input.metadata ?? {}), + scope: input.scope, + }, + } + rows.push({ seq, input, hit }) + return { + accepted: true, + id: memoryId, + uri: hit.uri, + kind: input.kind, + sourceRecord: memoryWriteResultToSourceRecord( + { + accepted: true, + id: memoryId, + uri: hit.uri, + kind: input.kind, + metadata: hit.metadata, + }, + input.text, + { scope: input.scope }, + ), + metadata: hit.metadata, + } + }, + async clear(scope) { + for (let index = rows.length - 1; index >= 0; index -= 1) { + if (memoryScopeMatches(rows[index]!.input.scope, scope)) rows.splice(index, 1) + } + }, + async flush() {}, + } + return adapter +} + +export function memoryEventId(hit: AgentMemoryHit): string | undefined { + const eventId = hit.metadata?.eventId + return typeof eventId === 'string' ? eventId : undefined +} + +export function memoryActorId(hit: AgentMemoryHit): string | undefined { + const actorId = hit.metadata?.actorId + return typeof actorId === 'string' ? actorId : undefined +} + +function memoryScopeMatches(stored?: AgentMemoryScope, requested?: AgentMemoryScope): boolean { + if (!requested) return true + if (requested.tenantId !== undefined && stored?.tenantId !== requested.tenantId) return false + if (requested.userId !== undefined && stored?.userId !== requested.userId) return false + if (requested.agentId !== undefined && stored?.agentId !== requested.agentId) return false + if (requested.teamId !== undefined && stored?.teamId !== requested.teamId) return false + if (requested.runId !== undefined && stored?.runId !== requested.runId) return false + if (requested.sessionId !== undefined && stored?.sessionId !== requested.sessionId) return false + if (requested.namespace !== undefined && stored?.namespace !== requested.namespace) return false + for (const [key, value] of Object.entries(requested.tags ?? {})) { + if (stored?.tags?.[key] !== value) return false + } + return true +} + +function tokenOverlap(query: string, text: string): number { + const queryTokens = new Set(tokenize(query)) + if (queryTokens.size === 0) return 0 + const textTokens = new Set(tokenize(text)) + let matched = 0 + for (const token of queryTokens) { + if (textTokens.has(token)) matched += 1 + } + return matched / queryTokens.size +} + +function tokenize(text: string): string[] { + const stop = new Set([ + 'the', + 'and', + 'for', + 'this', + 'that', + 'with', + 'what', + 'should', + 'agent', + 'user', + 'current', + 'now', + 'use', + ]) + return text + .toLowerCase() + .split(/[^a-z0-9/]+/) + .filter((token) => token.length > 2 && !stop.has(token)) +} + +function renderMemoryHits(hits: readonly AgentMemoryHit[]): string { + return hits + .map((hit, index) => { + const eventId = memoryEventId(hit) + const actorId = memoryActorId(hit) + return [ + `[${index + 1}] ${hit.title ?? hit.id}`, + eventId ? `event=${eventId}` : '', + actorId ? `actor=${actorId}` : '', + hit.text, + ] + .filter(Boolean) + .join('\n') + }) + .join('\n\n') +} diff --git a/src/benchmarks/catalog.ts b/src/benchmarks/catalog.ts new file mode 100644 index 0000000..31f70d0 --- /dev/null +++ b/src/benchmarks/catalog.ts @@ -0,0 +1,661 @@ +import type { RetrievalGoldTarget, RetrievedKnowledgeHit } from '../retrieval-eval' + +import type { + KnowledgeAnswerBenchmarkTaskKind, + KnowledgeBenchmarkArtifact, + KnowledgeBenchmarkCase, + KnowledgeBenchmarkFamily, + KnowledgeBenchmarkSource, + KnowledgeBenchmarkSpec, + KnowledgeBenchmarkSplit, + KnowledgeMemoryBenchmarkCase, + KnowledgeMemoryBenchmarkTaskKind, +} from './types' + +import { unique } from './utils' + +import { isKnowledgeMemoryBenchmarkCase } from './validation' + +export const INDUSTRY_RAG_BENCHMARKS: readonly (KnowledgeBenchmarkSpec & { + taskKind: 'retrieval' | KnowledgeAnswerBenchmarkTaskKind +})[] = [ + { + id: 'beir', + family: 'beir', + taskKind: 'retrieval', + primaryMetrics: ['nDCG@10', 'Recall@100', 'MRR@10'], + adapter: 'buildRetrievalBenchmarkCasesFromQrels', + notes: 'Classic zero-shot retrieval suites using query/corpus/qrels files.', + }, + { + id: 'mteb-retrieval', + family: 'mteb-retrieval', + taskKind: 'retrieval', + primaryMetrics: ['nDCG@10', 'Recall@100'], + adapter: 'buildRetrievalBenchmarkCasesFromQrels', + notes: 'MTEB retrieval task shape; same qrels bridge, different dataset provenance.', + }, + { + id: 'msmarco', + family: 'msmarco', + taskKind: 'retrieval', + primaryMetrics: ['MRR@10', 'Recall@100'], + adapter: 'buildRetrievalBenchmarkCasesFromQrels', + notes: 'Passage retrieval and reranking smoke for web-style questions.', + }, + { + id: 'trec-dl', + family: 'trec-dl', + taskKind: 'retrieval', + primaryMetrics: ['nDCG@10', 'MAP', 'Recall@100'], + adapter: 'buildRetrievalBenchmarkCasesFromQrels', + notes: 'Deep Learning Track judgments over MS MARCO-derived corpora.', + }, + { + id: 'miracl', + family: 'miracl', + taskKind: 'retrieval', + primaryMetrics: ['nDCG@10', 'Recall@100'], + adapter: 'buildRetrievalBenchmarkCasesFromQrels', + notes: 'Multilingual retrieval; use language tags on cases.', + }, + { + id: 'lotte', + family: 'lotte', + taskKind: 'retrieval', + primaryMetrics: ['Success@5', 'Recall@100'], + adapter: 'buildRetrievalBenchmarkCasesFromQrels', + notes: 'Long-tail search tasks; map collection/domain into tags.', + }, + { + id: 'bright', + family: 'bright', + taskKind: 'retrieval', + primaryMetrics: ['nDCG@10', 'Recall@100'], + adapter: 'buildRetrievalBenchmarkCasesFromQrels', + notes: 'Reasoning-heavy retrieval; preserve domain tags for slice reporting.', + }, + { + id: 'crag', + family: 'crag', + taskKind: 'rag-answer', + primaryMetrics: ['claim_recall', 'citation_recall', 'hallucination_safe'], + adapter: 'KnowledgeAnswerBenchmarkCase', + notes: 'Answer quality and freshness cases; use required/forbidden claims plus citations.', + }, + { + id: 'hotpotqa', + family: 'hotpotqa', + taskKind: 'rag-answer', + primaryMetrics: ['claim_recall', 'citation_recall'], + adapter: 'KnowledgeAnswerBenchmarkCase', + notes: 'Multihop QA; encode each supporting fact as a required claim.', + }, + { + id: 'kilt', + family: 'kilt', + taskKind: 'rag-answer', + primaryMetrics: ['claim_recall', 'citation_recall'], + adapter: 'KnowledgeAnswerBenchmarkCase', + notes: 'Knowledge-intensive generation with provenance; encode expected pages/sources.', + }, + { + id: 'ragtruth', + family: 'ragtruth', + taskKind: 'hallucination', + primaryMetrics: ['hallucination_safe', 'forbidden_claim_rate'], + adapter: 'KnowledgeAnswerBenchmarkCase', + notes: 'Hallucination detection; encode hallucinated spans as forbidden claims.', + }, + { + id: 'faithbench', + family: 'faithbench', + taskKind: 'hallucination', + primaryMetrics: ['hallucination_safe', 'forbidden_claim_rate'], + adapter: 'KnowledgeAnswerBenchmarkCase', + notes: 'Faithfulness benchmark; score unsupported claims as forbidden claims.', + }, + { + id: 'first-party/kb-improvement', + family: 'first-party', + taskKind: 'kb-improvement', + primaryMetrics: ['claim_recall', 'hallucination_safe', 'score'], + adapter: 'KnowledgeAnswerBenchmarkCase', + notes: 'Project-owned candidate-KB validation; grade the produced KB text or answer bundle.', + }, +] + +export const INDUSTRY_MEMORY_BENCHMARKS: readonly (KnowledgeBenchmarkSpec & { + taskKind: KnowledgeMemoryBenchmarkTaskKind +})[] = [ + { + id: 'locomo/qa', + family: 'locomo', + taskKind: 'memory-recall', + primaryMetrics: ['memory_fact_recall', 'memory_event_recall'], + adapter: 'KnowledgeMemoryBenchmarkCase', + notes: 'Long-term conversational QA over multi-session histories.', + }, + { + id: 'locomo/event-summary', + family: 'locomo', + taskKind: 'memory-summarization', + primaryMetrics: ['memory_fact_recall', 'memory_event_recall'], + adapter: 'KnowledgeMemoryBenchmarkCase', + notes: 'Event summarization over long conversational histories.', + }, + { + id: 'longmemeval', + family: 'longmemeval', + taskKind: 'memory-temporal', + primaryMetrics: ['memory_fact_recall', 'memory_stale_safe', 'memory_event_recall'], + adapter: 'KnowledgeMemoryBenchmarkCase', + notes: 'Long-term assistant memory with temporal and update-sensitive probes.', + }, + { + id: 'longmemeval-v2', + family: 'longmemeval-v2', + taskKind: 'memory-reasoning', + primaryMetrics: ['memory_fact_recall', 'memory_event_recall', 'score'], + adapter: 'KnowledgeMemoryBenchmarkCase', + notes: 'Experience reuse from long agent histories; track accuracy and latency.', + }, + { + id: 'memora', + family: 'memora', + taskKind: 'memory-update', + primaryMetrics: ['memory_fact_recall', 'memory_stale_safe', 'memory_stale_rate'], + adapter: 'KnowledgeMemoryBenchmarkCase', + notes: 'Forgetting-aware memory accuracy: reward current facts and penalize obsolete ones.', + }, + { + id: 'memoryagentbench', + family: 'memoryagentbench', + taskKind: 'memory-ingest', + primaryMetrics: ['memory_fact_recall', 'memory_event_recall'], + adapter: 'KnowledgeMemoryBenchmarkCase', + notes: 'Incremental multi-turn information intake before later recall.', + }, + { + id: 'memorybank', + family: 'memorybank', + taskKind: 'memory-recommendation', + primaryMetrics: ['memory_fact_recall', 'memory_stale_safe'], + adapter: 'KnowledgeMemoryBenchmarkCase', + notes: 'Personalized memory use for preference-aware downstream choices.', + }, + { + id: 'groupmembench', + family: 'groupmembench', + taskKind: 'memory-multiparty', + primaryMetrics: ['memory_fact_recall', 'memory_actor_recall', 'memory_stale_safe'], + adapter: 'KnowledgeMemoryBenchmarkCase', + notes: 'Multi-party memory with speaker attribution, update, and term ambiguity pressure.', + }, + { + id: 'first-party/memory-lifecycle', + family: 'first-party', + taskKind: 'memory-forgetting', + primaryMetrics: ['memory_fact_recall', 'memory_stale_safe', 'memory_event_recall'], + adapter: 'KnowledgeMemoryBenchmarkCase', + notes: 'Project-owned lifecycle pack for ingest, recall, update, forgetting, and ambiguity.', + }, +] + +export function buildIndustryRagBenchmarkSmokeCases( + specs: readonly (KnowledgeBenchmarkSpec & { + taskKind: 'retrieval' | KnowledgeAnswerBenchmarkTaskKind + })[] = INDUSTRY_RAG_BENCHMARKS, +): KnowledgeBenchmarkCase[] { + return specs.map((spec) => { + const source = { + name: spec.id, + version: 'smoke', + } + const split = spec.taskKind === 'retrieval' ? 'search' : 'holdout' + const tags = unique(['industry-smoke', spec.id, spec.family, spec.taskKind]) + if (spec.taskKind === 'retrieval') { + return { + id: `${spec.id}/smoke:q1`, + family: spec.family, + taskKind: 'retrieval', + split, + tags, + source, + query: `${spec.id} smoke retrieval query`, + expected: [{ kind: 'page', pageId: `${spec.id}:doc-1` }], + k: 5, + metadata: { + adapter: spec.adapter, + primaryMetrics: spec.primaryMetrics, + }, + } + } + + return { + id: `${spec.id}/smoke:q1`, + family: spec.family, + taskKind: spec.taskKind, + split, + tags, + source, + prompt: `${spec.id} smoke benchmark prompt`, + requiredClaims: [ + { + id: `${spec.id}:required`, + anyOf: [`${spec.id} supported answer`], + }, + ], + forbiddenClaims: [ + { + id: `${spec.id}:unsupported`, + anyOf: [`${spec.id} unsupported claim`], + }, + ], + expectedSourceIds: [`${spec.id}:source-1`], + referenceAnswer: `${spec.id} supported answer`, + metadata: { + adapter: spec.adapter, + primaryMetrics: spec.primaryMetrics, + }, + } + }) +} + +export function respondToIndustryRagBenchmarkSmokeCase(input: { + case: KnowledgeBenchmarkCase +}): KnowledgeBenchmarkArtifact { + const testCase = input.case + if (isKnowledgeMemoryBenchmarkCase(testCase)) { + return respondToIndustryMemoryBenchmarkSmokeCase({ case: testCase }) + } + if (testCase.taskKind === 'retrieval') { + const expected = Array.isArray(testCase.expected) ? testCase.expected[0] : testCase.expected + const hit = hitForExpectedTarget(expected, testCase.id) + return { + hits: [hit], + durationMs: 1, + metadata: { + smoke: true, + }, + } + } + + return { + answer: (testCase.requiredClaims ?? []) + .map((claim) => claim.anyOf[0]) + .filter((fragment): fragment is string => Boolean(fragment)) + .join(' '), + citedSourceIds: testCase.expectedSourceIds ?? [], + durationMs: 1, + metadata: { + smoke: true, + }, + } +} + +export function buildIndustryMemoryBenchmarkSmokeCases( + specs: readonly (KnowledgeBenchmarkSpec & { + taskKind: KnowledgeMemoryBenchmarkTaskKind + })[] = INDUSTRY_MEMORY_BENCHMARKS, +): KnowledgeMemoryBenchmarkCase[] { + return specs.map((spec) => { + const currentEventId = `${spec.id}:event-current` + const staleEventId = `${spec.id}:event-stale` + const actorId = spec.taskKind === 'memory-multiparty' ? 'teammate-ada' : 'user' + const currentFact = `${spec.id} current memory` + const staleFact = `${spec.id} stale memory` + return { + id: `${spec.id}/smoke:q1`, + family: spec.family, + taskKind: spec.taskKind, + split: spec.taskKind === 'memory-forgetting' ? 'holdout' : 'dev', + tags: unique(['memory-smoke', spec.id, spec.family, spec.taskKind]), + source: { + name: spec.id, + version: 'smoke', + }, + events: [ + { + id: staleEventId, + actorId, + sessionId: `${spec.id}:session-1`, + timestamp: '2026-01-01T00:00:00.000Z', + text: `${actorId} once had this obsolete fact: ${staleFact}.`, + }, + { + id: currentEventId, + actorId, + sessionId: `${spec.id}:session-2`, + timestamp: '2026-02-01T00:00:00.000Z', + text: `${actorId} updated the durable fact to: ${currentFact}.`, + }, + ], + prompt: `Use memory to answer the ${spec.id} smoke probe.`, + requiredFacts: [ + { + id: `${spec.id}:current`, + anyOf: [currentFact], + sourceEventIds: [currentEventId], + }, + ], + forbiddenFacts: [ + { + id: `${spec.id}:stale`, + anyOf: [staleFact], + sourceEventIds: [staleEventId], + obsolete: true, + }, + ], + expectedEventIds: [currentEventId], + expectedActorIds: [actorId], + referenceAnswer: currentFact, + metadata: { + adapter: spec.adapter, + primaryMetrics: spec.primaryMetrics, + }, + } + }) +} + +export function respondToIndustryMemoryBenchmarkSmokeCase(input: { + case: KnowledgeMemoryBenchmarkCase +}): KnowledgeBenchmarkArtifact { + const testCase = input.case + const facts = testCase.requiredFacts + ?.map((fact) => fact.anyOf[0]) + .filter((fragment): fragment is string => Boolean(fragment)) + return { + answer: facts?.join(' ') ?? '', + rememberedFacts: facts ?? [], + citedEventIds: testCase.expectedEventIds ?? [], + actorIds: testCase.expectedActorIds ?? [], + durationMs: 1, + metadata: { + smoke: true, + }, + } +} + +export function buildFirstPartyMemoryLifecycleBenchmarkCases(): KnowledgeMemoryBenchmarkCase[] { + const base = { + family: 'first-party', + source: { + name: 'first-party/memory-lifecycle', + version: 'real-v1', + }, + } as const + return [ + memoryLifecycleCase({ + ...base, + id: 'first-party/memory-lifecycle:allergy', + taskKind: 'memory-ingest', + split: 'dev', + actorId: 'user', + prompt: 'What food restriction should catering remember for this user?', + staleText: 'The user said they had no food allergies on the first onboarding form.', + currentText: 'The user later corrected the profile: they have a severe peanut allergy.', + required: 'severe peanut allergy', + forbidden: 'no food allergies', + }), + memoryLifecycleCase({ + ...base, + id: 'first-party/memory-lifecycle:account-tier', + taskKind: 'memory-recall', + split: 'dev', + actorId: 'sales-ops', + prompt: 'What is the customer account tier now?', + staleText: 'Sales ops originally marked the customer as starter tier.', + currentText: 'Sales ops updated the customer to enterprise tier after procurement approval.', + required: 'enterprise tier', + forbidden: 'starter tier', + }), + memoryLifecycleCase({ + ...base, + id: 'first-party/memory-lifecycle:launch-date', + taskKind: 'memory-temporal', + split: 'holdout', + actorId: 'pm', + prompt: 'What launch date should the agent use?', + staleText: 'The PM first planned the launch for April 3.', + currentText: 'The PM moved the launch date to April 17 after legal review.', + required: 'April 17', + forbidden: 'April 3', + }), + memoryLifecycleCase({ + ...base, + id: 'first-party/memory-lifecycle:briefing-channel', + taskKind: 'memory-update', + split: 'holdout', + actorId: 'user', + prompt: 'How should daily briefings be delivered now?', + staleText: 'The user used to want daily briefings by SMS.', + currentText: 'The user changed daily briefings to email only.', + required: 'email only', + forbidden: 'SMS', + }), + memoryLifecycleCase({ + ...base, + id: 'first-party/memory-lifecycle:shipping-address', + taskKind: 'memory-forgetting', + split: 'holdout', + actorId: 'user', + prompt: 'What shipping address is current?', + staleText: 'The old shipping address was 14 Pine Street, Apartment 2.', + currentText: 'The current shipping address is 88 Cedar Avenue, Suite 9.', + required: '88 Cedar Avenue, Suite 9', + forbidden: '14 Pine Street', + }), + memoryLifecycleCase({ + ...base, + id: 'first-party/memory-lifecycle:approval-owner', + taskKind: 'memory-reasoning', + split: 'holdout', + actorId: 'finance', + prompt: 'Who owns travel approval now?', + staleText: 'Finance said Liam owned travel approvals last quarter.', + currentText: 'Finance reassigned travel approvals to Maya this quarter.', + required: 'Maya', + forbidden: 'Liam', + }), + memoryLifecycleCase({ + ...base, + id: 'first-party/memory-lifecycle:project-summary', + taskKind: 'memory-summarization', + split: 'dev', + actorId: 'pm', + prompt: 'Summarize the current Project Orion risks.', + staleText: 'The old Project Orion risk was logo color churn.', + currentText: 'The current Project Orion risks are vendor delay and a QA staffing gap.', + required: 'vendor delay', + extraRequired: 'QA staffing gap', + forbidden: 'logo color churn', + }), + memoryLifecycleCase({ + ...base, + id: 'first-party/memory-lifecycle:meeting-format', + taskKind: 'memory-recommendation', + split: 'holdout', + actorId: 'user', + prompt: 'What meeting format should the agent recommend?', + staleText: 'The user previously preferred long video calls for planning.', + currentText: 'The user now prefers async docs for planning instead of video calls.', + required: 'async docs', + forbidden: 'long video calls', + }), + memoryLifecycleCase({ + ...base, + id: 'first-party/memory-lifecycle:multiparty-ada', + taskKind: 'memory-multiparty', + split: 'holdout', + actorId: 'ada', + prompt: 'Which SDK language did Ada ask for?', + staleText: 'Ben asked for a Python notebook example.', + currentText: 'Ada asked for a Rust SDK example.', + required: 'Rust SDK', + forbidden: 'Python notebook', + }), + memoryLifecycleCase({ + ...base, + id: 'first-party/memory-lifecycle:timezone', + taskKind: 'memory-recall', + split: 'dev', + actorId: 'user', + prompt: 'What timezone should scheduling use for this user?', + staleText: 'The user profile originally listed Pacific time.', + currentText: 'The user corrected scheduling to America/Denver time.', + required: 'America/Denver', + forbidden: 'Pacific time', + }), + memoryLifecycleCase({ + ...base, + id: 'first-party/memory-lifecycle:dinner-preference', + taskKind: 'memory-update', + split: 'holdout', + actorId: 'user', + prompt: 'What dinner preference should the assistant use?', + staleText: 'The user was previously vegetarian for team dinners.', + currentText: 'The user updated dinner restrictions to no shellfish.', + required: 'no shellfish', + forbidden: 'vegetarian', + }), + memoryLifecycleCase({ + ...base, + id: 'first-party/memory-lifecycle:support-sla', + taskKind: 'memory-temporal', + split: 'holdout', + actorId: 'support-lead', + prompt: 'What support SLA is current?', + staleText: 'Support used to promise a 24 hour response SLA.', + currentText: 'Support changed the current response SLA to 2 business hours.', + required: '2 business hours', + forbidden: '24 hour', + }), + ] +} + +function hitForExpectedTarget( + expected: RetrievalGoldTarget | undefined, + fallbackId: string, +): RetrievedKnowledgeHit { + if (!expected) { + return { + pageId: fallbackId, + path: `${fallbackId}.md`, + rank: 1, + } + } + switch (expected.kind) { + case 'page': + return { + pageId: expected.pageId, + path: `${expected.pageId}.md`, + rank: 1, + } + case 'page-path': + return { + pageId: expected.path, + path: expected.path, + rank: 1, + } + case 'source': + return { + pageId: expected.sourceId, + path: `${expected.sourceId}.md`, + sourceIds: [expected.sourceId], + rank: 1, + } + case 'source-anchor': + return { + pageId: expected.sourceId, + path: `${expected.sourceId}.md`, + sourceIds: [expected.sourceId], + sourceSpans: [{ sourceId: expected.sourceId, anchorId: expected.anchorId }], + rank: 1, + } + case 'source-span': + return { + pageId: expected.sourceId, + path: `${expected.sourceId}.md`, + sourceIds: [expected.sourceId], + sourceSpans: [ + { + sourceId: expected.sourceId, + charStart: expected.charStart, + charEnd: expected.charEnd, + }, + ], + rank: 1, + } + } +} + +function memoryLifecycleCase(input: { + id: string + family: KnowledgeBenchmarkFamily + source: KnowledgeBenchmarkSource + taskKind: KnowledgeMemoryBenchmarkTaskKind + split: KnowledgeBenchmarkSplit + actorId: string + prompt: string + staleText: string + currentText: string + required: string + extraRequired?: string + forbidden: string +}): KnowledgeMemoryBenchmarkCase { + const staleEventId = `${input.id}:stale` + const currentEventId = `${input.id}:current` + return { + id: input.id, + family: input.family, + taskKind: input.taskKind, + split: input.split, + tags: unique(['first-party-memory-lifecycle', input.taskKind, input.split]), + source: input.source, + events: [ + { + id: staleEventId, + actorId: input.actorId, + sessionId: `${input.id}:session-1`, + timestamp: '2026-01-01T00:00:00.000Z', + text: input.staleText, + }, + { + id: currentEventId, + actorId: input.actorId, + sessionId: `${input.id}:session-2`, + timestamp: '2026-02-01T00:00:00.000Z', + text: input.currentText, + }, + ], + prompt: input.prompt, + requiredFacts: [ + { + id: `${input.id}:required-1`, + anyOf: [input.required], + sourceEventIds: [currentEventId], + }, + ...(input.extraRequired + ? [ + { + id: `${input.id}:required-2`, + anyOf: [input.extraRequired], + sourceEventIds: [currentEventId], + }, + ] + : []), + ], + forbiddenFacts: [ + { + id: `${input.id}:stale`, + anyOf: [input.forbidden], + sourceEventIds: [staleEventId], + obsolete: true, + }, + ], + expectedEventIds: [currentEventId], + expectedActorIds: [input.actorId], + referenceAnswer: input.required, + } +} diff --git a/src/benchmarks/index.ts b/src/benchmarks/index.ts index 9e86aec..5ced2b4 100644 --- a/src/benchmarks/index.ts +++ b/src/benchmarks/index.ts @@ -1,3126 +1,64 @@ -import { randomUUID } from 'node:crypto' -import { join } from 'node:path' -import { canonicalJson } from '@tangle-network/agent-eval' -import { - type CampaignResult, - type CampaignStorage, - type CostLedgerHandle, - createRunCostLedger, - type DispatchContext, - fsCampaignStorage, - type JudgeConfig, - type RunCampaignOptions, - resolveRunDir, - runCampaign, - type Scenario, -} from '@tangle-network/agent-eval/campaign' -import { stableId } from '../ids' -import { - appendAttemptJournalEvent, - assertNoInterruptedPaidCalls, - DEFAULT_MEMORY_RECOVERY_RETRIES_PER_ATTEMPT, - hasSettledPaidCall, - readActiveAttemptJournal, - reconcileInterruptedMemoryPaidCalls, - reserveRecoveryAttempts, -} from '../memory/attempt-log' -import { - createMemoryExecutionPool, - memoryRecoveryDelayMs, - releaseMemoryAdapterCreatedAfterAbort, - resolveMemoryCleanupTimeoutMs, - runBoundedMemoryLifecycle, - sleepForMemoryRecovery, -} from '../memory/lifecycle' -import { - type AgentMemoryAcquireRunLease, - type AgentMemoryControllerMode, - acquireAgentMemoryRunLease, - type OwnedAgentMemoryRunLease, -} from '../memory/run-control' -import { memoryHitToSourceRecord, memoryWriteResultToSourceRecord } from '../memory/source-record' -import type { - AgentMemoryAdapter, - AgentMemoryHit, - AgentMemoryScope, - AgentMemoryWriteInput, -} from '../memory/types' -import { - type RetrievalEvalArtifact, - type RetrievalEvalScenario, - type RetrievalGoldTarget, - type RetrievedKnowledgeHit, - scoreRetrievalArtifact, -} from '../retrieval-eval' - -const KNOWLEDGE_BENCHMARK_IMPLEMENTATION_REF = 'agent-knowledge:benchmark-suite:v2' -const MEMORY_ADAPTER_BENCHMARK_IMPLEMENTATION_REF = 'agent-knowledge:memory-adapter-benchmark:v7' - -export type KnowledgeBenchmarkTaskKind = - | 'retrieval' - | 'rag-answer' - | 'hallucination' - | 'kb-improvement' - | 'memory-ingest' - | 'memory-recall' - | 'memory-temporal' - | 'memory-update' - | 'memory-forgetting' - | 'memory-reasoning' - | 'memory-summarization' - | 'memory-recommendation' - | 'memory-multiparty' - -export type KnowledgeAnswerBenchmarkTaskKind = 'rag-answer' | 'hallucination' | 'kb-improvement' - -export type KnowledgeMemoryBenchmarkTaskKind = Exclude< +export { + createInMemoryBenchmarkAdapter, + createNoopMemoryBenchmarkAdapter, +} from './adapters' +export { + buildFirstPartyMemoryLifecycleBenchmarkCases, + buildIndustryMemoryBenchmarkSmokeCases, + buildIndustryRagBenchmarkSmokeCases, + INDUSTRY_MEMORY_BENCHMARKS, + INDUSTRY_RAG_BENCHMARKS, + respondToIndustryMemoryBenchmarkSmokeCase, + respondToIndustryRagBenchmarkSmokeCase, +} from './catalog' +export { runMemoryAdapterBenchmark } from './memory-runner' +export { summarizeKnowledgeBenchmarkCampaign } from './metrics' +export { + buildRetrievalBenchmarkCasesFromQrels, + parseKnowledgeBenchmarkJsonl, + parseKnowledgeBenchmarkQrels, +} from './retrieval' +export { + scoreKnowledgeBenchmarkArtifact, + scoreMemoryBenchmarkArtifact, +} from './scoring' +export { + buildKnowledgeBenchmarkScenarios, + knowledgeBenchmarkJudge, + renderKnowledgeBenchmarkReportMarkdown, + runKnowledgeBenchmarkSuite, +} from './suite' +export type { + BuildRetrievalBenchmarkCasesFromQrelsOptions, + KnowledgeAnswerBenchmarkCase, + KnowledgeAnswerBenchmarkTaskKind, + KnowledgeBenchmarkArtifact, + KnowledgeBenchmarkCase, + KnowledgeBenchmarkCaseBase, + KnowledgeBenchmarkDistribution, + KnowledgeBenchmarkEvaluation, + KnowledgeBenchmarkFamily, + KnowledgeBenchmarkReport, + KnowledgeBenchmarkResponder, + KnowledgeBenchmarkScenario, + KnowledgeBenchmarkSliceSummary, + KnowledgeBenchmarkSource, + KnowledgeBenchmarkSpec, + KnowledgeBenchmarkSplit, KnowledgeBenchmarkTaskKind, - 'retrieval' | KnowledgeAnswerBenchmarkTaskKind -> - -export type KnowledgeBenchmarkFamily = - | 'beir' - | 'mteb-retrieval' - | 'msmarco' - | 'trec-dl' - | 'miracl' - | 'lotte' - | 'bright' - | 'crag' - | 'hotpotqa' - | 'kilt' - | 'ragtruth' - | 'faithbench' - | 'locomo' - | 'longmemeval' - | 'longmemeval-v2' - | 'memora' - | 'memoryagentbench' - | 'memorybank' - | 'groupmembench' - | 'first-party' - | 'custom' - -export type KnowledgeBenchmarkSplit = 'search' | 'dev' | 'holdout' | string - -export interface KnowledgeBenchmarkSource { - name?: string - url?: string - version?: string - license?: string - citation?: string -} - -export interface KnowledgeBenchmarkSpec { - id: string - family: KnowledgeBenchmarkFamily - taskKind: KnowledgeBenchmarkTaskKind - primaryMetrics: readonly string[] - adapter: string - notes: string -} - -export interface KnowledgeBenchmarkCaseBase { - id: string - family: KnowledgeBenchmarkFamily | string - taskKind: KnowledgeBenchmarkTaskKind - split?: KnowledgeBenchmarkSplit - tags?: readonly string[] - source?: KnowledgeBenchmarkSource - metadata?: Record -} - -export interface KnowledgeRetrievalBenchmarkCase extends KnowledgeBenchmarkCaseBase { - taskKind: 'retrieval' - query: string - expected: RetrievalGoldTarget | readonly RetrievalGoldTarget[] - k?: number -} - -export interface KnowledgeClaimMatcher { - id: string - anyOf: readonly string[] - weight?: number -} - -export interface KnowledgeMemoryEvent { - id: string - text: string - actorId?: string - sessionId?: string - timestamp?: string - metadata?: Record -} - -export interface KnowledgeMemoryFactMatcher extends KnowledgeClaimMatcher { - sourceEventIds?: readonly string[] - validAt?: string - obsolete?: boolean -} - -export interface KnowledgeAnswerBenchmarkCase extends KnowledgeBenchmarkCaseBase { - taskKind: KnowledgeAnswerBenchmarkTaskKind - prompt: string - requiredClaims?: readonly KnowledgeClaimMatcher[] - forbiddenClaims?: readonly KnowledgeClaimMatcher[] - expectedSourceIds?: readonly string[] - referenceAnswer?: string -} - -export interface KnowledgeMemoryBenchmarkCase extends KnowledgeBenchmarkCaseBase { - taskKind: KnowledgeMemoryBenchmarkTaskKind - events: readonly KnowledgeMemoryEvent[] - prompt: string - requiredFacts?: readonly KnowledgeMemoryFactMatcher[] - forbiddenFacts?: readonly KnowledgeMemoryFactMatcher[] - expectedEventIds?: readonly string[] - expectedActorIds?: readonly string[] - referenceAnswer?: string -} - -export type KnowledgeBenchmarkCase = - | KnowledgeRetrievalBenchmarkCase - | KnowledgeAnswerBenchmarkCase - | KnowledgeMemoryBenchmarkCase - -export interface KnowledgeBenchmarkArtifact { - answer?: string - text?: string - hits?: readonly RetrievedKnowledgeHit[] - citedSourceIds?: readonly string[] - rememberedFacts?: readonly string[] - citedEventIds?: readonly string[] - usedMemoryIds?: readonly string[] - actorIds?: readonly string[] - /** Informational copy. Billable responders account through context.cost.runPaidCall. */ - costUsd?: number - durationMs?: number - metadata?: Record -} - -export interface KnowledgeBenchmarkEvaluation { - score: number - passed: boolean - dimensions: Record - /** Dimensions for which this case declared an actual target. */ - applicableDimensions?: readonly string[] - notes: string - raw: Record -} - -export interface KnowledgeBenchmarkScenario extends Scenario { - kind: 'knowledge-benchmark' - family: KnowledgeBenchmarkFamily | string - taskKind: KnowledgeBenchmarkTaskKind - splitTag: KnowledgeBenchmarkSplit - case: KnowledgeBenchmarkCase -} - -export type KnowledgeBenchmarkResponder = (input: { - case: KnowledgeBenchmarkCase - scenario: KnowledgeBenchmarkScenario - context: DispatchContext -}) => Promise | TArtifact - -export interface RunKnowledgeBenchmarkSuiteOptions { - cases: readonly KnowledgeBenchmarkCase[] - respond: KnowledgeBenchmarkResponder - /** Versioned identity for the model, prompt, retrieval, and runtime behavior. */ - respondRef?: string - runDir: string - splits?: readonly KnowledgeBenchmarkSplit[] - repo?: string - seed?: number - reps?: number - resumable?: boolean - costCeiling?: number - /** Shared across nested benchmark suites when an outer run owns spend. */ - costLedger?: CostLedgerHandle - costPhase?: string - maxConcurrency?: number - dispatchTimeoutMs?: number - expectUsage?: 'assert' | 'warn' | 'off' - storage?: CampaignStorage - now?: () => Date -} - -export interface KnowledgeBenchmarkDistribution { - n: number - min: number - mean: number - median: number - p90: number - max: number -} - -export interface KnowledgeBenchmarkSliceSummary { - n: number - meanScore: number - passRate: number - score: KnowledgeBenchmarkDistribution -} - -export interface KnowledgeBenchmarkReport { - totalCases: number - totalCells: number - cellsFailed: number - cellsCached: number - totalCostUsd: number - bySplit: Record - byFamily: Record - byTaskKind: Record - dimensions: Record - score: KnowledgeBenchmarkDistribution -} - -export interface RunKnowledgeBenchmarkSuiteResult { - scenarios: readonly KnowledgeBenchmarkScenario[] - campaign: CampaignResult - report: KnowledgeBenchmarkReport - reportJsonPath: string - reportMarkdownPath: string -} - -export interface MemoryAdapterBenchmarkCandidate { - id: string - /** Versioned adapter and configuration identity used by resumable caches. */ - ref: string - /** Expected adapter.id. Defaults to candidate id and permits lazy no-work resume. */ - adapterId?: string - label?: string - /** Local construction is free; call markExternalCall before billable provisioning or reconnects. */ - createAdapter: (input: { - purpose: 'execute' | 'recovery' - signal: AbortSignal - markExternalCall(): void - }) => AgentMemoryAdapter | Promise - /** Conservative charge for one billable adapter provisioning or reconnect call. */ - adapterCreationCostUsd?: number - searchLimit?: number - costUsdPerCase?: number - /** Conservative extra provider charge for recovering one interrupted case. */ - recoveryCostUsdPerAttempt?: number - scope?: AgentMemoryScope -} - -export interface RunMemoryAdapterBenchmarkOptions { - cases: readonly KnowledgeMemoryBenchmarkCase[] - candidates: readonly MemoryAdapterBenchmarkCandidate[] - /** Retired candidates retained only so interrupted scopes can be cleaned on resume. */ - recoveryCandidates?: readonly MemoryAdapterBenchmarkCandidate[] - runDir: string - storage?: CampaignStorage - repo?: string - seed?: number - reps?: number - resumable?: boolean - costCeiling?: number - /** Shared with nested benchmark suites so the dollar limit applies to the whole comparison. */ - costLedger?: CostLedgerHandle - costPhase?: string - maxConcurrency?: number - dispatchTimeoutMs?: number - cleanupTimeoutMs?: number - /** Refuse a damaged run with more unfinished attempts than this. Default 1000. */ - maxRecoveryAttempts?: number - /** Bound repeated provider cleanup after process crashes. Default 3 per attempt. */ - maxRecoveryRetriesPerAttempt?: number - expectUsage?: 'assert' | 'warn' | 'off' - now?: () => Date - /** Required with custom storage when all controllers are confined to one process. */ - controllerMode?: AgentMemoryControllerMode - /** Required for distributed controllers that share custom storage. */ - acquireRunLease?: AgentMemoryAcquireRunLease -} - -export interface MemoryAdapterBenchmarkRankingRow { - rank: number - candidateId: string - label: string - adapterId: string - scoreMean: number - passRate: number - totalCases: number - totalCells: number - cellsFailed: number - totalCostUsd: number - reportJsonPath: string - reportMarkdownPath: string - report: KnowledgeBenchmarkReport -} - -export interface RunMemoryAdapterBenchmarkResult { - rows: readonly MemoryAdapterBenchmarkRankingRow[] - totalCostUsd: number - /** Recovery spend for retired candidates, excluded from ranking rows but included in totalCostUsd. */ - unrankedRecoveryCostUsd: number - rankingJsonPath: string - rankingMarkdownPath: string - attemptLogPath: string - recoveryLogPath: string -} - -class MemoryAdapterBenchmarkCleanupError extends AggregateError {} - -interface MemoryAdapterBenchmarkAttemptEvent { - schema: 3 - status: 'started' | 'cleaned' - attemptId: string - candidateId: string - candidateRef: string - adapterId: string - caseId: string - cellId: string - scope: AgentMemoryScope - adapterCreationCostUsd: number - costUsdPerCase: number - recoveryCostUsdPerAttempt: number - recordedAt: string - recovery: boolean -} - -export interface KnowledgeRetrievalBenchmarkQuery { - id: string - text: string - split?: KnowledgeBenchmarkSplit - tags?: readonly string[] - metadata?: Record -} - -export interface KnowledgeRetrievalBenchmarkQrel { - queryId: string - documentId: string - score: number -} - -export interface BuildRetrievalBenchmarkCasesFromQrelsOptions { - benchmarkId: string - family: KnowledgeBenchmarkFamily | string - queries: readonly KnowledgeRetrievalBenchmarkQuery[] - qrels: readonly KnowledgeRetrievalBenchmarkQrel[] - source?: KnowledgeBenchmarkSource - tags?: readonly string[] - k?: number - targetKind?: 'page' | 'page-path' | 'source' - documentTarget?: ( - documentId: string, - qrel: KnowledgeRetrievalBenchmarkQrel, - ) => RetrievalGoldTarget - splitOf?: (queryId: string) => KnowledgeBenchmarkSplit -} - -export const INDUSTRY_RAG_BENCHMARKS: readonly (KnowledgeBenchmarkSpec & { - taskKind: 'retrieval' | KnowledgeAnswerBenchmarkTaskKind -})[] = [ - { - id: 'beir', - family: 'beir', - taskKind: 'retrieval', - primaryMetrics: ['nDCG@10', 'Recall@100', 'MRR@10'], - adapter: 'buildRetrievalBenchmarkCasesFromQrels', - notes: 'Classic zero-shot retrieval suites using query/corpus/qrels files.', - }, - { - id: 'mteb-retrieval', - family: 'mteb-retrieval', - taskKind: 'retrieval', - primaryMetrics: ['nDCG@10', 'Recall@100'], - adapter: 'buildRetrievalBenchmarkCasesFromQrels', - notes: 'MTEB retrieval task shape; same qrels bridge, different dataset provenance.', - }, - { - id: 'msmarco', - family: 'msmarco', - taskKind: 'retrieval', - primaryMetrics: ['MRR@10', 'Recall@100'], - adapter: 'buildRetrievalBenchmarkCasesFromQrels', - notes: 'Passage retrieval and reranking smoke for web-style questions.', - }, - { - id: 'trec-dl', - family: 'trec-dl', - taskKind: 'retrieval', - primaryMetrics: ['nDCG@10', 'MAP', 'Recall@100'], - adapter: 'buildRetrievalBenchmarkCasesFromQrels', - notes: 'Deep Learning Track judgments over MS MARCO-derived corpora.', - }, - { - id: 'miracl', - family: 'miracl', - taskKind: 'retrieval', - primaryMetrics: ['nDCG@10', 'Recall@100'], - adapter: 'buildRetrievalBenchmarkCasesFromQrels', - notes: 'Multilingual retrieval; use language tags on cases.', - }, - { - id: 'lotte', - family: 'lotte', - taskKind: 'retrieval', - primaryMetrics: ['Success@5', 'Recall@100'], - adapter: 'buildRetrievalBenchmarkCasesFromQrels', - notes: 'Long-tail search tasks; map collection/domain into tags.', - }, - { - id: 'bright', - family: 'bright', - taskKind: 'retrieval', - primaryMetrics: ['nDCG@10', 'Recall@100'], - adapter: 'buildRetrievalBenchmarkCasesFromQrels', - notes: 'Reasoning-heavy retrieval; preserve domain tags for slice reporting.', - }, - { - id: 'crag', - family: 'crag', - taskKind: 'rag-answer', - primaryMetrics: ['claim_recall', 'citation_recall', 'hallucination_safe'], - adapter: 'KnowledgeAnswerBenchmarkCase', - notes: 'Answer quality and freshness cases; use required/forbidden claims plus citations.', - }, - { - id: 'hotpotqa', - family: 'hotpotqa', - taskKind: 'rag-answer', - primaryMetrics: ['claim_recall', 'citation_recall'], - adapter: 'KnowledgeAnswerBenchmarkCase', - notes: 'Multihop QA; encode each supporting fact as a required claim.', - }, - { - id: 'kilt', - family: 'kilt', - taskKind: 'rag-answer', - primaryMetrics: ['claim_recall', 'citation_recall'], - adapter: 'KnowledgeAnswerBenchmarkCase', - notes: 'Knowledge-intensive generation with provenance; encode expected pages/sources.', - }, - { - id: 'ragtruth', - family: 'ragtruth', - taskKind: 'hallucination', - primaryMetrics: ['hallucination_safe', 'forbidden_claim_rate'], - adapter: 'KnowledgeAnswerBenchmarkCase', - notes: 'Hallucination detection; encode hallucinated spans as forbidden claims.', - }, - { - id: 'faithbench', - family: 'faithbench', - taskKind: 'hallucination', - primaryMetrics: ['hallucination_safe', 'forbidden_claim_rate'], - adapter: 'KnowledgeAnswerBenchmarkCase', - notes: 'Faithfulness benchmark; score unsupported claims as forbidden claims.', - }, - { - id: 'first-party/kb-improvement', - family: 'first-party', - taskKind: 'kb-improvement', - primaryMetrics: ['claim_recall', 'hallucination_safe', 'score'], - adapter: 'KnowledgeAnswerBenchmarkCase', - notes: 'Project-owned candidate-KB validation; grade the produced KB text or answer bundle.', - }, -] - -export const INDUSTRY_MEMORY_BENCHMARKS: readonly (KnowledgeBenchmarkSpec & { - taskKind: KnowledgeMemoryBenchmarkTaskKind -})[] = [ - { - id: 'locomo/qa', - family: 'locomo', - taskKind: 'memory-recall', - primaryMetrics: ['memory_fact_recall', 'memory_event_recall'], - adapter: 'KnowledgeMemoryBenchmarkCase', - notes: 'Long-term conversational QA over multi-session histories.', - }, - { - id: 'locomo/event-summary', - family: 'locomo', - taskKind: 'memory-summarization', - primaryMetrics: ['memory_fact_recall', 'memory_event_recall'], - adapter: 'KnowledgeMemoryBenchmarkCase', - notes: 'Event summarization over long conversational histories.', - }, - { - id: 'longmemeval', - family: 'longmemeval', - taskKind: 'memory-temporal', - primaryMetrics: ['memory_fact_recall', 'memory_stale_safe', 'memory_event_recall'], - adapter: 'KnowledgeMemoryBenchmarkCase', - notes: 'Long-term assistant memory with temporal and update-sensitive probes.', - }, - { - id: 'longmemeval-v2', - family: 'longmemeval-v2', - taskKind: 'memory-reasoning', - primaryMetrics: ['memory_fact_recall', 'memory_event_recall', 'score'], - adapter: 'KnowledgeMemoryBenchmarkCase', - notes: 'Experience reuse from long agent histories; track accuracy and latency.', - }, - { - id: 'memora', - family: 'memora', - taskKind: 'memory-update', - primaryMetrics: ['memory_fact_recall', 'memory_stale_safe', 'memory_stale_rate'], - adapter: 'KnowledgeMemoryBenchmarkCase', - notes: 'Forgetting-aware memory accuracy: reward current facts and penalize obsolete ones.', - }, - { - id: 'memoryagentbench', - family: 'memoryagentbench', - taskKind: 'memory-ingest', - primaryMetrics: ['memory_fact_recall', 'memory_event_recall'], - adapter: 'KnowledgeMemoryBenchmarkCase', - notes: 'Incremental multi-turn information intake before later recall.', - }, - { - id: 'memorybank', - family: 'memorybank', - taskKind: 'memory-recommendation', - primaryMetrics: ['memory_fact_recall', 'memory_stale_safe'], - adapter: 'KnowledgeMemoryBenchmarkCase', - notes: 'Personalized memory use for preference-aware downstream choices.', - }, - { - id: 'groupmembench', - family: 'groupmembench', - taskKind: 'memory-multiparty', - primaryMetrics: ['memory_fact_recall', 'memory_actor_recall', 'memory_stale_safe'], - adapter: 'KnowledgeMemoryBenchmarkCase', - notes: 'Multi-party memory with speaker attribution, update, and term ambiguity pressure.', - }, - { - id: 'first-party/memory-lifecycle', - family: 'first-party', - taskKind: 'memory-forgetting', - primaryMetrics: ['memory_fact_recall', 'memory_stale_safe', 'memory_event_recall'], - adapter: 'KnowledgeMemoryBenchmarkCase', - notes: 'Project-owned lifecycle pack for ingest, recall, update, forgetting, and ambiguity.', - }, -] - -export function buildIndustryRagBenchmarkSmokeCases( - specs: readonly (KnowledgeBenchmarkSpec & { - taskKind: 'retrieval' | KnowledgeAnswerBenchmarkTaskKind - })[] = INDUSTRY_RAG_BENCHMARKS, -): KnowledgeBenchmarkCase[] { - return specs.map((spec) => { - const source = { - name: spec.id, - version: 'smoke', - } - const split = spec.taskKind === 'retrieval' ? 'search' : 'holdout' - const tags = unique(['industry-smoke', spec.id, spec.family, spec.taskKind]) - if (spec.taskKind === 'retrieval') { - return { - id: `${spec.id}/smoke:q1`, - family: spec.family, - taskKind: 'retrieval', - split, - tags, - source, - query: `${spec.id} smoke retrieval query`, - expected: [{ kind: 'page', pageId: `${spec.id}:doc-1` }], - k: 5, - metadata: { - adapter: spec.adapter, - primaryMetrics: spec.primaryMetrics, - }, - } - } - - return { - id: `${spec.id}/smoke:q1`, - family: spec.family, - taskKind: spec.taskKind, - split, - tags, - source, - prompt: `${spec.id} smoke benchmark prompt`, - requiredClaims: [ - { - id: `${spec.id}:required`, - anyOf: [`${spec.id} supported answer`], - }, - ], - forbiddenClaims: [ - { - id: `${spec.id}:unsupported`, - anyOf: [`${spec.id} unsupported claim`], - }, - ], - expectedSourceIds: [`${spec.id}:source-1`], - referenceAnswer: `${spec.id} supported answer`, - metadata: { - adapter: spec.adapter, - primaryMetrics: spec.primaryMetrics, - }, - } - }) -} - -export function respondToIndustryRagBenchmarkSmokeCase(input: { - case: KnowledgeBenchmarkCase -}): KnowledgeBenchmarkArtifact { - const testCase = input.case - if (isKnowledgeMemoryBenchmarkCase(testCase)) { - return respondToIndustryMemoryBenchmarkSmokeCase({ case: testCase }) - } - if (testCase.taskKind === 'retrieval') { - const expected = Array.isArray(testCase.expected) ? testCase.expected[0] : testCase.expected - const hit = hitForExpectedTarget(expected, testCase.id) - return { - hits: [hit], - durationMs: 1, - metadata: { - smoke: true, - }, - } - } - - return { - answer: (testCase.requiredClaims ?? []) - .map((claim) => claim.anyOf[0]) - .filter((fragment): fragment is string => Boolean(fragment)) - .join(' '), - citedSourceIds: testCase.expectedSourceIds ?? [], - durationMs: 1, - metadata: { - smoke: true, - }, - } -} - -export function buildIndustryMemoryBenchmarkSmokeCases( - specs: readonly (KnowledgeBenchmarkSpec & { - taskKind: KnowledgeMemoryBenchmarkTaskKind - })[] = INDUSTRY_MEMORY_BENCHMARKS, -): KnowledgeMemoryBenchmarkCase[] { - return specs.map((spec) => { - const currentEventId = `${spec.id}:event-current` - const staleEventId = `${spec.id}:event-stale` - const actorId = spec.taskKind === 'memory-multiparty' ? 'teammate-ada' : 'user' - const currentFact = `${spec.id} current memory` - const staleFact = `${spec.id} stale memory` - return { - id: `${spec.id}/smoke:q1`, - family: spec.family, - taskKind: spec.taskKind, - split: spec.taskKind === 'memory-forgetting' ? 'holdout' : 'dev', - tags: unique(['memory-smoke', spec.id, spec.family, spec.taskKind]), - source: { - name: spec.id, - version: 'smoke', - }, - events: [ - { - id: staleEventId, - actorId, - sessionId: `${spec.id}:session-1`, - timestamp: '2026-01-01T00:00:00.000Z', - text: `${actorId} once had this obsolete fact: ${staleFact}.`, - }, - { - id: currentEventId, - actorId, - sessionId: `${spec.id}:session-2`, - timestamp: '2026-02-01T00:00:00.000Z', - text: `${actorId} updated the durable fact to: ${currentFact}.`, - }, - ], - prompt: `Use memory to answer the ${spec.id} smoke probe.`, - requiredFacts: [ - { - id: `${spec.id}:current`, - anyOf: [currentFact], - sourceEventIds: [currentEventId], - }, - ], - forbiddenFacts: [ - { - id: `${spec.id}:stale`, - anyOf: [staleFact], - sourceEventIds: [staleEventId], - obsolete: true, - }, - ], - expectedEventIds: [currentEventId], - expectedActorIds: [actorId], - referenceAnswer: currentFact, - metadata: { - adapter: spec.adapter, - primaryMetrics: spec.primaryMetrics, - }, - } - }) -} - -export function respondToIndustryMemoryBenchmarkSmokeCase(input: { - case: KnowledgeMemoryBenchmarkCase -}): KnowledgeBenchmarkArtifact { - const testCase = input.case - const facts = testCase.requiredFacts - ?.map((fact) => fact.anyOf[0]) - .filter((fragment): fragment is string => Boolean(fragment)) - return { - answer: facts?.join(' ') ?? '', - rememberedFacts: facts ?? [], - citedEventIds: testCase.expectedEventIds ?? [], - actorIds: testCase.expectedActorIds ?? [], - durationMs: 1, - metadata: { - smoke: true, - }, - } -} - -export function buildFirstPartyMemoryLifecycleBenchmarkCases(): KnowledgeMemoryBenchmarkCase[] { - const base = { - family: 'first-party', - source: { - name: 'first-party/memory-lifecycle', - version: 'real-v1', - }, - } as const - return [ - memoryLifecycleCase({ - ...base, - id: 'first-party/memory-lifecycle:allergy', - taskKind: 'memory-ingest', - split: 'dev', - actorId: 'user', - prompt: 'What food restriction should catering remember for this user?', - staleText: 'The user said they had no food allergies on the first onboarding form.', - currentText: 'The user later corrected the profile: they have a severe peanut allergy.', - required: 'severe peanut allergy', - forbidden: 'no food allergies', - }), - memoryLifecycleCase({ - ...base, - id: 'first-party/memory-lifecycle:account-tier', - taskKind: 'memory-recall', - split: 'dev', - actorId: 'sales-ops', - prompt: 'What is the customer account tier now?', - staleText: 'Sales ops originally marked the customer as starter tier.', - currentText: 'Sales ops updated the customer to enterprise tier after procurement approval.', - required: 'enterprise tier', - forbidden: 'starter tier', - }), - memoryLifecycleCase({ - ...base, - id: 'first-party/memory-lifecycle:launch-date', - taskKind: 'memory-temporal', - split: 'holdout', - actorId: 'pm', - prompt: 'What launch date should the agent use?', - staleText: 'The PM first planned the launch for April 3.', - currentText: 'The PM moved the launch date to April 17 after legal review.', - required: 'April 17', - forbidden: 'April 3', - }), - memoryLifecycleCase({ - ...base, - id: 'first-party/memory-lifecycle:briefing-channel', - taskKind: 'memory-update', - split: 'holdout', - actorId: 'user', - prompt: 'How should daily briefings be delivered now?', - staleText: 'The user used to want daily briefings by SMS.', - currentText: 'The user changed daily briefings to email only.', - required: 'email only', - forbidden: 'SMS', - }), - memoryLifecycleCase({ - ...base, - id: 'first-party/memory-lifecycle:shipping-address', - taskKind: 'memory-forgetting', - split: 'holdout', - actorId: 'user', - prompt: 'What shipping address is current?', - staleText: 'The old shipping address was 14 Pine Street, Apartment 2.', - currentText: 'The current shipping address is 88 Cedar Avenue, Suite 9.', - required: '88 Cedar Avenue, Suite 9', - forbidden: '14 Pine Street', - }), - memoryLifecycleCase({ - ...base, - id: 'first-party/memory-lifecycle:approval-owner', - taskKind: 'memory-reasoning', - split: 'holdout', - actorId: 'finance', - prompt: 'Who owns travel approval now?', - staleText: 'Finance said Liam owned travel approvals last quarter.', - currentText: 'Finance reassigned travel approvals to Maya this quarter.', - required: 'Maya', - forbidden: 'Liam', - }), - memoryLifecycleCase({ - ...base, - id: 'first-party/memory-lifecycle:project-summary', - taskKind: 'memory-summarization', - split: 'dev', - actorId: 'pm', - prompt: 'Summarize the current Project Orion risks.', - staleText: 'The old Project Orion risk was logo color churn.', - currentText: 'The current Project Orion risks are vendor delay and a QA staffing gap.', - required: 'vendor delay', - extraRequired: 'QA staffing gap', - forbidden: 'logo color churn', - }), - memoryLifecycleCase({ - ...base, - id: 'first-party/memory-lifecycle:meeting-format', - taskKind: 'memory-recommendation', - split: 'holdout', - actorId: 'user', - prompt: 'What meeting format should the agent recommend?', - staleText: 'The user previously preferred long video calls for planning.', - currentText: 'The user now prefers async docs for planning instead of video calls.', - required: 'async docs', - forbidden: 'long video calls', - }), - memoryLifecycleCase({ - ...base, - id: 'first-party/memory-lifecycle:multiparty-ada', - taskKind: 'memory-multiparty', - split: 'holdout', - actorId: 'ada', - prompt: 'Which SDK language did Ada ask for?', - staleText: 'Ben asked for a Python notebook example.', - currentText: 'Ada asked for a Rust SDK example.', - required: 'Rust SDK', - forbidden: 'Python notebook', - }), - memoryLifecycleCase({ - ...base, - id: 'first-party/memory-lifecycle:timezone', - taskKind: 'memory-recall', - split: 'dev', - actorId: 'user', - prompt: 'What timezone should scheduling use for this user?', - staleText: 'The user profile originally listed Pacific time.', - currentText: 'The user corrected scheduling to America/Denver time.', - required: 'America/Denver', - forbidden: 'Pacific time', - }), - memoryLifecycleCase({ - ...base, - id: 'first-party/memory-lifecycle:dinner-preference', - taskKind: 'memory-update', - split: 'holdout', - actorId: 'user', - prompt: 'What dinner preference should the assistant use?', - staleText: 'The user was previously vegetarian for team dinners.', - currentText: 'The user updated dinner restrictions to no shellfish.', - required: 'no shellfish', - forbidden: 'vegetarian', - }), - memoryLifecycleCase({ - ...base, - id: 'first-party/memory-lifecycle:support-sla', - taskKind: 'memory-temporal', - split: 'holdout', - actorId: 'support-lead', - prompt: 'What support SLA is current?', - staleText: 'Support used to promise a 24 hour response SLA.', - currentText: 'Support changed the current response SLA to 2 business hours.', - required: '2 business hours', - forbidden: '24 hour', - }), - ] -} - -function createMemoryAdapterBenchmarkResponder(options: { - adapter: AgentMemoryAdapter - candidateId: string - candidateRef: string - storage: CampaignStorage - attemptLogPath: string - lease: OwnedAgentMemoryRunLease - cleanupTimeoutMs: number - searchLimit?: number - scope?: AgentMemoryScope - costUsdPerCase?: number - adapterCreationCostUsd?: number - recoveryCostUsdPerAttempt?: number - now?: () => Date -}): KnowledgeBenchmarkResponder { - assertScopedMemoryBenchmarkAdapter(options.adapter) - return async ({ case: testCase, context: dispatchContext }) => { - if (!isKnowledgeMemoryBenchmarkCase(testCase)) { - return { answer: '', metadata: { candidateId: options.candidateId, skipped: true } } - } - const costUsd = options.costUsdPerCase ?? 0 - if (!Number.isFinite(costUsd) || costUsd < 0) { - throw new Error(`memory adapter costUsdPerCase must be non-negative finite, got ${costUsd}`) - } - - dispatchContext.signal.throwIfAborted() - await options.lease.assertOwned() - const startedAt = Date.now() - const attemptId = randomUUID() - const scope = benchmarkMemoryScope( - options.candidateId, - testCase, - dispatchContext.cellId, - attemptId, - options.scope, - ) - const attempt: MemoryAdapterBenchmarkAttemptEvent = { - schema: 3, - status: 'started', - attemptId, - candidateId: options.candidateId, - candidateRef: options.candidateRef, - adapterId: options.adapter.id, - caseId: testCase.id, - cellId: dispatchContext.cellId, - scope, - adapterCreationCostUsd: options.adapterCreationCostUsd ?? 0, - costUsdPerCase: costUsd, - recoveryCostUsdPerAttempt: options.recoveryCostUsdPerAttempt ?? 0, - recordedAt: (options.now ?? (() => new Date()))().toISOString(), - recovery: false, - } - appendMemoryBenchmarkAttemptEvent(options.storage, options.attemptLogPath, attempt) - - let externalCallAttempted = false - const appendCleanedAttempt = (priorError?: unknown): void => { - try { - appendMemoryBenchmarkAttemptEvent(options.storage, options.attemptLogPath, { - ...attempt, - status: 'cleaned', - recordedAt: (options.now ?? (() => new Date()))().toISOString(), - }) - } catch (error) { - throw new MemoryAdapterBenchmarkCleanupError( - [...(priorError ? [priorError] : []), error], - `${options.candidateId}: memory benchmark cleanup could not be recorded`, - ) - } - } - const execute = async (): Promise => { - dispatchContext.signal.throwIfAborted() - await options.lease.assertOwned() - let artifact: KnowledgeBenchmarkArtifact | undefined - let primaryError: unknown - try { - for (const event of testCase.events) { - dispatchContext.signal.throwIfAborted() - await options.lease.assertOwned() - externalCallAttempted = true - await options.adapter.write({ - id: event.id, - kind: 'message', - text: event.text, - role: event.actorId === 'user' ? 'user' : 'assistant', - title: `${testCase.id}:${event.id}`, - scope, - metadata: compactObject({ - benchmarkCaseId: testCase.id, - benchmarkCellId: dispatchContext.cellId, - benchmarkAttemptId: attemptId, - eventId: event.id, - actorId: event.actorId, - sessionId: event.sessionId, - timestamp: event.timestamp, - ...event.metadata, - }) as Record, - }) - dispatchContext.signal.throwIfAborted() - await options.lease.assertOwned() - } - externalCallAttempted = true - await options.adapter.flush?.() - dispatchContext.signal.throwIfAborted() - await options.lease.assertOwned() - - externalCallAttempted = true - const adapterContext = await options.adapter.getContext(testCase.prompt, { - scope, - limit: options.searchLimit ?? 1, - metadata: { - benchmarkCaseId: testCase.id, - benchmarkCellId: dispatchContext.cellId, - benchmarkAttemptId: attemptId, - candidateId: options.candidateId, - }, - }) - dispatchContext.signal.throwIfAborted() - await options.lease.assertOwned() - const hits = adapterContext.hits - artifact = { - answer: adapterContext.text, - rememberedFacts: hits.map((hit) => hit.text), - citedEventIds: unique(hits.map(memoryEventId).filter((id): id is string => Boolean(id))), - usedMemoryIds: hits.map((hit) => hit.id), - actorIds: unique(hits.map(memoryActorId).filter((id): id is string => Boolean(id))), - costUsd, - durationMs: Math.max(0, Date.now() - startedAt), - metadata: { - candidateId: options.candidateId, - adapterId: options.adapter.id, - hitCount: hits.length, - }, - } - } catch (error) { - primaryError = error - } - - const cleanupErrors: unknown[] = [] - let cleanupOwned = true - try { - await options.lease.assertOwned() - } catch (error) { - cleanupOwned = false - cleanupErrors.push(error) - } - if (cleanupOwned) { - try { - await runBoundedMemoryLifecycle({ - operation: `${options.candidateId}: benchmark attempt flush`, - timeoutMs: options.cleanupTimeoutMs, - resource: options.adapter, - run: () => { - externalCallAttempted = true - return options.adapter.flush?.() - }, - }) - } catch (error) { - cleanupErrors.push(error) - } - try { - await runBoundedMemoryLifecycle({ - operation: `${options.candidateId}: benchmark attempt cleanup`, - timeoutMs: options.cleanupTimeoutMs, - resource: options.adapter, - run: () => { - externalCallAttempted = true - return options.adapter.clear!(scope) - }, - }) - } catch (error) { - cleanupErrors.push(error) - } - } - if (cleanupOwned && cleanupErrors.length === 0) appendCleanedAttempt(primaryError) - if (cleanupErrors.length > 0) { - const errors = [...(primaryError ? [primaryError] : []), ...cleanupErrors] - throw new MemoryAdapterBenchmarkCleanupError( - errors, - `${options.candidateId}: memory benchmark attempt cleanup failed`, - ) - } - if (primaryError) throw primaryError - if (!artifact) - throw new Error(`${options.candidateId}: memory benchmark produced no artifact`) - return artifact - } - - if (costUsd === 0) { - let artifact: KnowledgeBenchmarkArtifact | undefined - let error: unknown - try { - artifact = await execute() - } catch (caught) { - error = caught - } - if (error) throw error - if (!artifact) - throw new Error(`${options.candidateId}: memory benchmark produced no artifact`) - return artifact - } - const receipt = { - model: options.adapter.id, - inputTokens: 0, - outputTokens: 0, - actualCostUsd: costUsd, - } as const - const paid = await dispatchContext.cost.runPaidCall({ - callId: memoryBenchmarkCostCallId(attempt, 'execute', 0), - actor: `agent-knowledge:memory-adapter:${options.adapter.id}`, - model: options.adapter.id, - maximumCharge: { externallyEnforcedMaximumUsd: costUsd }, - execute, - receipt: () => receipt, - receiptFromError: () => ({ - ...receipt, - actualCostUsd: externalCallAttempted ? costUsd : 0, - }), - }) - if (!paid.succeeded) throw paid.error - return paid.value - } -} - -export async function runMemoryAdapterBenchmark( - options: RunMemoryAdapterBenchmarkOptions, -): Promise { - if (options.candidates.length === 0) - throw new Error('memory adapter benchmark requires candidates') - const allCandidates = [...options.candidates, ...(options.recoveryCandidates ?? [])] - assertUniqueNonEmptyStrings( - allCandidates.map((candidate) => candidate.id), - 'memory adapter candidate id', - ) - for (const candidate of allCandidates) { - assertNonEmptyBenchmarkString(candidate.ref, `memory adapter candidate ${candidate.id} ref`) - if (candidate.adapterId !== undefined) { - assertNonEmptyBenchmarkString( - candidate.adapterId, - `memory adapter candidate ${candidate.id} adapterId`, - ) - } - if (!/^[A-Za-z0-9][A-Za-z0-9._-]*$/.test(candidate.id)) { - throw new Error( - `memory adapter candidate id '${candidate.id}' must be a safe directory segment`, - ) - } - if ( - candidate.adapterCreationCostUsd !== undefined && - (!Number.isFinite(candidate.adapterCreationCostUsd) || candidate.adapterCreationCostUsd < 0) - ) { - throw new Error( - `${candidate.id}: adapterCreationCostUsd must be a non-negative finite number`, - ) - } - if ( - candidate.costUsdPerCase !== undefined && - (!Number.isFinite(candidate.costUsdPerCase) || candidate.costUsdPerCase < 0) - ) { - throw new Error(`${candidate.id}: costUsdPerCase must be a non-negative finite number`) - } - if ( - candidate.recoveryCostUsdPerAttempt !== undefined && - (!Number.isFinite(candidate.recoveryCostUsdPerAttempt) || - candidate.recoveryCostUsdPerAttempt < 0) - ) { - throw new Error( - `${candidate.id}: recoveryCostUsdPerAttempt must be a non-negative finite number`, - ) - } - } - const storage = options.storage ?? fsCampaignStorage() - if (!storage.append) { - throw new Error('memory adapter benchmark requires CampaignStorage.append') - } - const runDir = resolveRunDir(options.runDir, options.repo) - const cleanupTimeoutMs = resolveMemoryCleanupTimeoutMs( - options.cleanupTimeoutMs, - 'memory adapter benchmark', - ) - const maxRecoveryAttempts = options.maxRecoveryAttempts ?? 1_000 - if (!Number.isSafeInteger(maxRecoveryAttempts) || maxRecoveryAttempts <= 0) { - throw new Error('memory adapter benchmark maxRecoveryAttempts must be a positive safe integer') - } - const maxRecoveryRetriesPerAttempt = - options.maxRecoveryRetriesPerAttempt ?? DEFAULT_MEMORY_RECOVERY_RETRIES_PER_ATTEMPT - if (!Number.isSafeInteger(maxRecoveryRetriesPerAttempt) || maxRecoveryRetriesPerAttempt <= 0) { - throw new Error( - 'memory adapter benchmark maxRecoveryRetriesPerAttempt must be a positive safe integer', - ) - } - storage.ensureDir(runDir) - const lease = await acquireAgentMemoryRunLease({ - experimentId: `memory-adapter-benchmark:${runDir}`, - runDir, - storage, - customStorage: options.storage !== undefined, - lockFileName: 'memory-adapter-benchmark.lock', - label: 'memory adapter benchmark', - controllerMode: options.controllerMode, - acquireRunLease: options.acquireRunLease, - }) - let result: RunMemoryAdapterBenchmarkResult | undefined - let primaryError: unknown - try { - result = await runOwnedMemoryAdapterBenchmark( - options, - storage, - runDir, - lease, - cleanupTimeoutMs, - maxRecoveryAttempts, - maxRecoveryRetriesPerAttempt, - ) - } catch (error) { - primaryError = error - } - let releaseError: unknown - try { - await lease.release() - } catch (error) { - releaseError = error - } - if (primaryError && releaseError) { - throw new AggregateError( - [primaryError, releaseError], - 'memory adapter benchmark failed and its controller lease could not be released', - ) - } - if (primaryError) throw primaryError - if (releaseError) throw releaseError - if (!result) throw new Error('memory adapter benchmark produced no result') - return result -} - -async function runOwnedMemoryAdapterBenchmark( - options: RunMemoryAdapterBenchmarkOptions, - storage: CampaignStorage, - runDir: string, - lease: OwnedAgentMemoryRunLease, - cleanupTimeoutMs: number, - maxRecoveryAttempts: number, - maxRecoveryRetriesPerAttempt: number, -): Promise { - const maxConcurrency = options.maxConcurrency ?? 2 - if (!Number.isSafeInteger(maxConcurrency) || maxConcurrency <= 0) { - throw new Error('memory adapter benchmark maxConcurrency must be a positive safe integer') - } - const costCeiling = options.costCeiling ?? options.costLedger?.costCeilingUsd ?? 0 - const costLedger = - options.costLedger ?? - createRunCostLedger({ - storage, - runDir, - costCeilingUsd: costCeiling, - }) - if (costLedger.costCeilingUsd !== costCeiling) { - throw new Error( - 'memory adapter benchmark costCeiling must match the shared cost ledger ceiling', - ) - } - const attemptLogPath = join(runDir, 'memory-adapter-attempts.jsonl') - const recoveryLogPath = join(runDir, 'memory-adapter-recovery-attempts.jsonl') - await recoverMemoryAdapterBenchmarkAttempts({ - candidates: [...options.candidates, ...(options.recoveryCandidates ?? [])], - storage, - attemptLogPath, - lease, - cleanupTimeoutMs, - maxConcurrency, - now: options.now, - runDir, - costLedger, - costPhase: options.costPhase ?? 'memory.adapter-benchmark', - maxRecoveryAttempts, - recoveryLogPath, - maxRecoveryRetriesPerAttempt, - }) - await lease.assertOwned() - const rows: MemoryAdapterBenchmarkRankingRow[] = [] - for (const candidate of options.candidates) { - await lease.assertOwned() - const expectedAdapterId = memoryAdapterBenchmarkExpectedId(candidate) - let adapter: AgentMemoryAdapter | undefined - let adapterPromise: Promise | undefined - let adapterCreationError: unknown - const getAdapter = (): Promise => { - if (!adapterPromise) { - const abortController = new AbortController() - const creation = createMemoryAdapterBenchmarkAdapter({ - candidate, - purpose: 'execute', - signal: abortController.signal, - costLedger, - runDir, - costPhase: options.costPhase ?? 'memory.adapter-benchmark', - }) - releaseMemoryAdapterCreatedAfterAbort({ creation, signal: abortController.signal }) - adapterPromise = runBoundedMemoryLifecycle({ - operation: `${candidate.id}: benchmark execute adapter creation`, - timeoutMs: Math.min(cleanupTimeoutMs, options.dispatchTimeoutMs ?? cleanupTimeoutMs), - abortController, - run: () => creation, - }) - .then((created) => { - adapter = created - if (created.id !== expectedAdapterId) { - throw new Error( - `${candidate.id}: createAdapter returned id '${created.id}', expected '${expectedAdapterId}'`, - ) - } - assertScopedMemoryBenchmarkAdapter(created) - return created - }) - .catch((error) => { - adapterCreationError = error - throw error - }) - } - return adapterPromise - } - const dispatchedExecutions: Promise[] = [] - let run: RunKnowledgeBenchmarkSuiteResult | undefined - let primaryError: unknown - try { - await lease.assertOwned() - let respond: KnowledgeBenchmarkResponder | undefined - run = await runKnowledgeBenchmarkSuite({ - cases: options.cases, - respond(input) { - const operation = getAdapter().then((activeAdapter) => { - respond ??= createMemoryAdapterBenchmarkResponder({ - adapter: activeAdapter, - candidateId: candidate.id, - candidateRef: candidate.ref, - storage, - attemptLogPath, - lease, - cleanupTimeoutMs, - searchLimit: candidate.searchLimit, - scope: candidate.scope, - adapterCreationCostUsd: candidate.adapterCreationCostUsd, - costUsdPerCase: candidate.costUsdPerCase, - recoveryCostUsdPerAttempt: candidate.recoveryCostUsdPerAttempt, - now: options.now, - }) - return respond(input) - }) - dispatchedExecutions.push(operation) - return operation - }, - respondRef: stableId( - 'memory_adapter_benchmark', - canonicalJson({ - implementationRef: MEMORY_ADAPTER_BENCHMARK_IMPLEMENTATION_REF, - candidateRef: candidate.ref, - adapterId: expectedAdapterId, - searchLimit: candidate.searchLimit ?? null, - adapterCreationCostUsd: candidate.adapterCreationCostUsd ?? 0, - costUsdPerCase: candidate.costUsdPerCase ?? 0, - recoveryCostUsdPerAttempt: candidate.recoveryCostUsdPerAttempt ?? 0, - scope: candidate.scope ?? null, - }), - ), - runDir: join(runDir, candidate.id), - storage, - seed: options.seed, - reps: options.reps, - resumable: options.resumable, - costCeiling, - costLedger, - costPhase: `${options.costPhase ?? 'memory.adapter-benchmark'}.${candidate.id}`, - maxConcurrency: options.maxConcurrency, - dispatchTimeoutMs: options.dispatchTimeoutMs, - expectUsage: options.expectUsage ?? 'off', - now: options.now, - }) - } catch (error) { - primaryError = error - } - - const settledExecutions = await Promise.allSettled(dispatchedExecutions) - const dispatchCleanupErrors = settledExecutions.flatMap((settled) => - settled.status === 'rejected' && settled.reason instanceof MemoryAdapterBenchmarkCleanupError - ? [settled.reason] - : [], - ) - if (!primaryError && adapterCreationError) primaryError = adapterCreationError - if (run) { - rows.push({ - rank: 0, - candidateId: candidate.id, - label: candidate.label ?? candidate.id, - adapterId: expectedAdapterId, - scoreMean: run.report.score.mean, - passRate: run.report.dimensions.passed?.mean ?? 0, - totalCases: run.report.totalCases, - totalCells: run.report.totalCells, - cellsFailed: run.report.cellsFailed, - totalCostUsd: run.report.totalCostUsd, - reportJsonPath: run.reportJsonPath, - reportMarkdownPath: run.reportMarkdownPath, - report: run.report, - }) - } - const cleanupErrors: unknown[] = [] - let cleanupOwned = true - try { - await lease.assertOwned() - } catch (error) { - cleanupOwned = false - cleanupErrors.push(error) - } - if (cleanupOwned && adapter && !adapterCreationError) { - const activeAdapter = adapter - try { - await runBoundedMemoryLifecycle({ - operation: `${candidate.id}: benchmark adapter flush`, - timeoutMs: cleanupTimeoutMs, - resource: activeAdapter, - run: () => activeAdapter.flush?.(), - }) - } catch (error) { - cleanupErrors.push(error) - } - } - if (adapter) { - try { - await runBoundedMemoryLifecycle({ - operation: `${candidate.id}: benchmark adapter close`, - timeoutMs: cleanupTimeoutMs, - resource: adapter, - run: () => adapter!.close?.(), - }) - } catch (error) { - cleanupErrors.push(error) - } - } - if (primaryError || dispatchCleanupErrors.length > 0 || cleanupErrors.length > 0) { - const errors = [ - ...(primaryError ? [primaryError] : []), - ...dispatchCleanupErrors, - ...cleanupErrors, - ] - if (errors.length === 1) throw errors[0] - throw new AggregateError(errors, `${candidate.id}: memory adapter benchmark cleanup failed`) - } - await lease.assertOwned() - } - - const costByCandidate = memoryAdapterBenchmarkCostByCandidate(costLedger, runDir, [ - ...options.candidates, - ...(options.recoveryCandidates ?? []), - ]) - const ranked = rows - .map((row) => { - const totalCostUsd = normalizeUsd(costByCandidate.get(row.candidateId) ?? 0) - return { - ...row, - totalCostUsd, - } - }) - .sort( - (a, b) => - Number(a.cellsFailed > 0) - Number(b.cellsFailed > 0) || - b.scoreMean - a.scoreMean || - b.passRate - a.passRate || - a.totalCostUsd - b.totalCostUsd || - a.candidateId.localeCompare(b.candidateId), - ) - .map((row, index) => ({ ...row, rank: index + 1 })) - const rankingJsonPath = join(runDir, 'memory-adapter-ranking.json') - const rankingMarkdownPath = join(runDir, 'memory-adapter-ranking.md') - const unrankedRecoveryCostUsd = normalizeUsd( - (options.recoveryCandidates ?? []).reduce( - (sum, candidate) => sum + (costByCandidate.get(candidate.id) ?? 0), - 0, - ), - ) - const totalCostUsd = normalizeUsd( - [...costByCandidate.values()].reduce((sum, cost) => sum + cost, 0), - ) - storage.write( - rankingJsonPath, - `${JSON.stringify({ totalCostUsd, unrankedRecoveryCostUsd, rows: ranked }, null, 2)}\n`, - ) - storage.write( - rankingMarkdownPath, - renderMemoryAdapterRankingMarkdown(ranked, totalCostUsd, unrankedRecoveryCostUsd), - ) - return { - rows: ranked, - totalCostUsd, - unrankedRecoveryCostUsd, - rankingJsonPath, - rankingMarkdownPath, - attemptLogPath, - recoveryLogPath, - } -} - -async function createMemoryAdapterBenchmarkAdapter(input: { - candidate: MemoryAdapterBenchmarkCandidate - purpose: 'execute' | 'recovery' - signal: AbortSignal - costLedger: CostLedgerHandle - runDir: string - costPhase: string -}): Promise { - const { candidate, purpose, signal, costLedger, runDir, costPhase } = input - const costUsd = candidate.adapterCreationCostUsd ?? 0 - let externalCallAttempted = false - const create = async (): Promise => { - const adapter = await candidate.createAdapter({ - purpose, - signal, - markExternalCall: () => { - externalCallAttempted = true - }, - }) - if (!adapter || typeof adapter !== 'object') { - throw new Error(`${candidate.id}: createAdapter returned no ${purpose} adapter`) - } - return adapter - } - if (costUsd === 0) return create() - - const tags = memoryAdapterCreationCostTags(runDir, candidate.id, purpose) - const generation = costLedger.list({ tags }).length - const receipt = { - model: candidate.id, - inputTokens: 0, - outputTokens: 0, - actualCostUsd: costUsd, - } as const - const paid = await costLedger.runPaidCall({ - callId: memoryAdapterCreationCostCallId(candidate, purpose, generation), - channel: 'driver', - phase: `${costPhase}.${candidate.id}.adapter-${purpose}`, - actor: `agent-knowledge:memory-adapter:${candidate.id}`, - model: candidate.id, - tags, - maximumCharge: { externallyEnforcedMaximumUsd: costUsd }, - execute: create, - receipt: () => ({ - ...receipt, - actualCostUsd: externalCallAttempted ? costUsd : 0, - }), - receiptFromError: () => ({ - ...receipt, - actualCostUsd: externalCallAttempted ? costUsd : 0, - }), - }) - if (!paid.succeeded) throw paid.error - return paid.value -} - -function memoryAdapterCreationCostTags( - runDir: string, - candidateId: string, - purpose: 'execute' | 'recovery', -): Record { - return { - runDir: join(runDir, candidateId), - candidateId, - memoryAdapterCreation: purpose, - } -} - -function memoryAdapterCreationCostCallId( - candidate: MemoryAdapterBenchmarkCandidate, - purpose: 'execute' | 'recovery', - generation: number, -): string { - return stableId( - 'memory_adapter_creation_cost_call', - canonicalJson({ - purpose, - generation, - candidateId: candidate.id, - candidateRef: candidate.ref, - adapterCreationCostUsd: candidate.adapterCreationCostUsd ?? 0, - }), - ) -} - -function memoryAdapterBenchmarkExpectedId(candidate: MemoryAdapterBenchmarkCandidate): string { - return candidate.adapterId ?? candidate.id -} - -function memoryAdapterBenchmarkCostByCandidate( - costLedger: CostLedgerHandle, - runDir: string, - candidates: readonly MemoryAdapterBenchmarkCandidate[], -): ReadonlyMap { - const candidateByRunDir = new Map( - candidates.map((candidate) => [join(runDir, candidate.id), candidate.id]), - ) - const totals = new Map() - for (const receipt of costLedger.list()) { - const candidateId = receipt.tags?.runDir - ? candidateByRunDir.get(receipt.tags.runDir) - : undefined - if (!candidateId) continue - totals.set(candidateId, (totals.get(candidateId) ?? 0) + receipt.costUsd) - } - return totals -} - -async function recoverMemoryAdapterBenchmarkAttempts(input: { - candidates: readonly MemoryAdapterBenchmarkCandidate[] - storage: CampaignStorage - attemptLogPath: string - lease: OwnedAgentMemoryRunLease - cleanupTimeoutMs: number - maxConcurrency: number - now?: () => Date - runDir: string - costLedger: CostLedgerHandle - costPhase: string - maxRecoveryAttempts: number - recoveryLogPath: string - maxRecoveryRetriesPerAttempt: number -}): Promise { - let attempts = readActiveMemoryBenchmarkAttempts(input.storage, input.attemptLogPath) - if (attempts.length > input.maxRecoveryAttempts) { - throw new Error( - `memory adapter benchmark has ${attempts.length} unfinished attempts; maxRecoveryAttempts is ${input.maxRecoveryAttempts}`, - ) - } - const candidateById = new Map(input.candidates.map((candidate) => [candidate.id, candidate])) - for (const attempt of attempts) { - const candidate = candidateById.get(attempt.candidateId) - if (!candidate) { - throw new Error( - `cannot recover memory benchmark attempts: candidate '${attempt.candidateId}' is missing; pass it in recoveryCandidates`, - ) - } - assertMemoryBenchmarkAttemptCandidateMatches(attempt, candidate) - } - - reconcileInterruptedMemoryPaidCalls(input.costLedger) - assertNoInterruptedPaidCalls(input.costLedger, 'memory adapter benchmark recovery') - - for (const attempt of attempts) { - const candidate = candidateById.get(attempt.candidateId)! - const costUsd = candidate.costUsdPerCase ?? 0 - if ( - costUsd > 0 && - !hasSettledPaidCall(input.costLedger, memoryBenchmarkCostCallId(attempt, 'execute', 0)) - ) { - appendMemoryBenchmarkAttemptEvent(input.storage, input.attemptLogPath, { - ...attempt, - status: 'cleaned', - recovery: true, - recordedAt: (input.now ?? (() => new Date()))().toISOString(), - }) - } - } - attempts = readActiveMemoryBenchmarkAttempts(input.storage, input.attemptLogPath) - const recoveryGenerations = reserveRecoveryAttempts({ - storage: input.storage, - path: input.recoveryLogPath, - attemptIds: attempts.map((attempt) => attempt.attemptId), - maxRetriesPerAttempt: input.maxRecoveryRetriesPerAttempt, - label: 'memory benchmark recovery attempt log', - now: input.now, - }) - const grouped = groupMemoryBenchmarkAttempts(attempts) - const pool = createMemoryExecutionPool(input.maxConcurrency) - const settled = await Promise.allSettled( - [...grouped] - .sort(([left], [right]) => left.localeCompare(right)) - .map(([candidateId, candidateAttempts]) => - pool.run(async () => { - await input.lease.assertOwned() - const candidate = candidateById.get(candidateId) - if (!candidate) { - throw new Error( - `cannot recover memory benchmark attempts: candidate '${candidateId}' is missing; pass it in recoveryCandidates`, - ) - } - for (const attempt of candidateAttempts) { - assertMemoryBenchmarkAttemptCandidateMatches(attempt, candidate) - if (recoveryGenerations.get(attempt.attemptId) === undefined) { - throw new Error( - `missing recovery generation for memory benchmark attempt '${attempt.attemptId}'`, - ) - } - } - - let recoveryAttemptsStarted = 0 - let adapterCreationExternalCallAttempted = false - const cleared: MemoryAdapterBenchmarkAttemptEvent[] = [] - const recover = async (): Promise => { - let adapter: AgentMemoryAdapter | undefined - let primaryError: unknown - try { - const abortController = new AbortController() - const creation = Promise.resolve().then(() => - candidate.createAdapter({ - purpose: 'recovery', - signal: abortController.signal, - markExternalCall: () => { - adapterCreationExternalCallAttempted = true - }, - }), - ) - releaseMemoryAdapterCreatedAfterAbort({ - creation, - signal: abortController.signal, - }) - adapter = await runBoundedMemoryLifecycle({ - operation: `${candidate.id}: benchmark recovery adapter creation`, - timeoutMs: input.cleanupTimeoutMs, - abortController, - run: () => creation, - }) - assertScopedMemoryBenchmarkAdapter(adapter) - for (const attempt of candidateAttempts) { - if (adapter.id !== attempt.adapterId) { - throw new Error( - `cannot recover memory benchmark attempt '${attempt.attemptId}': adapter changed from '${attempt.adapterId}' to '${adapter.id}'`, - ) - } - } - await sleepForMemoryRecovery( - memoryRecoveryDelayMs(adapter), - () => input.lease.assertOwned(), - input.cleanupTimeoutMs, - `${candidate.id}: benchmark recovery visibility wait`, - ) - for (const attempt of candidateAttempts.sort((left, right) => - left.attemptId.localeCompare(right.attemptId), - )) { - await input.lease.assertOwned() - try { - recoveryAttemptsStarted += 1 - await runBoundedMemoryLifecycle({ - operation: `${candidate.id}: abandoned benchmark attempt cleanup`, - timeoutMs: input.cleanupTimeoutMs, - resource: adapter, - run: () => adapter!.clear!(attempt.scope), - }) - await input.lease.assertOwned() - cleared.push(attempt) - } catch (error) { - primaryError = primaryError - ? new AggregateError( - [primaryError, error], - `${candidate.id}: multiple benchmark attempts failed recovery`, - ) - : error - } - } - } catch (error) { - primaryError = primaryError - ? new AggregateError( - [primaryError, error], - `${candidate.id}: benchmark recovery failed in multiple operations`, - ) - : error - } - - let closeError: unknown - if (adapter) { - try { - await runBoundedMemoryLifecycle({ - operation: `${candidate.id}: benchmark recovery adapter close`, - timeoutMs: input.cleanupTimeoutMs, - resource: adapter, - run: () => adapter!.close?.(), - }) - } catch (error) { - closeError = error - } - } - let journalError: unknown - if (!closeError) { - try { - for (const attempt of cleared) { - await input.lease.assertOwned() - appendMemoryBenchmarkAttemptEvent(input.storage, input.attemptLogPath, { - ...attempt, - status: 'cleaned', - recovery: true, - recordedAt: (input.now ?? (() => new Date()))().toISOString(), - }) - } - } catch (error) { - journalError = error - } - } - const failures = [primaryError, closeError, journalError].filter( - (error) => error !== undefined, - ) - if (failures.length > 1) { - throw new AggregateError( - failures, - `${candidate.id}: benchmark attempt recovery, adapter close, or cleanup journal failed`, - ) - } - if (primaryError) throw primaryError - if (closeError) throw closeError - if (journalError) throw journalError - } - - const recoveryCostUsd = candidate.recoveryCostUsdPerAttempt ?? 0 - const adapterCreationCostUsd = candidate.adapterCreationCostUsd ?? 0 - const maximumCostUsd = adapterCreationCostUsd + recoveryCostUsd * candidateAttempts.length - const actualCostUsd = (): number => - (adapterCreationExternalCallAttempted ? adapterCreationCostUsd : 0) + - recoveryCostUsd * recoveryAttemptsStarted - let recoveryError: unknown - if (maximumCostUsd === 0) { - try { - await recover() - } catch (error) { - recoveryError = error - } - } else { - const receipt = { - model: candidate.id, - inputTokens: 0, - outputTokens: 0, - } as const - const tags = memoryBenchmarkRecoveryCostTags(input.runDir, candidate.id) - const paid = await input.costLedger.runPaidCall({ - callId: memoryBenchmarkRecoveryCostCallId( - candidate, - candidateAttempts, - recoveryGenerations, - ), - channel: 'driver', - phase: `${input.costPhase}.${candidate.id}.recovery`, - actor: `agent-knowledge:memory-adapter-recovery:${candidate.id}`, - model: candidate.id, - tags, - maximumCharge: { externallyEnforcedMaximumUsd: maximumCostUsd }, - execute: recover, - receipt: () => ({ ...receipt, actualCostUsd: actualCostUsd() }), - receiptFromError: () => ({ - ...receipt, - actualCostUsd: actualCostUsd(), - }), - }) - if (!paid.succeeded) recoveryError = paid.error - } - if (recoveryError) throw recoveryError - }), - ), - ) - const failures = settled.flatMap((result) => - result.status === 'rejected' ? [result.reason] : [], - ) - if (failures.length === 1) throw failures[0] - if (failures.length > 1) { - throw new AggregateError(failures, 'multiple memory benchmark candidates failed recovery') - } -} - -function groupMemoryBenchmarkAttempts( - attempts: readonly MemoryAdapterBenchmarkAttemptEvent[], -): Map { - const grouped = new Map() - for (const attempt of attempts) { - const group = grouped.get(attempt.candidateId) ?? [] - group.push(attempt) - grouped.set(attempt.candidateId, group) - } - return grouped -} - -function memoryBenchmarkCostCallId( - attempt: MemoryAdapterBenchmarkAttemptEvent, - purpose: 'execute' | 'recovery', - generation: number, -): string { - return stableId( - 'memory_benchmark_cost_call', - canonicalJson({ - purpose, - generation, - attemptId: attempt.attemptId, - candidateId: attempt.candidateId, - candidateRef: attempt.candidateRef, - adapterId: attempt.adapterId, - adapterCreationCostUsd: attempt.adapterCreationCostUsd, - costUsdPerCase: attempt.costUsdPerCase, - recoveryCostUsdPerAttempt: attempt.recoveryCostUsdPerAttempt, - caseId: attempt.caseId, - cellId: attempt.cellId, - }), - ) -} - -function memoryBenchmarkRecoveryCostCallId( - candidate: MemoryAdapterBenchmarkCandidate, - attempts: readonly MemoryAdapterBenchmarkAttemptEvent[], - generations: ReadonlyMap, -): string { - return stableId( - 'memory_benchmark_recovery_cost_call', - canonicalJson({ - candidateId: candidate.id, - candidateRef: candidate.ref, - adapterCreationCostUsd: candidate.adapterCreationCostUsd ?? 0, - recoveryCostUsdPerAttempt: candidate.recoveryCostUsdPerAttempt ?? 0, - attempts: attempts - .map((attempt) => ({ - attemptId: attempt.attemptId, - generation: generations.get(attempt.attemptId), - })) - .sort((left, right) => left.attemptId.localeCompare(right.attemptId)), - }), - ) -} - -function memoryBenchmarkRecoveryCostTags( - runDir: string, - candidateId: string, -): Record { - return { - runDir: join(runDir, candidateId), - candidateId, - memoryRecovery: 'benchmark', - } -} - -function appendMemoryBenchmarkAttemptEvent( - storage: CampaignStorage, - path: string, - event: MemoryAdapterBenchmarkAttemptEvent, -): void { - appendAttemptJournalEvent({ - storage, - path, - event, - label: 'memory benchmark attempt log', - }) -} - -function readActiveMemoryBenchmarkAttempts( - storage: CampaignStorage, - path: string, -): MemoryAdapterBenchmarkAttemptEvent[] { - return readActiveAttemptJournal({ - storage, - path, - label: 'memory benchmark attempt log', - parse: parseMemoryBenchmarkAttemptEvent, - id: (event) => event.attemptId, - sameAttempt: sameMemoryBenchmarkAttempt, - }) -} - -function parseMemoryBenchmarkAttemptEvent( - value: unknown, - path: string, - line: number, -): MemoryAdapterBenchmarkAttemptEvent { - const event = value as Partial | null - const valid = - typeof event === 'object' && - event !== null && - event.schema === 3 && - (event.status === 'started' || event.status === 'cleaned') && - isNonEmptyString(event.attemptId) && - isNonEmptyString(event.candidateId) && - isNonEmptyString(event.candidateRef) && - isNonEmptyString(event.adapterId) && - isNonEmptyString(event.caseId) && - isNonEmptyString(event.cellId) && - isAgentMemoryScope(event.scope) && - typeof event.adapterCreationCostUsd === 'number' && - Number.isFinite(event.adapterCreationCostUsd) && - event.adapterCreationCostUsd >= 0 && - typeof event.costUsdPerCase === 'number' && - Number.isFinite(event.costUsdPerCase) && - event.costUsdPerCase >= 0 && - typeof event.recoveryCostUsdPerAttempt === 'number' && - Number.isFinite(event.recoveryCostUsdPerAttempt) && - event.recoveryCostUsdPerAttempt >= 0 && - typeof event.recordedAt === 'string' && - !Number.isNaN(Date.parse(event.recordedAt)) && - typeof event.recovery === 'boolean' - if (!valid) { - throw new Error(`invalid memory benchmark attempt event in '${path}' line ${line}`) - } - return value as MemoryAdapterBenchmarkAttemptEvent -} - -function sameMemoryBenchmarkAttempt( - left: MemoryAdapterBenchmarkAttemptEvent, - right: MemoryAdapterBenchmarkAttemptEvent, -): boolean { - return ( - left.attemptId === right.attemptId && - left.candidateId === right.candidateId && - left.candidateRef === right.candidateRef && - left.adapterId === right.adapterId && - left.caseId === right.caseId && - left.cellId === right.cellId && - canonicalJson(left.scope) === canonicalJson(right.scope) && - left.adapterCreationCostUsd === right.adapterCreationCostUsd && - left.costUsdPerCase === right.costUsdPerCase && - left.recoveryCostUsdPerAttempt === right.recoveryCostUsdPerAttempt - ) -} - -function assertMemoryBenchmarkAttemptCandidateMatches( - attempt: MemoryAdapterBenchmarkAttemptEvent, - candidate: MemoryAdapterBenchmarkCandidate, -): void { - if (attempt.candidateRef !== candidate.ref) { - throw new Error( - `cannot recover memory benchmark attempt '${attempt.attemptId}': candidate ref changed from '${attempt.candidateRef}' to '${candidate.ref}'`, - ) - } - const expectedAdapterId = memoryAdapterBenchmarkExpectedId(candidate) - if (attempt.adapterId !== expectedAdapterId) { - throw new Error( - `cannot recover memory benchmark attempt '${attempt.attemptId}': adapter id changed from '${attempt.adapterId}' to '${expectedAdapterId}'`, - ) - } - const executionCost = candidate.costUsdPerCase ?? 0 - const adapterCreationCost = candidate.adapterCreationCostUsd ?? 0 - const recoveryCost = candidate.recoveryCostUsdPerAttempt ?? 0 - if ( - adapterCreationCost !== attempt.adapterCreationCostUsd || - executionCost !== attempt.costUsdPerCase || - recoveryCost !== attempt.recoveryCostUsdPerAttempt - ) { - throw new Error( - `cannot recover memory benchmark attempt '${attempt.attemptId}': candidate cost settings changed; start a new run or restore the recorded costs`, - ) - } -} - -function isAgentMemoryScope(value: unknown): value is AgentMemoryScope { - if (!isRecordValue(value)) return false - const allowed = new Set([ - 'tenantId', - 'userId', - 'agentId', - 'teamId', - 'runId', - 'sessionId', - 'namespace', - 'tags', - ]) - if (Object.keys(value).some((key) => !allowed.has(key))) return false - for (const key of [ - 'tenantId', - 'userId', - 'agentId', - 'teamId', - 'runId', - 'sessionId', - 'namespace', - ]) { - if (value[key] !== undefined && typeof value[key] !== 'string') return false - } - if (value.tags === undefined) return true - return ( - isRecordValue(value.tags) && - Object.values(value.tags).every((entry) => typeof entry === 'string') - ) -} - -function isNonEmptyString(value: unknown): value is string { - return typeof value === 'string' && value.trim().length > 0 -} - -function isRecordValue(value: unknown): value is Record { - return typeof value === 'object' && value !== null && !Array.isArray(value) -} - -export function createNoopMemoryBenchmarkAdapter(id = 'no-memory'): AgentMemoryAdapter { - return { - id, - branchIsolation: { mode: 'scoped' }, - async search() { - return [] - }, - async getContext(query) { - return { query, text: '', hits: [], sourceRecords: [] } - }, - async write(input) { - return { - accepted: false, - id: input.id ?? `${id}:ignored`, - uri: `memory://${id}/ignored`, - kind: input.kind, - } - }, - async clear() {}, - async flush() {}, - } -} - -export function createInMemoryBenchmarkAdapter(options: { id?: string } = {}): AgentMemoryAdapter { - const id = options.id ?? 'in-memory' - const rows: Array<{ - seq: number - input: AgentMemoryWriteInput - hit: AgentMemoryHit - }> = [] - let seq = 0 - const adapter: AgentMemoryAdapter = { - id, - branchIsolation: { mode: 'scoped' }, - async search(query, searchOptions = {}) { - const scored = rows - .filter((row) => memoryScopeMatches(row.input.scope, searchOptions.scope)) - .filter((row) => !searchOptions.kinds?.length || searchOptions.kinds.includes(row.hit.kind)) - .map((row) => { - const lexical = tokenOverlap(query, row.hit.text) - const recency = row.seq / Math.max(1, seq) - return { - ...row.hit, - score: lexical + recency * 0.01, - normalizedScore: lexical, - } - }) - .filter((hit) => - searchOptions.minScore === undefined ? true : hit.score! >= searchOptions.minScore, - ) - .sort((a, b) => (b.score ?? 0) - (a.score ?? 0)) - return scored.slice(0, searchOptions.limit ?? 5) - }, - async getContext(query, searchOptions = {}) { - const hits = await adapter.search(query, searchOptions) - return { - query, - hits, - sourceRecords: hits.map((hit) => - memoryHitToSourceRecord(hit, { scope: searchOptions.scope }), - ), - text: renderMemoryHits(hits), - } - }, - async write(input) { - seq += 1 - const memoryId = input.id ?? `${id}:${seq}` - const hit: AgentMemoryHit = { - id: memoryId, - uri: `memory://${id}/${encodeURIComponent(memoryId)}`, - kind: input.kind, - text: input.text, - title: input.title, - score: 1, - normalizedScore: 1, - createdAt: input.metadata?.timestamp as string | undefined, - metadata: { - ...(input.metadata ?? {}), - scope: input.scope, - }, - } - rows.push({ seq, input, hit }) - return { - accepted: true, - id: memoryId, - uri: hit.uri, - kind: input.kind, - sourceRecord: memoryWriteResultToSourceRecord( - { - accepted: true, - id: memoryId, - uri: hit.uri, - kind: input.kind, - metadata: hit.metadata, - }, - input.text, - { scope: input.scope }, - ), - metadata: hit.metadata, - } - }, - async clear(scope) { - for (let index = rows.length - 1; index >= 0; index -= 1) { - if (memoryScopeMatches(rows[index]!.input.scope, scope)) rows.splice(index, 1) - } - }, - async flush() {}, - } - return adapter -} - -export function parseKnowledgeBenchmarkJsonl(text: string): T[] { - return text - .split(/\r?\n/) - .map((line) => line.trim()) - .filter(Boolean) - .map((line, index) => { - try { - return JSON.parse(line) as T - } catch (error) { - throw new Error(`invalid JSONL row ${index + 1}: ${(error as Error).message}`) - } - }) -} - -export function parseKnowledgeBenchmarkQrels(text: string): KnowledgeRetrievalBenchmarkQrel[] { - return text - .split(/\r?\n/) - .map((line) => line.trim()) - .filter((line) => line && !line.startsWith('#')) - .flatMap((line, index) => { - const parts = line.split(/\t|\s+/) - if (parts.length < 3) return [] - const [queryId, maybeZeroOrDocId, maybeDocIdOrScore, maybeScore] = parts - if (!queryId || !maybeZeroOrDocId || !maybeDocIdOrScore) return [] - if (queryId.toLowerCase() === 'qid' || queryId.toLowerCase() === 'query-id') return [] - const documentId = maybeScore === undefined ? maybeZeroOrDocId : maybeDocIdOrScore - const scoreText = maybeScore === undefined ? maybeDocIdOrScore : maybeScore - const score = Number(scoreText) - if (!documentId || !Number.isFinite(score)) { - throw new Error(`invalid qrels row ${index + 1}: expected query id, doc id, score`) - } - return [{ queryId, documentId, score }] - }) -} - -export function buildRetrievalBenchmarkCasesFromQrels( - options: BuildRetrievalBenchmarkCasesFromQrelsOptions, -): KnowledgeRetrievalBenchmarkCase[] { - const qrelsByQuery = new Map() - for (const qrel of options.qrels) { - if (qrel.score <= 0) continue - const list = qrelsByQuery.get(qrel.queryId) ?? [] - list.push(qrel) - qrelsByQuery.set(qrel.queryId, list) - } - - return options.queries.flatMap((query) => { - const qrels = qrelsByQuery.get(query.id) ?? [] - if (qrels.length === 0) return [] - const split = query.split ?? options.splitOf?.(query.id) - const expected = qrels.map((qrel) => - options.documentTarget - ? options.documentTarget(qrel.documentId, qrel) - : defaultDocumentTarget(qrel.documentId, options.targetKind ?? 'page'), - ) - return [ - compactObject({ - id: `${options.benchmarkId}:${query.id}`, - family: options.family, - taskKind: 'retrieval' as const, - query: query.text, - expected, - k: options.k, - split, - tags: unique([...(options.tags ?? []), ...(query.tags ?? []), ...(split ? [split] : [])]), - source: options.source, - metadata: query.metadata, - }) as KnowledgeRetrievalBenchmarkCase, - ] - }) -} - -export async function runKnowledgeBenchmarkSuite( - options: RunKnowledgeBenchmarkSuiteOptions, -): Promise> { - assertKnowledgeBenchmarkCases(options.cases) - if (options.respondRef !== undefined) { - assertNonEmptyBenchmarkString(options.respondRef, 'knowledge benchmark respondRef') - } else if (options.resumable !== false) { - throw new Error('knowledge benchmark respondRef is required when resumable is enabled') - } - const storage = options.storage ?? fsCampaignStorage() - const costCeiling = options.costCeiling ?? options.costLedger?.costCeilingUsd ?? 0 - if (options.costLedger && options.costLedger.costCeilingUsd !== costCeiling) { - throw new Error('knowledge benchmark costCeiling must match the shared cost ledger ceiling') - } - const scenarios = buildKnowledgeBenchmarkScenarios(options.cases, options.splits) - const dispatch: RunCampaignOptions['dispatch'] = async ( - scenario, - context, - ) => { - const artifact = await options.respond({ case: scenario.case, scenario, context }) - return artifact - } - const campaign = await runCampaign({ - scenarios, - dispatch, - dispatchRef: stableId( - 'knowledge_benchmark', - canonicalJson({ - implementationRef: KNOWLEDGE_BENCHMARK_IMPLEMENTATION_REF, - respondRef: options.respondRef ?? 'non-resumable', - }), - ), - judges: [knowledgeBenchmarkJudge()], - runDir: options.runDir, - repo: options.repo, - seed: options.seed, - reps: options.reps, - resumable: options.resumable, - costCeiling, - costLedger: options.costLedger, - costPhase: options.costPhase, - maxConcurrency: options.maxConcurrency, - dispatchTimeoutMs: options.dispatchTimeoutMs, - expectUsage: options.expectUsage ?? 'off', - storage, - now: options.now, - }) - const report = summarizeKnowledgeBenchmarkCampaign({ scenarios, campaign }) - const reportJsonPath = join(campaign.runDir, 'knowledge-benchmark-report.json') - const reportMarkdownPath = join(campaign.runDir, 'knowledge-benchmark-report.md') - storage.write(reportJsonPath, `${JSON.stringify(report, null, 2)}\n`) - storage.write(reportMarkdownPath, renderKnowledgeBenchmarkReportMarkdown(report)) - return { - scenarios, - campaign, - report, - reportJsonPath, - reportMarkdownPath, - } -} - -export function renderKnowledgeBenchmarkReportMarkdown(report: KnowledgeBenchmarkReport): string { - return [ - '# Knowledge Benchmark Report', - '', - `- cases: ${report.totalCases}`, - `- cells: ${report.totalCells} total, ${report.cellsFailed} failed, ${report.cellsCached} cached`, - `- cost: $${formatNumber(report.totalCostUsd)}`, - `- score: mean ${formatNumber(report.score.mean)}, median ${formatNumber(report.score.median)}, p90 ${formatNumber(report.score.p90)}, n=${report.score.n}`, - '', - '## Task Kinds', - '', - renderSliceTable(report.byTaskKind), - '', - '## Splits', - '', - renderSliceTable(report.bySplit), - '', - '## Dimensions', - '', - '| dimension | n | mean | p90 |', - '| --- | ---: | ---: | ---: |', - ...Object.entries(report.dimensions) - .sort(([a], [b]) => a.localeCompare(b)) - .map( - ([key, dist]) => - `| ${key} | ${dist.n} | ${formatNumber(dist.mean)} | ${formatNumber(dist.p90)} |`, - ), - '', - ].join('\n') -} - -export function buildKnowledgeBenchmarkScenarios( - cases: readonly KnowledgeBenchmarkCase[], - splits?: readonly KnowledgeBenchmarkSplit[], -): KnowledgeBenchmarkScenario[] { - const splitSet = splits ? new Set(splits) : null - return cases.flatMap((testCase) => { - const splitTag = testCase.split ?? 'dev' - if (splitSet && !splitSet.has(splitTag)) return [] - return [ - compactObject({ - id: testCase.id, - kind: 'knowledge-benchmark' as const, - family: testCase.family, - taskKind: testCase.taskKind, - splitTag, - tags: unique([splitTag, ...(testCase.tags ?? [])]), - case: compactObject(testCase), - }) as KnowledgeBenchmarkScenario, - ] - }) -} - -export function knowledgeBenchmarkJudge(): JudgeConfig< - TArtifact, - KnowledgeBenchmarkScenario -> { - return { - name: 'knowledge-benchmark', - judgeVersion: 'agent-knowledge:knowledge-benchmark:v2', - dimensions: [ - { key: 'score', description: 'primary knowledge benchmark score' }, - { key: 'passed', description: '1 when the benchmark case passes' }, - { key: 'claim_recall', description: 'required claim coverage' }, - { key: 'citation_recall', description: 'expected citation/source coverage' }, - { key: 'hallucination_safe', description: '1 when no forbidden claim appears' }, - { key: 'memory_fact_recall', description: 'current memory fact coverage' }, - { key: 'memory_event_recall', description: 'expected memory event/source coverage' }, - { key: 'memory_stale_safe', description: '1 when obsolete memory is not reused' }, - { key: 'memory_actor_recall', description: 'expected speaker/user attribution coverage' }, - ], - appliesTo: (scenario) => scenario.kind === 'knowledge-benchmark', - score({ artifact, scenario }) { - const evaluation = scoreKnowledgeBenchmarkArtifact(scenario.case, artifact) - return { - dimensions: { - score: evaluation.score, - passed: evaluation.passed ? 1 : 0, - ...evaluation.dimensions, - }, - composite: evaluation.score, - notes: evaluation.notes, - } - }, - } -} - -export function scoreKnowledgeBenchmarkArtifact( - testCase: KnowledgeBenchmarkCase, - artifact: TArtifact, -): KnowledgeBenchmarkEvaluation { - if (testCase.taskKind === 'retrieval') { - const retrievalArtifact = normalizeRetrievalArtifact(testCase, artifact) - const metrics = scoreRetrievalArtifact(retrievalArtifact, retrievalScenarioForCase(testCase)) - return { - score: metrics.recall, - passed: metrics.recall >= 1, - dimensions: { - recall: metrics.recall, - mrr: metrics.mrr, - ndcg: metrics.ndcg, - precision_at_k: metrics.precisionAtK, - expected_count: metrics.expectedCount, - matched_count: metrics.matchedCount, - }, - notes: `matched ${metrics.matchedCount}/${metrics.expectedCount}; first_hit_rank=${metrics.firstHitRank ?? 'none'}`, - raw: { matchedTargetIds: metrics.matchedTargetIds }, - } - } - if (isKnowledgeMemoryBenchmarkCase(testCase)) { - return scoreMemoryBenchmarkArtifact(testCase, artifact) - } - - const answerArtifact = artifact as KnowledgeBenchmarkArtifact - const text = answerArtifact.text ?? answerArtifact.answer ?? '' - const required = scoreClaims(text, testCase.requiredClaims ?? []) - const forbidden = scoreForbiddenClaims(text, testCase.forbiddenClaims ?? []) - const citation = scoreCitationRecall( - answerArtifact.citedSourceIds ?? [], - testCase.expectedSourceIds ?? [], - ) - const components = [ - required.totalWeight > 0 ? required.recall : undefined, - testCase.expectedSourceIds && testCase.expectedSourceIds.length > 0 ? citation : undefined, - forbidden.safe, - ].filter((value): value is number => value !== undefined) - const score = mean(components) - return { - score, - passed: score >= 1, - dimensions: { - claim_recall: required.recall, - citation_recall: citation, - hallucination_safe: forbidden.safe, - forbidden_claim_rate: forbidden.rate, - required_claim_count: required.total, - matched_claim_count: required.matched, - forbidden_claim_count: forbidden.total, - matched_forbidden_claim_count: forbidden.matched, - }, - notes: `required=${required.matched}/${required.total}; forbidden=${forbidden.matched}/${forbidden.total}; citation_recall=${citation.toFixed(3)}`, - raw: { - matchedRequiredClaimIds: required.matchedIds, - matchedForbiddenClaimIds: forbidden.matchedIds, - }, - } -} - -export function summarizeKnowledgeBenchmarkCampaign(input: { - scenarios: readonly KnowledgeBenchmarkScenario[] - campaign: CampaignResult -}): KnowledgeBenchmarkReport { - const scenariosById = new Map(input.scenarios.map((scenario) => [scenario.id, scenario])) - const rows = input.campaign.cells.map((cell) => { - const score = Object.values(cell.judgeScores)[0] - const scenario = scenariosById.get(cell.scenarioId) - return { - cell, - scenario, - composite: score?.composite ?? 0, - passed: (score?.dimensions.passed ?? 0) >= 1, - dimensions: score?.dimensions ?? {}, - } - }) - const successful = rows.filter((row) => !row.cell.error) - return { - totalCases: input.scenarios.length, - totalCells: input.campaign.cells.length, - cellsFailed: input.campaign.aggregates.cellsFailed, - cellsCached: input.campaign.aggregates.cellsCached, - totalCostUsd: input.campaign.aggregates.totalCostUsd, - bySplit: summarizeSlices(successful, (row) => row.scenario?.splitTag ?? 'unknown'), - byFamily: summarizeSlices(successful, (row) => row.scenario?.family ?? 'unknown'), - byTaskKind: summarizeSlices(successful, (row) => row.scenario?.taskKind ?? 'unknown'), - dimensions: summarizeDimensions(successful.map((row) => row.dimensions)), - score: distribution(successful.map((row) => row.composite)), - } -} - -export function scoreMemoryBenchmarkArtifact( - testCase: KnowledgeMemoryBenchmarkCase, - artifact: TArtifact, -): KnowledgeBenchmarkEvaluation { - const memoryArtifact = artifact as KnowledgeBenchmarkArtifact - const text = [ - memoryArtifact.text, - memoryArtifact.answer, - ...(memoryArtifact.rememberedFacts ?? []), - ] - .filter((part): part is string => typeof part === 'string' && part.length > 0) - .join('\n') - const required = scoreClaims(text, testCase.requiredFacts ?? []) - const forbidden = scoreForbiddenClaims(text, testCase.forbiddenFacts ?? []) - const eventIds = unique([ - ...(memoryArtifact.citedEventIds ?? []), - ...(memoryArtifact.usedMemoryIds ?? []), - ]) - const eventRecall = scoreCitationRecall(eventIds, testCase.expectedEventIds ?? []) - const actorRecall = scoreCitationRecall( - memoryArtifact.actorIds ?? [], - testCase.expectedActorIds ?? [], - ) - const components = [ - required.totalWeight > 0 ? required.recall : undefined, - testCase.expectedEventIds && testCase.expectedEventIds.length > 0 ? eventRecall : undefined, - testCase.expectedActorIds && testCase.expectedActorIds.length > 0 ? actorRecall : undefined, - testCase.forbiddenFacts && testCase.forbiddenFacts.length > 0 ? forbidden.safe : undefined, - ].filter((value): value is number => value !== undefined) - const score = mean(components) - const dimensions: Record = {} - if (required.totalWeight > 0) { - dimensions.memory_fact_recall = required.recall - dimensions.memory_required_fact_count = required.total - dimensions.memory_matched_fact_count = required.matched - } - if (testCase.expectedEventIds && testCase.expectedEventIds.length > 0) { - dimensions.memory_event_recall = eventRecall - } - if (testCase.expectedActorIds && testCase.expectedActorIds.length > 0) { - dimensions.memory_actor_recall = actorRecall - } - if (testCase.forbiddenFacts && testCase.forbiddenFacts.length > 0) { - dimensions.memory_stale_safe = forbidden.safe - dimensions.memory_stale_rate = forbidden.rate - dimensions.memory_forbidden_fact_count = forbidden.total - dimensions.memory_matched_forbidden_fact_count = forbidden.matched - } - return { - score, - passed: score >= 1, - dimensions, - applicableDimensions: Object.keys(dimensions), - notes: `memory required=${required.matched}/${required.total}; stale=${forbidden.matched}/${forbidden.total}; event_recall=${eventRecall.toFixed(3)}; actor_recall=${actorRecall.toFixed(3)}`, - raw: { - matchedRequiredFactIds: required.matchedIds, - matchedForbiddenFactIds: forbidden.matchedIds, - citedEventIds: eventIds, - actorIds: memoryArtifact.actorIds ?? [], - }, - } -} - -function retrievalScenarioForCase( - testCase: KnowledgeRetrievalBenchmarkCase, -): RetrievalEvalScenario { - return { - id: testCase.id, - kind: 'retrieval-eval', - query: testCase.query, - expected: testCase.expected, - ...(testCase.k !== undefined ? { k: testCase.k } : {}), - } -} - -function normalizeRetrievalArtifact( - testCase: KnowledgeRetrievalBenchmarkCase, - artifact: TArtifact, -): RetrievalEvalArtifact { - const maybe = artifact as Partial & KnowledgeBenchmarkArtifact - const hits = maybe.hits ?? [] - if (Array.isArray(maybe.hits) && maybe.query && maybe.requestedK !== undefined) { - return maybe as RetrievalEvalArtifact - } - return { - config: {}, - query: testCase.query, - requestedK: testCase.k ?? Math.max(1, hits.length), - hits, - durationMs: maybe.durationMs ?? 0, - ...(maybe.costUsd !== undefined ? { costUsd: maybe.costUsd } : {}), - ...(maybe.metadata ? { metadata: maybe.metadata } : {}), - } -} - -function defaultDocumentTarget( - documentId: string, - targetKind: 'page' | 'page-path' | 'source', -): RetrievalGoldTarget { - switch (targetKind) { - case 'page': - return { kind: 'page', pageId: documentId } - case 'page-path': - return { kind: 'page-path', path: documentId } - case 'source': - return { kind: 'source', sourceId: documentId } - } -} - -function hitForExpectedTarget( - expected: RetrievalGoldTarget | undefined, - fallbackId: string, -): RetrievedKnowledgeHit { - if (!expected) { - return { - pageId: fallbackId, - path: `${fallbackId}.md`, - rank: 1, - } - } - switch (expected.kind) { - case 'page': - return { - pageId: expected.pageId, - path: `${expected.pageId}.md`, - rank: 1, - } - case 'page-path': - return { - pageId: expected.path, - path: expected.path, - rank: 1, - } - case 'source': - return { - pageId: expected.sourceId, - path: `${expected.sourceId}.md`, - sourceIds: [expected.sourceId], - rank: 1, - } - case 'source-anchor': - return { - pageId: expected.sourceId, - path: `${expected.sourceId}.md`, - sourceIds: [expected.sourceId], - sourceSpans: [{ sourceId: expected.sourceId, anchorId: expected.anchorId }], - rank: 1, - } - case 'source-span': - return { - pageId: expected.sourceId, - path: `${expected.sourceId}.md`, - sourceIds: [expected.sourceId], - sourceSpans: [ - { - sourceId: expected.sourceId, - charStart: expected.charStart, - charEnd: expected.charEnd, - }, - ], - rank: 1, - } - } -} - -function memoryLifecycleCase(input: { - id: string - family: KnowledgeBenchmarkFamily - source: KnowledgeBenchmarkSource - taskKind: KnowledgeMemoryBenchmarkTaskKind - split: KnowledgeBenchmarkSplit - actorId: string - prompt: string - staleText: string - currentText: string - required: string - extraRequired?: string - forbidden: string -}): KnowledgeMemoryBenchmarkCase { - const staleEventId = `${input.id}:stale` - const currentEventId = `${input.id}:current` - return { - id: input.id, - family: input.family, - taskKind: input.taskKind, - split: input.split, - tags: unique(['first-party-memory-lifecycle', input.taskKind, input.split]), - source: input.source, - events: [ - { - id: staleEventId, - actorId: input.actorId, - sessionId: `${input.id}:session-1`, - timestamp: '2026-01-01T00:00:00.000Z', - text: input.staleText, - }, - { - id: currentEventId, - actorId: input.actorId, - sessionId: `${input.id}:session-2`, - timestamp: '2026-02-01T00:00:00.000Z', - text: input.currentText, - }, - ], - prompt: input.prompt, - requiredFacts: [ - { - id: `${input.id}:required-1`, - anyOf: [input.required], - sourceEventIds: [currentEventId], - }, - ...(input.extraRequired - ? [ - { - id: `${input.id}:required-2`, - anyOf: [input.extraRequired], - sourceEventIds: [currentEventId], - }, - ] - : []), - ], - forbiddenFacts: [ - { - id: `${input.id}:stale`, - anyOf: [input.forbidden], - sourceEventIds: [staleEventId], - obsolete: true, - }, - ], - expectedEventIds: [currentEventId], - expectedActorIds: [input.actorId], - referenceAnswer: input.required, - } -} - -function renderMemoryAdapterRankingMarkdown( - rows: readonly MemoryAdapterBenchmarkRankingRow[], - totalCostUsd: number, - unrankedRecoveryCostUsd: number, -): string { - return [ - '# Memory Adapter Ranking', - '', - `- total cost: $${formatNumber(totalCostUsd)}`, - `- retired-candidate recovery cost: $${formatNumber(unrankedRecoveryCostUsd)}`, - '', - '| rank | candidate | adapter | cases | cells | failed | mean score | pass rate | cost |', - '| ---: | --- | --- | ---: | ---: | ---: | ---: | ---: | ---: |', - ...rows.map( - (row) => - `| ${row.rank} | ${row.label} | ${row.adapterId} | ${row.totalCases} | ${row.totalCells} | ${row.cellsFailed} | ${formatNumber(row.scoreMean)} | ${formatNumber(row.passRate)} | $${formatNumber(row.totalCostUsd)} |`, - ), - '', - ].join('\n') -} - -function benchmarkMemoryScope( - candidateId: string, - testCase: KnowledgeMemoryBenchmarkCase, - cellId: string, - attemptId: string, - scope: AgentMemoryScope = {}, -): AgentMemoryScope { - return { - ...scope, - namespace: `${scope.namespace ?? 'agent-knowledge-memory-benchmark'}:${attemptId}`, - tags: { - ...(scope.tags ?? {}), - benchmarkCandidateId: candidateId, - benchmarkCaseId: testCase.id, - benchmarkCellId: cellId, - benchmarkAttemptId: attemptId, - }, - } -} - -function assertScopedMemoryBenchmarkAdapter(adapter: AgentMemoryAdapter): void { - if (adapter.branchIsolation?.mode !== 'scoped') { - throw new Error( - `${adapter.id}: direct memory benchmark requires branchIsolation mode scoped; use runAgentMemoryExperiment for dedicated instances`, - ) - } - if (!adapter.clear) { - throw new Error(`${adapter.id}: direct memory benchmark requires exact scoped clear`) - } -} - -function memoryEventId(hit: AgentMemoryHit): string | undefined { - const eventId = hit.metadata?.eventId - return typeof eventId === 'string' ? eventId : undefined -} - -function memoryActorId(hit: AgentMemoryHit): string | undefined { - const actorId = hit.metadata?.actorId - return typeof actorId === 'string' ? actorId : undefined -} - -function memoryScopeMatches(stored?: AgentMemoryScope, requested?: AgentMemoryScope): boolean { - if (!requested) return true - if (requested.tenantId !== undefined && stored?.tenantId !== requested.tenantId) return false - if (requested.userId !== undefined && stored?.userId !== requested.userId) return false - if (requested.agentId !== undefined && stored?.agentId !== requested.agentId) return false - if (requested.teamId !== undefined && stored?.teamId !== requested.teamId) return false - if (requested.runId !== undefined && stored?.runId !== requested.runId) return false - if (requested.sessionId !== undefined && stored?.sessionId !== requested.sessionId) return false - if (requested.namespace !== undefined && stored?.namespace !== requested.namespace) return false - for (const [key, value] of Object.entries(requested.tags ?? {})) { - if (stored?.tags?.[key] !== value) return false - } - return true -} - -function tokenOverlap(query: string, text: string): number { - const queryTokens = new Set(tokenize(query)) - if (queryTokens.size === 0) return 0 - const textTokens = new Set(tokenize(text)) - let matched = 0 - for (const token of queryTokens) { - if (textTokens.has(token)) matched += 1 - } - return matched / queryTokens.size -} - -function tokenize(text: string): string[] { - const stop = new Set([ - 'the', - 'and', - 'for', - 'this', - 'that', - 'with', - 'what', - 'should', - 'agent', - 'user', - 'current', - 'now', - 'use', - ]) - return text - .toLowerCase() - .split(/[^a-z0-9/]+/) - .filter((token) => token.length > 2 && !stop.has(token)) -} - -function renderMemoryHits(hits: readonly AgentMemoryHit[]): string { - return hits - .map((hit, index) => { - const eventId = memoryEventId(hit) - const actorId = memoryActorId(hit) - return [ - `[${index + 1}] ${hit.title ?? hit.id}`, - eventId ? `event=${eventId}` : '', - actorId ? `actor=${actorId}` : '', - hit.text, - ] - .filter(Boolean) - .join('\n') - }) - .join('\n\n') -} - -export function isKnowledgeMemoryBenchmarkCase( - testCase: KnowledgeBenchmarkCase, -): testCase is KnowledgeMemoryBenchmarkCase { - return testCase.taskKind.startsWith('memory-') -} - -function assertKnowledgeBenchmarkCases(cases: readonly KnowledgeBenchmarkCase[]): void { - if (cases.length === 0) throw new Error('knowledge benchmark requires cases') - assertUniqueNonEmptyStrings( - cases.map((testCase) => testCase.id), - 'knowledge benchmark case id', - ) - for (const testCase of cases) { - if (typeof testCase.family !== 'string' || !testCase.family.trim()) { - throw new Error(`knowledge benchmark case ${testCase.id} requires a family`) - } - if (testCase.taskKind === 'retrieval') continue - if (isKnowledgeMemoryBenchmarkCase(testCase)) { - assertUniqueNonEmptyStrings( - testCase.events.map((event) => event.id), - `${testCase.id} memory event id`, - ) - for (const event of testCase.events) { - assertNonEmptyBenchmarkString(event.text, `${testCase.id} memory event ${event.id} text`) - } - assertClaimMatchers(testCase.requiredFacts ?? [], `${testCase.id} requiredFacts`) - assertClaimMatchers(testCase.forbiddenFacts ?? [], `${testCase.id} forbiddenFacts`) - assertUniqueNonEmptyStrings( - testCase.expectedEventIds ?? [], - `${testCase.id} expected event id`, - ) - assertUniqueNonEmptyStrings( - testCase.expectedActorIds ?? [], - `${testCase.id} expected actor id`, - ) - } else { - assertClaimMatchers(testCase.requiredClaims ?? [], `${testCase.id} requiredClaims`) - assertClaimMatchers(testCase.forbiddenClaims ?? [], `${testCase.id} forbiddenClaims`) - assertUniqueNonEmptyStrings(testCase.expectedSourceIds ?? [], `${testCase.id} source id`) - } - } -} - -function assertClaimMatchers(claims: readonly KnowledgeClaimMatcher[], label: string): void { - assertUniqueNonEmptyStrings( - claims.map((claim) => claim.id), - `${label} matcher id`, - ) - for (const claim of claims) { - if (claim.anyOf.length === 0) throw new Error(`${label} matcher ${claim.id} requires anyOf`) - assertUniqueNonEmptyStrings(claim.anyOf, `${label} matcher ${claim.id} anyOf`) - if (claim.weight !== undefined && (!Number.isFinite(claim.weight) || claim.weight <= 0)) { - throw new Error(`${label} matcher ${claim.id} weight must be a positive finite number`) - } - const sourceEventIds = (claim as Partial).sourceEventIds - if (sourceEventIds !== undefined) { - assertUniqueNonEmptyStrings(sourceEventIds, `${label} matcher ${claim.id} source event id`) - } - } -} - -function assertUniqueNonEmptyStrings(values: readonly string[], label: string): void { - const seen = new Set() - for (const value of values) { - assertNonEmptyBenchmarkString(value, label) - if (seen.has(value)) throw new Error(`duplicate ${label}: ${value}`) - seen.add(value) - } -} - -function assertNonEmptyBenchmarkString(value: unknown, label: string): asserts value is string { - if (typeof value !== 'string' || !value.trim()) throw new Error(`${label} must be non-empty`) -} - -function scoreClaims(text: string, claims: readonly KnowledgeClaimMatcher[]) { - let matched = 0 - let matchedWeight = 0 - let totalWeight = 0 - const matchedIds: string[] = [] - const haystack = text.toLowerCase() - for (const claim of claims) { - if ( - !claim.id.trim() || - claim.anyOf.length === 0 || - claim.anyOf.some((value) => !value.trim()) - ) { - throw new Error( - 'claim matchers require a non-empty id and at least one non-empty alternative', - ) - } - const weight = claim.weight ?? 1 - if (!Number.isFinite(weight) || weight <= 0) { - throw new Error(`claim matcher ${claim.id} weight must be a positive finite number`) - } - totalWeight += weight - if (claim.anyOf.some((fragment) => haystack.includes(fragment.toLowerCase()))) { - matched += 1 - matchedWeight += weight - matchedIds.push(claim.id) - } - } - return { - total: claims.length, - matched, - totalWeight, - recall: totalWeight === 0 ? 1 : matchedWeight / totalWeight, - matchedIds, - } -} - -function scoreForbiddenClaims(text: string, claims: readonly KnowledgeClaimMatcher[]) { - const matched = scoreClaims(text, claims) - return { - total: claims.length, - matched: matched.matched, - matchedIds: matched.matchedIds, - rate: claims.length === 0 ? 0 : matched.matched / claims.length, - safe: matched.matched === 0 ? 1 : 0, - } -} - -function scoreCitationRecall( - citedSourceIds: readonly string[], - expectedSourceIds: readonly string[], -): number { - if (expectedSourceIds.length === 0) return 1 - const cited = new Set(citedSourceIds) - const matched = expectedSourceIds.filter((sourceId) => cited.has(sourceId)).length - return matched / expectedSourceIds.length -} - -function summarizeDimensions( - rows: Array>, -): Record { - const values = new Map() - for (const row of rows) { - for (const [key, value] of Object.entries(row)) { - if (!Number.isFinite(value)) continue - const list = values.get(key) ?? [] - list.push(value) - values.set(key, list) - } - } - return Object.fromEntries([...values.entries()].map(([key, vals]) => [key, distribution(vals)])) -} - -function summarizeSlices( - rows: T[], - keyOf: (row: T) => string, -): Record { - const grouped = new Map() - for (const row of rows) { - const key = keyOf(row) - const list = grouped.get(key) ?? [] - list.push(row) - grouped.set(key, list) - } - return Object.fromEntries( - [...grouped.entries()].map(([key, list]) => { - const withShape = list as Array<{ composite: number; passed: boolean }> - return [ - key, - { - n: list.length, - meanScore: mean(withShape.map((row) => row.composite)), - passRate: mean(withShape.map((row) => (row.passed ? 1 : 0))), - score: distribution(withShape.map((row) => row.composite)), - }, - ] - }), - ) -} - -function distribution(values: readonly number[]): KnowledgeBenchmarkDistribution { - const finite = [...values].filter(Number.isFinite).sort((a, b) => a - b) - if (finite.length === 0) return { n: 0, min: 0, mean: 0, median: 0, p90: 0, max: 0 } - return { - n: finite.length, - min: finite[0]!, - mean: mean(finite), - median: percentile(finite, 0.5), - p90: percentile(finite, 0.9), - max: finite[finite.length - 1]!, - } -} - -function percentile(sortedValues: readonly number[], p: number): number { - if (sortedValues.length === 0) return 0 - const index = Math.min( - sortedValues.length - 1, - Math.max(0, Math.ceil(p * sortedValues.length) - 1), - ) - return sortedValues[index]! -} - -function mean(values: readonly number[]): number { - const finite = values.filter(Number.isFinite) - if (finite.length === 0) return 0 - return finite.reduce((sum, value) => sum + value, 0) / finite.length -} - -function unique(values: readonly string[]): string[] { - return [...new Set(values.filter(Boolean))] -} - -function renderSliceTable(slices: Record): string { - const rows = Object.entries(slices).map( - ([key, slice]) => - `| ${key} | ${slice.n} | ${formatNumber(slice.meanScore)} | ${formatNumber(slice.passRate)} | ${formatNumber(slice.score.p90)} |`, - ) - return [ - '| slice | n | mean score | pass rate | score p90 |', - '| --- | ---: | ---: | ---: | ---: |', - ...(rows.length ? rows : ['| none | 0 | 0 | 0 | 0 |']), - ].join('\n') -} - -function formatNumber(value: number): string { - if (!Number.isFinite(value)) return '0' - return value.toFixed(value === 0 || Math.abs(value) >= 10 ? 0 : 3) -} - -function normalizeUsd(value: number): number { - return Number(value.toFixed(12)) -} - -function compactObject(value: unknown): unknown { - if (Array.isArray(value)) return value.map(compactObject) - if (!value || typeof value !== 'object') return value - return Object.fromEntries( - Object.entries(value as Record) - .filter(([, entry]) => entry !== undefined) - .map(([key, entry]) => [key, compactObject(entry)]), - ) -} + KnowledgeClaimMatcher, + KnowledgeMemoryBenchmarkCase, + KnowledgeMemoryBenchmarkTaskKind, + KnowledgeMemoryEvent, + KnowledgeMemoryFactMatcher, + KnowledgeRetrievalBenchmarkCase, + KnowledgeRetrievalBenchmarkQrel, + KnowledgeRetrievalBenchmarkQuery, + MemoryAdapterBenchmarkCandidate, + MemoryAdapterBenchmarkRankingRow, + RunKnowledgeBenchmarkSuiteOptions, + RunKnowledgeBenchmarkSuiteResult, + RunMemoryAdapterBenchmarkOptions, + RunMemoryAdapterBenchmarkResult, +} from './types' +export { isKnowledgeMemoryBenchmarkCase } from './validation' diff --git a/src/benchmarks/memory-recovery.ts b/src/benchmarks/memory-recovery.ts new file mode 100644 index 0000000..8219fc7 --- /dev/null +++ b/src/benchmarks/memory-recovery.ts @@ -0,0 +1,533 @@ +import { join } from 'node:path' + +import { canonicalJson } from '@tangle-network/agent-eval' + +import type { CampaignStorage, CostLedgerHandle } from '@tangle-network/agent-eval/campaign' + +import { stableId } from '../ids' + +import { + appendAttemptJournalEvent, + assertNoInterruptedPaidCalls, + hasSettledPaidCall, + readActiveAttemptJournal, + reconcileInterruptedMemoryPaidCalls, + reserveRecoveryAttempts, +} from '../memory/attempt-log' + +import { + createMemoryExecutionPool, + memoryRecoveryDelayMs, + releaseMemoryAdapterCreatedAfterAbort, + runBoundedMemoryLifecycle, + sleepForMemoryRecovery, +} from '../memory/lifecycle' + +import type { OwnedAgentMemoryRunLease } from '../memory/run-control' + +import type { AgentMemoryAdapter, AgentMemoryScope } from '../memory/types' + +import type { MemoryAdapterBenchmarkCandidate } from './types' + +export function memoryAdapterBenchmarkExpectedId( + candidate: MemoryAdapterBenchmarkCandidate, +): string { + return candidate.adapterId ?? candidate.id +} + +export async function recoverMemoryAdapterBenchmarkAttempts(input: { + candidates: readonly MemoryAdapterBenchmarkCandidate[] + storage: CampaignStorage + attemptLogPath: string + lease: OwnedAgentMemoryRunLease + cleanupTimeoutMs: number + maxConcurrency: number + now?: () => Date + runDir: string + costLedger: CostLedgerHandle + costPhase: string + maxRecoveryAttempts: number + recoveryLogPath: string + maxRecoveryRetriesPerAttempt: number +}): Promise { + let attempts = readActiveMemoryBenchmarkAttempts(input.storage, input.attemptLogPath) + if (attempts.length > input.maxRecoveryAttempts) { + throw new Error( + `memory adapter benchmark has ${attempts.length} unfinished attempts; maxRecoveryAttempts is ${input.maxRecoveryAttempts}`, + ) + } + const candidateById = new Map(input.candidates.map((candidate) => [candidate.id, candidate])) + for (const attempt of attempts) { + const candidate = candidateById.get(attempt.candidateId) + if (!candidate) { + throw new Error( + `cannot recover memory benchmark attempts: candidate '${attempt.candidateId}' is missing; pass it in recoveryCandidates`, + ) + } + assertMemoryBenchmarkAttemptCandidateMatches(attempt, candidate) + } + + reconcileInterruptedMemoryPaidCalls(input.costLedger) + assertNoInterruptedPaidCalls(input.costLedger, 'memory adapter benchmark recovery') + + for (const attempt of attempts) { + const candidate = candidateById.get(attempt.candidateId)! + const costUsd = candidate.costUsdPerCase ?? 0 + if ( + costUsd > 0 && + !hasSettledPaidCall(input.costLedger, memoryBenchmarkCostCallId(attempt, 'execute', 0)) + ) { + appendMemoryBenchmarkAttemptEvent(input.storage, input.attemptLogPath, { + ...attempt, + status: 'cleaned', + recovery: true, + recordedAt: (input.now ?? (() => new Date()))().toISOString(), + }) + } + } + attempts = readActiveMemoryBenchmarkAttempts(input.storage, input.attemptLogPath) + const recoveryGenerations = reserveRecoveryAttempts({ + storage: input.storage, + path: input.recoveryLogPath, + attemptIds: attempts.map((attempt) => attempt.attemptId), + maxRetriesPerAttempt: input.maxRecoveryRetriesPerAttempt, + label: 'memory benchmark recovery attempt log', + now: input.now, + }) + const grouped = groupMemoryBenchmarkAttempts(attempts) + const pool = createMemoryExecutionPool(input.maxConcurrency) + const settled = await Promise.allSettled( + [...grouped] + .sort(([left], [right]) => left.localeCompare(right)) + .map(([candidateId, candidateAttempts]) => + pool.run(async () => { + await input.lease.assertOwned() + const candidate = candidateById.get(candidateId) + if (!candidate) { + throw new Error( + `cannot recover memory benchmark attempts: candidate '${candidateId}' is missing; pass it in recoveryCandidates`, + ) + } + for (const attempt of candidateAttempts) { + assertMemoryBenchmarkAttemptCandidateMatches(attempt, candidate) + if (recoveryGenerations.get(attempt.attemptId) === undefined) { + throw new Error( + `missing recovery generation for memory benchmark attempt '${attempt.attemptId}'`, + ) + } + } + + let recoveryAttemptsStarted = 0 + let adapterCreationExternalCallAttempted = false + const cleared: MemoryAdapterBenchmarkAttemptEvent[] = [] + const recover = async (): Promise => { + let adapter: AgentMemoryAdapter | undefined + let primaryError: unknown + try { + const abortController = new AbortController() + const creation = Promise.resolve().then(() => + candidate.createAdapter({ + purpose: 'recovery', + signal: abortController.signal, + markExternalCall: () => { + adapterCreationExternalCallAttempted = true + }, + }), + ) + releaseMemoryAdapterCreatedAfterAbort({ + creation, + signal: abortController.signal, + }) + adapter = await runBoundedMemoryLifecycle({ + operation: `${candidate.id}: benchmark recovery adapter creation`, + timeoutMs: input.cleanupTimeoutMs, + abortController, + run: () => creation, + }) + assertScopedMemoryBenchmarkAdapter(adapter) + for (const attempt of candidateAttempts) { + if (adapter.id !== attempt.adapterId) { + throw new Error( + `cannot recover memory benchmark attempt '${attempt.attemptId}': adapter changed from '${attempt.adapterId}' to '${adapter.id}'`, + ) + } + } + await sleepForMemoryRecovery( + memoryRecoveryDelayMs(adapter), + () => input.lease.assertOwned(), + input.cleanupTimeoutMs, + `${candidate.id}: benchmark recovery visibility wait`, + ) + for (const attempt of candidateAttempts.sort((left, right) => + left.attemptId.localeCompare(right.attemptId), + )) { + await input.lease.assertOwned() + try { + recoveryAttemptsStarted += 1 + await runBoundedMemoryLifecycle({ + operation: `${candidate.id}: abandoned benchmark attempt cleanup`, + timeoutMs: input.cleanupTimeoutMs, + resource: adapter, + run: () => adapter!.clear!(attempt.scope), + }) + await input.lease.assertOwned() + cleared.push(attempt) + } catch (error) { + primaryError = primaryError + ? new AggregateError( + [primaryError, error], + `${candidate.id}: multiple benchmark attempts failed recovery`, + ) + : error + } + } + } catch (error) { + primaryError = primaryError + ? new AggregateError( + [primaryError, error], + `${candidate.id}: benchmark recovery failed in multiple operations`, + ) + : error + } + + let closeError: unknown + if (adapter) { + try { + await runBoundedMemoryLifecycle({ + operation: `${candidate.id}: benchmark recovery adapter close`, + timeoutMs: input.cleanupTimeoutMs, + resource: adapter, + run: () => adapter!.close?.(), + }) + } catch (error) { + closeError = error + } + } + let journalError: unknown + if (!closeError) { + try { + for (const attempt of cleared) { + await input.lease.assertOwned() + appendMemoryBenchmarkAttemptEvent(input.storage, input.attemptLogPath, { + ...attempt, + status: 'cleaned', + recovery: true, + recordedAt: (input.now ?? (() => new Date()))().toISOString(), + }) + } + } catch (error) { + journalError = error + } + } + const failures = [primaryError, closeError, journalError].filter( + (error) => error !== undefined, + ) + if (failures.length > 1) { + throw new AggregateError( + failures, + `${candidate.id}: benchmark attempt recovery, adapter close, or cleanup journal failed`, + ) + } + if (primaryError) throw primaryError + if (closeError) throw closeError + if (journalError) throw journalError + } + + const recoveryCostUsd = candidate.recoveryCostUsdPerAttempt ?? 0 + const adapterCreationCostUsd = candidate.adapterCreationCostUsd ?? 0 + const maximumCostUsd = adapterCreationCostUsd + recoveryCostUsd * candidateAttempts.length + const actualCostUsd = (): number => + (adapterCreationExternalCallAttempted ? adapterCreationCostUsd : 0) + + recoveryCostUsd * recoveryAttemptsStarted + let recoveryError: unknown + if (maximumCostUsd === 0) { + try { + await recover() + } catch (error) { + recoveryError = error + } + } else { + const receipt = { + model: candidate.id, + inputTokens: 0, + outputTokens: 0, + } as const + const tags = memoryBenchmarkRecoveryCostTags(input.runDir, candidate.id) + const paid = await input.costLedger.runPaidCall({ + callId: memoryBenchmarkRecoveryCostCallId( + candidate, + candidateAttempts, + recoveryGenerations, + ), + channel: 'driver', + phase: `${input.costPhase}.${candidate.id}.recovery`, + actor: `agent-knowledge:memory-adapter-recovery:${candidate.id}`, + model: candidate.id, + tags, + maximumCharge: { externallyEnforcedMaximumUsd: maximumCostUsd }, + execute: recover, + receipt: () => ({ ...receipt, actualCostUsd: actualCostUsd() }), + receiptFromError: () => ({ + ...receipt, + actualCostUsd: actualCostUsd(), + }), + }) + if (!paid.succeeded) recoveryError = paid.error + } + if (recoveryError) throw recoveryError + }), + ), + ) + const failures = settled.flatMap((result) => + result.status === 'rejected' ? [result.reason] : [], + ) + if (failures.length === 1) throw failures[0] + if (failures.length > 1) { + throw new AggregateError(failures, 'multiple memory benchmark candidates failed recovery') + } +} + +function groupMemoryBenchmarkAttempts( + attempts: readonly MemoryAdapterBenchmarkAttemptEvent[], +): Map { + const grouped = new Map() + for (const attempt of attempts) { + const group = grouped.get(attempt.candidateId) ?? [] + group.push(attempt) + grouped.set(attempt.candidateId, group) + } + return grouped +} + +export function memoryBenchmarkCostCallId( + attempt: MemoryAdapterBenchmarkAttemptEvent, + purpose: 'execute' | 'recovery', + generation: number, +): string { + return stableId( + 'memory_benchmark_cost_call', + canonicalJson({ + purpose, + generation, + attemptId: attempt.attemptId, + candidateId: attempt.candidateId, + candidateRef: attempt.candidateRef, + adapterId: attempt.adapterId, + adapterCreationCostUsd: attempt.adapterCreationCostUsd, + costUsdPerCase: attempt.costUsdPerCase, + recoveryCostUsdPerAttempt: attempt.recoveryCostUsdPerAttempt, + caseId: attempt.caseId, + cellId: attempt.cellId, + }), + ) +} + +function memoryBenchmarkRecoveryCostCallId( + candidate: MemoryAdapterBenchmarkCandidate, + attempts: readonly MemoryAdapterBenchmarkAttemptEvent[], + generations: ReadonlyMap, +): string { + return stableId( + 'memory_benchmark_recovery_cost_call', + canonicalJson({ + candidateId: candidate.id, + candidateRef: candidate.ref, + adapterCreationCostUsd: candidate.adapterCreationCostUsd ?? 0, + recoveryCostUsdPerAttempt: candidate.recoveryCostUsdPerAttempt ?? 0, + attempts: attempts + .map((attempt) => ({ + attemptId: attempt.attemptId, + generation: generations.get(attempt.attemptId), + })) + .sort((left, right) => left.attemptId.localeCompare(right.attemptId)), + }), + ) +} + +function memoryBenchmarkRecoveryCostTags( + runDir: string, + candidateId: string, +): Record { + return { + runDir: join(runDir, candidateId), + candidateId, + memoryRecovery: 'benchmark', + } +} + +export function appendMemoryBenchmarkAttemptEvent( + storage: CampaignStorage, + path: string, + event: MemoryAdapterBenchmarkAttemptEvent, +): void { + appendAttemptJournalEvent({ + storage, + path, + event, + label: 'memory benchmark attempt log', + }) +} + +function readActiveMemoryBenchmarkAttempts( + storage: CampaignStorage, + path: string, +): MemoryAdapterBenchmarkAttemptEvent[] { + return readActiveAttemptJournal({ + storage, + path, + label: 'memory benchmark attempt log', + parse: parseMemoryBenchmarkAttemptEvent, + id: (event) => event.attemptId, + sameAttempt: sameMemoryBenchmarkAttempt, + }) +} + +function parseMemoryBenchmarkAttemptEvent( + value: unknown, + path: string, + line: number, +): MemoryAdapterBenchmarkAttemptEvent { + const event = value as Partial | null + const valid = + typeof event === 'object' && + event !== null && + event.schema === 3 && + (event.status === 'started' || event.status === 'cleaned') && + isNonEmptyString(event.attemptId) && + isNonEmptyString(event.candidateId) && + isNonEmptyString(event.candidateRef) && + isNonEmptyString(event.adapterId) && + isNonEmptyString(event.caseId) && + isNonEmptyString(event.cellId) && + isAgentMemoryScope(event.scope) && + typeof event.adapterCreationCostUsd === 'number' && + Number.isFinite(event.adapterCreationCostUsd) && + event.adapterCreationCostUsd >= 0 && + typeof event.costUsdPerCase === 'number' && + Number.isFinite(event.costUsdPerCase) && + event.costUsdPerCase >= 0 && + typeof event.recoveryCostUsdPerAttempt === 'number' && + Number.isFinite(event.recoveryCostUsdPerAttempt) && + event.recoveryCostUsdPerAttempt >= 0 && + typeof event.recordedAt === 'string' && + !Number.isNaN(Date.parse(event.recordedAt)) && + typeof event.recovery === 'boolean' + if (!valid) { + throw new Error(`invalid memory benchmark attempt event in '${path}' line ${line}`) + } + return value as MemoryAdapterBenchmarkAttemptEvent +} + +function sameMemoryBenchmarkAttempt( + left: MemoryAdapterBenchmarkAttemptEvent, + right: MemoryAdapterBenchmarkAttemptEvent, +): boolean { + return ( + left.attemptId === right.attemptId && + left.candidateId === right.candidateId && + left.candidateRef === right.candidateRef && + left.adapterId === right.adapterId && + left.caseId === right.caseId && + left.cellId === right.cellId && + canonicalJson(left.scope) === canonicalJson(right.scope) && + left.adapterCreationCostUsd === right.adapterCreationCostUsd && + left.costUsdPerCase === right.costUsdPerCase && + left.recoveryCostUsdPerAttempt === right.recoveryCostUsdPerAttempt + ) +} + +function assertMemoryBenchmarkAttemptCandidateMatches( + attempt: MemoryAdapterBenchmarkAttemptEvent, + candidate: MemoryAdapterBenchmarkCandidate, +): void { + if (attempt.candidateRef !== candidate.ref) { + throw new Error( + `cannot recover memory benchmark attempt '${attempt.attemptId}': candidate ref changed from '${attempt.candidateRef}' to '${candidate.ref}'`, + ) + } + const expectedAdapterId = memoryAdapterBenchmarkExpectedId(candidate) + if (attempt.adapterId !== expectedAdapterId) { + throw new Error( + `cannot recover memory benchmark attempt '${attempt.attemptId}': adapter id changed from '${attempt.adapterId}' to '${expectedAdapterId}'`, + ) + } + const executionCost = candidate.costUsdPerCase ?? 0 + const adapterCreationCost = candidate.adapterCreationCostUsd ?? 0 + const recoveryCost = candidate.recoveryCostUsdPerAttempt ?? 0 + if ( + adapterCreationCost !== attempt.adapterCreationCostUsd || + executionCost !== attempt.costUsdPerCase || + recoveryCost !== attempt.recoveryCostUsdPerAttempt + ) { + throw new Error( + `cannot recover memory benchmark attempt '${attempt.attemptId}': candidate cost settings changed; start a new run or restore the recorded costs`, + ) + } +} + +export function assertScopedMemoryBenchmarkAdapter(adapter: AgentMemoryAdapter): void { + if (adapter.branchIsolation?.mode !== 'scoped') { + throw new Error( + `${adapter.id}: direct memory benchmark requires branchIsolation mode scoped; use runAgentMemoryExperiment for dedicated instances`, + ) + } + if (!adapter.clear) { + throw new Error(`${adapter.id}: direct memory benchmark requires exact scoped clear`) + } +} + +function isAgentMemoryScope(value: unknown): value is AgentMemoryScope { + if (!isRecordValue(value)) return false + const allowed = new Set([ + 'tenantId', + 'userId', + 'agentId', + 'teamId', + 'runId', + 'sessionId', + 'namespace', + 'tags', + ]) + if (Object.keys(value).some((key) => !allowed.has(key))) return false + for (const key of [ + 'tenantId', + 'userId', + 'agentId', + 'teamId', + 'runId', + 'sessionId', + 'namespace', + ]) { + if (value[key] !== undefined && typeof value[key] !== 'string') return false + } + if (value.tags === undefined) return true + return ( + isRecordValue(value.tags) && + Object.values(value.tags).every((entry) => typeof entry === 'string') + ) +} + +function isNonEmptyString(value: unknown): value is string { + return typeof value === 'string' && value.trim().length > 0 +} + +function isRecordValue(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value) +} + +export interface MemoryAdapterBenchmarkAttemptEvent { + schema: 3 + status: 'started' | 'cleaned' + attemptId: string + candidateId: string + candidateRef: string + adapterId: string + caseId: string + cellId: string + scope: AgentMemoryScope + adapterCreationCostUsd: number + costUsdPerCase: number + recoveryCostUsdPerAttempt: number + recordedAt: string + recovery: boolean +} diff --git a/src/benchmarks/memory-responder.ts b/src/benchmarks/memory-responder.ts new file mode 100644 index 0000000..9a7841f --- /dev/null +++ b/src/benchmarks/memory-responder.ts @@ -0,0 +1,264 @@ +import { randomUUID } from 'node:crypto' +import type { CampaignStorage } from '@tangle-network/agent-eval/campaign' +import { runBoundedMemoryLifecycle } from '../memory/lifecycle' +import type { OwnedAgentMemoryRunLease } from '../memory/run-control' +import type { AgentMemoryAdapter, AgentMemoryScope } from '../memory/types' +import { memoryActorId, memoryEventId } from './adapters' +import { + appendMemoryBenchmarkAttemptEvent, + assertScopedMemoryBenchmarkAdapter, + type MemoryAdapterBenchmarkAttemptEvent, + memoryBenchmarkCostCallId, +} from './memory-recovery' +import type { + KnowledgeBenchmarkArtifact, + KnowledgeBenchmarkResponder, + KnowledgeMemoryBenchmarkCase, +} from './types' +import { compactObject, unique } from './utils' +import { isKnowledgeMemoryBenchmarkCase } from './validation' + +export function createMemoryAdapterBenchmarkResponder(options: { + adapter: AgentMemoryAdapter + candidateId: string + candidateRef: string + storage: CampaignStorage + attemptLogPath: string + lease: OwnedAgentMemoryRunLease + cleanupTimeoutMs: number + searchLimit?: number + scope?: AgentMemoryScope + costUsdPerCase?: number + adapterCreationCostUsd?: number + recoveryCostUsdPerAttempt?: number + now?: () => Date +}): KnowledgeBenchmarkResponder { + assertScopedMemoryBenchmarkAdapter(options.adapter) + return async ({ case: testCase, context: dispatchContext }) => { + if (!isKnowledgeMemoryBenchmarkCase(testCase)) { + return { answer: '', metadata: { candidateId: options.candidateId, skipped: true } } + } + const costUsd = options.costUsdPerCase ?? 0 + if (!Number.isFinite(costUsd) || costUsd < 0) { + throw new Error(`memory adapter costUsdPerCase must be non-negative finite, got ${costUsd}`) + } + + dispatchContext.signal.throwIfAborted() + await options.lease.assertOwned() + const startedAt = Date.now() + const attemptId = randomUUID() + const scope = benchmarkMemoryScope( + options.candidateId, + testCase, + dispatchContext.cellId, + attemptId, + options.scope, + ) + const attempt: MemoryAdapterBenchmarkAttemptEvent = { + schema: 3, + status: 'started', + attemptId, + candidateId: options.candidateId, + candidateRef: options.candidateRef, + adapterId: options.adapter.id, + caseId: testCase.id, + cellId: dispatchContext.cellId, + scope, + adapterCreationCostUsd: options.adapterCreationCostUsd ?? 0, + costUsdPerCase: costUsd, + recoveryCostUsdPerAttempt: options.recoveryCostUsdPerAttempt ?? 0, + recordedAt: (options.now ?? (() => new Date()))().toISOString(), + recovery: false, + } + appendMemoryBenchmarkAttemptEvent(options.storage, options.attemptLogPath, attempt) + + let externalCallAttempted = false + const appendCleanedAttempt = (priorError?: unknown): void => { + try { + appendMemoryBenchmarkAttemptEvent(options.storage, options.attemptLogPath, { + ...attempt, + status: 'cleaned', + recordedAt: (options.now ?? (() => new Date()))().toISOString(), + }) + } catch (error) { + throw new MemoryAdapterBenchmarkCleanupError( + [...(priorError ? [priorError] : []), error], + `${options.candidateId}: memory benchmark cleanup could not be recorded`, + ) + } + } + const execute = async (): Promise => { + dispatchContext.signal.throwIfAborted() + await options.lease.assertOwned() + let artifact: KnowledgeBenchmarkArtifact | undefined + let primaryError: unknown + try { + for (const event of testCase.events) { + dispatchContext.signal.throwIfAborted() + await options.lease.assertOwned() + externalCallAttempted = true + await options.adapter.write({ + id: event.id, + kind: 'message', + text: event.text, + role: event.actorId === 'user' ? 'user' : 'assistant', + title: `${testCase.id}:${event.id}`, + scope, + metadata: compactObject({ + benchmarkCaseId: testCase.id, + benchmarkCellId: dispatchContext.cellId, + benchmarkAttemptId: attemptId, + eventId: event.id, + actorId: event.actorId, + sessionId: event.sessionId, + timestamp: event.timestamp, + ...event.metadata, + }) as Record, + }) + dispatchContext.signal.throwIfAborted() + await options.lease.assertOwned() + } + externalCallAttempted = true + await options.adapter.flush?.() + dispatchContext.signal.throwIfAborted() + await options.lease.assertOwned() + + externalCallAttempted = true + const adapterContext = await options.adapter.getContext(testCase.prompt, { + scope, + limit: options.searchLimit ?? 1, + metadata: { + benchmarkCaseId: testCase.id, + benchmarkCellId: dispatchContext.cellId, + benchmarkAttemptId: attemptId, + candidateId: options.candidateId, + }, + }) + dispatchContext.signal.throwIfAborted() + await options.lease.assertOwned() + const hits = adapterContext.hits + artifact = { + answer: adapterContext.text, + rememberedFacts: hits.map((hit) => hit.text), + citedEventIds: unique(hits.map(memoryEventId).filter((id): id is string => Boolean(id))), + usedMemoryIds: hits.map((hit) => hit.id), + actorIds: unique(hits.map(memoryActorId).filter((id): id is string => Boolean(id))), + costUsd, + durationMs: Math.max(0, Date.now() - startedAt), + metadata: { + candidateId: options.candidateId, + adapterId: options.adapter.id, + hitCount: hits.length, + }, + } + } catch (error) { + primaryError = error + } + + const cleanupErrors: unknown[] = [] + let cleanupOwned = true + try { + await options.lease.assertOwned() + } catch (error) { + cleanupOwned = false + cleanupErrors.push(error) + } + if (cleanupOwned) { + try { + await runBoundedMemoryLifecycle({ + operation: `${options.candidateId}: benchmark attempt flush`, + timeoutMs: options.cleanupTimeoutMs, + resource: options.adapter, + run: () => { + externalCallAttempted = true + return options.adapter.flush?.() + }, + }) + } catch (error) { + cleanupErrors.push(error) + } + try { + await runBoundedMemoryLifecycle({ + operation: `${options.candidateId}: benchmark attempt cleanup`, + timeoutMs: options.cleanupTimeoutMs, + resource: options.adapter, + run: () => { + externalCallAttempted = true + return options.adapter.clear!(scope) + }, + }) + } catch (error) { + cleanupErrors.push(error) + } + } + if (cleanupOwned && cleanupErrors.length === 0) appendCleanedAttempt(primaryError) + if (cleanupErrors.length > 0) { + const errors = [...(primaryError ? [primaryError] : []), ...cleanupErrors] + throw new MemoryAdapterBenchmarkCleanupError( + errors, + `${options.candidateId}: memory benchmark attempt cleanup failed`, + ) + } + if (primaryError) throw primaryError + if (!artifact) + throw new Error(`${options.candidateId}: memory benchmark produced no artifact`) + return artifact + } + + if (costUsd === 0) { + let artifact: KnowledgeBenchmarkArtifact | undefined + let error: unknown + try { + artifact = await execute() + } catch (caught) { + error = caught + } + if (error) throw error + if (!artifact) + throw new Error(`${options.candidateId}: memory benchmark produced no artifact`) + return artifact + } + const receipt = { + model: options.adapter.id, + inputTokens: 0, + outputTokens: 0, + actualCostUsd: costUsd, + } as const + const paid = await dispatchContext.cost.runPaidCall({ + callId: memoryBenchmarkCostCallId(attempt, 'execute', 0), + actor: `agent-knowledge:memory-adapter:${options.adapter.id}`, + model: options.adapter.id, + maximumCharge: { externallyEnforcedMaximumUsd: costUsd }, + execute, + receipt: () => receipt, + receiptFromError: () => ({ + ...receipt, + actualCostUsd: externalCallAttempted ? costUsd : 0, + }), + }) + if (!paid.succeeded) throw paid.error + return paid.value + } +} + +export class MemoryAdapterBenchmarkCleanupError extends AggregateError {} + +function benchmarkMemoryScope( + candidateId: string, + testCase: KnowledgeMemoryBenchmarkCase, + cellId: string, + attemptId: string, + scope: AgentMemoryScope = {}, +): AgentMemoryScope { + return { + ...scope, + namespace: `${scope.namespace ?? 'agent-knowledge-memory-benchmark'}:${attemptId}`, + tags: { + ...(scope.tags ?? {}), + benchmarkCandidateId: candidateId, + benchmarkCaseId: testCase.id, + benchmarkCellId: cellId, + benchmarkAttemptId: attemptId, + }, + } +} diff --git a/src/benchmarks/memory-runner.ts b/src/benchmarks/memory-runner.ts new file mode 100644 index 0000000..3bfd1e4 --- /dev/null +++ b/src/benchmarks/memory-runner.ts @@ -0,0 +1,556 @@ +import { join } from 'node:path' + +import { canonicalJson } from '@tangle-network/agent-eval' + +import { + type CampaignStorage, + type CostLedgerHandle, + createRunCostLedger, + fsCampaignStorage, + resolveRunDir, +} from '@tangle-network/agent-eval/campaign' + +import { stableId } from '../ids' + +import { DEFAULT_MEMORY_RECOVERY_RETRIES_PER_ATTEMPT } from '../memory/attempt-log' + +import { + releaseMemoryAdapterCreatedAfterAbort, + resolveMemoryCleanupTimeoutMs, + runBoundedMemoryLifecycle, +} from '../memory/lifecycle' + +import { acquireAgentMemoryRunLease, type OwnedAgentMemoryRunLease } from '../memory/run-control' + +import type { AgentMemoryAdapter } from '../memory/types' + +import { + assertScopedMemoryBenchmarkAdapter, + memoryAdapterBenchmarkExpectedId, + recoverMemoryAdapterBenchmarkAttempts, +} from './memory-recovery' + +import { + createMemoryAdapterBenchmarkResponder, + MemoryAdapterBenchmarkCleanupError, +} from './memory-responder' + +import { runKnowledgeBenchmarkSuite } from './suite' + +import type { + KnowledgeBenchmarkArtifact, + KnowledgeBenchmarkResponder, + MemoryAdapterBenchmarkCandidate, + MemoryAdapterBenchmarkRankingRow, + RunKnowledgeBenchmarkSuiteResult, + RunMemoryAdapterBenchmarkOptions, + RunMemoryAdapterBenchmarkResult, +} from './types' + +import { formatNumber, normalizeUsd } from './utils' + +import { assertNonEmptyBenchmarkString, assertUniqueNonEmptyStrings } from './validation' + +const MEMORY_ADAPTER_BENCHMARK_IMPLEMENTATION_REF = 'agent-knowledge:memory-adapter-benchmark:v7' + +export async function runMemoryAdapterBenchmark( + options: RunMemoryAdapterBenchmarkOptions, +): Promise { + if (options.candidates.length === 0) + throw new Error('memory adapter benchmark requires candidates') + const allCandidates = [...options.candidates, ...(options.recoveryCandidates ?? [])] + assertUniqueNonEmptyStrings( + allCandidates.map((candidate) => candidate.id), + 'memory adapter candidate id', + ) + for (const candidate of allCandidates) { + assertNonEmptyBenchmarkString(candidate.ref, `memory adapter candidate ${candidate.id} ref`) + if (candidate.adapterId !== undefined) { + assertNonEmptyBenchmarkString( + candidate.adapterId, + `memory adapter candidate ${candidate.id} adapterId`, + ) + } + if (!/^[A-Za-z0-9][A-Za-z0-9._-]*$/.test(candidate.id)) { + throw new Error( + `memory adapter candidate id '${candidate.id}' must be a safe directory segment`, + ) + } + if ( + candidate.adapterCreationCostUsd !== undefined && + (!Number.isFinite(candidate.adapterCreationCostUsd) || candidate.adapterCreationCostUsd < 0) + ) { + throw new Error( + `${candidate.id}: adapterCreationCostUsd must be a non-negative finite number`, + ) + } + if ( + candidate.costUsdPerCase !== undefined && + (!Number.isFinite(candidate.costUsdPerCase) || candidate.costUsdPerCase < 0) + ) { + throw new Error(`${candidate.id}: costUsdPerCase must be a non-negative finite number`) + } + if ( + candidate.recoveryCostUsdPerAttempt !== undefined && + (!Number.isFinite(candidate.recoveryCostUsdPerAttempt) || + candidate.recoveryCostUsdPerAttempt < 0) + ) { + throw new Error( + `${candidate.id}: recoveryCostUsdPerAttempt must be a non-negative finite number`, + ) + } + } + const storage = options.storage ?? fsCampaignStorage() + if (!storage.append) { + throw new Error('memory adapter benchmark requires CampaignStorage.append') + } + const runDir = resolveRunDir(options.runDir, options.repo) + const cleanupTimeoutMs = resolveMemoryCleanupTimeoutMs( + options.cleanupTimeoutMs, + 'memory adapter benchmark', + ) + const maxRecoveryAttempts = options.maxRecoveryAttempts ?? 1_000 + if (!Number.isSafeInteger(maxRecoveryAttempts) || maxRecoveryAttempts <= 0) { + throw new Error('memory adapter benchmark maxRecoveryAttempts must be a positive safe integer') + } + const maxRecoveryRetriesPerAttempt = + options.maxRecoveryRetriesPerAttempt ?? DEFAULT_MEMORY_RECOVERY_RETRIES_PER_ATTEMPT + if (!Number.isSafeInteger(maxRecoveryRetriesPerAttempt) || maxRecoveryRetriesPerAttempt <= 0) { + throw new Error( + 'memory adapter benchmark maxRecoveryRetriesPerAttempt must be a positive safe integer', + ) + } + storage.ensureDir(runDir) + const lease = await acquireAgentMemoryRunLease({ + experimentId: `memory-adapter-benchmark:${runDir}`, + runDir, + storage, + customStorage: options.storage !== undefined, + lockFileName: 'memory-adapter-benchmark.lock', + label: 'memory adapter benchmark', + controllerMode: options.controllerMode, + acquireRunLease: options.acquireRunLease, + }) + let result: RunMemoryAdapterBenchmarkResult | undefined + let primaryError: unknown + try { + result = await runOwnedMemoryAdapterBenchmark( + options, + storage, + runDir, + lease, + cleanupTimeoutMs, + maxRecoveryAttempts, + maxRecoveryRetriesPerAttempt, + ) + } catch (error) { + primaryError = error + } + let releaseError: unknown + try { + await lease.release() + } catch (error) { + releaseError = error + } + if (primaryError && releaseError) { + throw new AggregateError( + [primaryError, releaseError], + 'memory adapter benchmark failed and its controller lease could not be released', + ) + } + if (primaryError) throw primaryError + if (releaseError) throw releaseError + if (!result) throw new Error('memory adapter benchmark produced no result') + return result +} + +async function runOwnedMemoryAdapterBenchmark( + options: RunMemoryAdapterBenchmarkOptions, + storage: CampaignStorage, + runDir: string, + lease: OwnedAgentMemoryRunLease, + cleanupTimeoutMs: number, + maxRecoveryAttempts: number, + maxRecoveryRetriesPerAttempt: number, +): Promise { + const maxConcurrency = options.maxConcurrency ?? 2 + if (!Number.isSafeInteger(maxConcurrency) || maxConcurrency <= 0) { + throw new Error('memory adapter benchmark maxConcurrency must be a positive safe integer') + } + const costCeiling = options.costCeiling ?? options.costLedger?.costCeilingUsd ?? 0 + const costLedger = + options.costLedger ?? + createRunCostLedger({ + storage, + runDir, + costCeilingUsd: costCeiling, + }) + if (costLedger.costCeilingUsd !== costCeiling) { + throw new Error( + 'memory adapter benchmark costCeiling must match the shared cost ledger ceiling', + ) + } + const attemptLogPath = join(runDir, 'memory-adapter-attempts.jsonl') + const recoveryLogPath = join(runDir, 'memory-adapter-recovery-attempts.jsonl') + await recoverMemoryAdapterBenchmarkAttempts({ + candidates: [...options.candidates, ...(options.recoveryCandidates ?? [])], + storage, + attemptLogPath, + lease, + cleanupTimeoutMs, + maxConcurrency, + now: options.now, + runDir, + costLedger, + costPhase: options.costPhase ?? 'memory.adapter-benchmark', + maxRecoveryAttempts, + recoveryLogPath, + maxRecoveryRetriesPerAttempt, + }) + await lease.assertOwned() + const rows: MemoryAdapterBenchmarkRankingRow[] = [] + for (const candidate of options.candidates) { + await lease.assertOwned() + const expectedAdapterId = memoryAdapterBenchmarkExpectedId(candidate) + let adapter: AgentMemoryAdapter | undefined + let adapterPromise: Promise | undefined + let adapterCreationError: unknown + const getAdapter = (): Promise => { + if (!adapterPromise) { + const abortController = new AbortController() + const creation = createMemoryAdapterBenchmarkAdapter({ + candidate, + purpose: 'execute', + signal: abortController.signal, + costLedger, + runDir, + costPhase: options.costPhase ?? 'memory.adapter-benchmark', + }) + releaseMemoryAdapterCreatedAfterAbort({ creation, signal: abortController.signal }) + adapterPromise = runBoundedMemoryLifecycle({ + operation: `${candidate.id}: benchmark execute adapter creation`, + timeoutMs: Math.min(cleanupTimeoutMs, options.dispatchTimeoutMs ?? cleanupTimeoutMs), + abortController, + run: () => creation, + }) + .then((created) => { + adapter = created + if (created.id !== expectedAdapterId) { + throw new Error( + `${candidate.id}: createAdapter returned id '${created.id}', expected '${expectedAdapterId}'`, + ) + } + assertScopedMemoryBenchmarkAdapter(created) + return created + }) + .catch((error) => { + adapterCreationError = error + throw error + }) + } + return adapterPromise + } + const dispatchedExecutions: Promise[] = [] + let run: RunKnowledgeBenchmarkSuiteResult | undefined + let primaryError: unknown + try { + await lease.assertOwned() + let respond: KnowledgeBenchmarkResponder | undefined + run = await runKnowledgeBenchmarkSuite({ + cases: options.cases, + respond(input) { + const operation = getAdapter().then((activeAdapter) => { + respond ??= createMemoryAdapterBenchmarkResponder({ + adapter: activeAdapter, + candidateId: candidate.id, + candidateRef: candidate.ref, + storage, + attemptLogPath, + lease, + cleanupTimeoutMs, + searchLimit: candidate.searchLimit, + scope: candidate.scope, + adapterCreationCostUsd: candidate.adapterCreationCostUsd, + costUsdPerCase: candidate.costUsdPerCase, + recoveryCostUsdPerAttempt: candidate.recoveryCostUsdPerAttempt, + now: options.now, + }) + return respond(input) + }) + dispatchedExecutions.push(operation) + return operation + }, + respondRef: stableId( + 'memory_adapter_benchmark', + canonicalJson({ + implementationRef: MEMORY_ADAPTER_BENCHMARK_IMPLEMENTATION_REF, + candidateRef: candidate.ref, + adapterId: expectedAdapterId, + searchLimit: candidate.searchLimit ?? null, + adapterCreationCostUsd: candidate.adapterCreationCostUsd ?? 0, + costUsdPerCase: candidate.costUsdPerCase ?? 0, + recoveryCostUsdPerAttempt: candidate.recoveryCostUsdPerAttempt ?? 0, + scope: candidate.scope ?? null, + }), + ), + runDir: join(runDir, candidate.id), + storage, + seed: options.seed, + reps: options.reps, + resumable: options.resumable, + costCeiling, + costLedger, + costPhase: `${options.costPhase ?? 'memory.adapter-benchmark'}.${candidate.id}`, + maxConcurrency: options.maxConcurrency, + dispatchTimeoutMs: options.dispatchTimeoutMs, + expectUsage: options.expectUsage ?? 'off', + now: options.now, + }) + } catch (error) { + primaryError = error + } + + const settledExecutions = await Promise.allSettled(dispatchedExecutions) + const dispatchCleanupErrors = settledExecutions.flatMap((settled) => + settled.status === 'rejected' && settled.reason instanceof MemoryAdapterBenchmarkCleanupError + ? [settled.reason] + : [], + ) + if (!primaryError && adapterCreationError) primaryError = adapterCreationError + if (run) { + rows.push({ + rank: 0, + candidateId: candidate.id, + label: candidate.label ?? candidate.id, + adapterId: expectedAdapterId, + scoreMean: run.report.score.mean, + passRate: run.report.dimensions.passed?.mean ?? 0, + totalCases: run.report.totalCases, + totalCells: run.report.totalCells, + cellsFailed: run.report.cellsFailed, + totalCostUsd: run.report.totalCostUsd, + reportJsonPath: run.reportJsonPath, + reportMarkdownPath: run.reportMarkdownPath, + report: run.report, + }) + } + const cleanupErrors: unknown[] = [] + let cleanupOwned = true + try { + await lease.assertOwned() + } catch (error) { + cleanupOwned = false + cleanupErrors.push(error) + } + if (cleanupOwned && adapter && !adapterCreationError) { + const activeAdapter = adapter + try { + await runBoundedMemoryLifecycle({ + operation: `${candidate.id}: benchmark adapter flush`, + timeoutMs: cleanupTimeoutMs, + resource: activeAdapter, + run: () => activeAdapter.flush?.(), + }) + } catch (error) { + cleanupErrors.push(error) + } + } + if (adapter) { + try { + await runBoundedMemoryLifecycle({ + operation: `${candidate.id}: benchmark adapter close`, + timeoutMs: cleanupTimeoutMs, + resource: adapter, + run: () => adapter!.close?.(), + }) + } catch (error) { + cleanupErrors.push(error) + } + } + if (primaryError || dispatchCleanupErrors.length > 0 || cleanupErrors.length > 0) { + const errors = [ + ...(primaryError ? [primaryError] : []), + ...dispatchCleanupErrors, + ...cleanupErrors, + ] + if (errors.length === 1) throw errors[0] + throw new AggregateError(errors, `${candidate.id}: memory adapter benchmark cleanup failed`) + } + await lease.assertOwned() + } + + const costByCandidate = memoryAdapterBenchmarkCostByCandidate(costLedger, runDir, [ + ...options.candidates, + ...(options.recoveryCandidates ?? []), + ]) + const ranked = rows + .map((row) => { + const totalCostUsd = normalizeUsd(costByCandidate.get(row.candidateId) ?? 0) + return { + ...row, + totalCostUsd, + } + }) + .sort( + (a, b) => + Number(a.cellsFailed > 0) - Number(b.cellsFailed > 0) || + b.scoreMean - a.scoreMean || + b.passRate - a.passRate || + a.totalCostUsd - b.totalCostUsd || + a.candidateId.localeCompare(b.candidateId), + ) + .map((row, index) => ({ ...row, rank: index + 1 })) + const rankingJsonPath = join(runDir, 'memory-adapter-ranking.json') + const rankingMarkdownPath = join(runDir, 'memory-adapter-ranking.md') + const unrankedRecoveryCostUsd = normalizeUsd( + (options.recoveryCandidates ?? []).reduce( + (sum, candidate) => sum + (costByCandidate.get(candidate.id) ?? 0), + 0, + ), + ) + const totalCostUsd = normalizeUsd( + [...costByCandidate.values()].reduce((sum, cost) => sum + cost, 0), + ) + storage.write( + rankingJsonPath, + `${JSON.stringify({ totalCostUsd, unrankedRecoveryCostUsd, rows: ranked }, null, 2)}\n`, + ) + storage.write( + rankingMarkdownPath, + renderMemoryAdapterRankingMarkdown(ranked, totalCostUsd, unrankedRecoveryCostUsd), + ) + return { + rows: ranked, + totalCostUsd, + unrankedRecoveryCostUsd, + rankingJsonPath, + rankingMarkdownPath, + attemptLogPath, + recoveryLogPath, + } +} + +async function createMemoryAdapterBenchmarkAdapter(input: { + candidate: MemoryAdapterBenchmarkCandidate + purpose: 'execute' | 'recovery' + signal: AbortSignal + costLedger: CostLedgerHandle + runDir: string + costPhase: string +}): Promise { + const { candidate, purpose, signal, costLedger, runDir, costPhase } = input + const costUsd = candidate.adapterCreationCostUsd ?? 0 + let externalCallAttempted = false + const create = async (): Promise => { + const adapter = await candidate.createAdapter({ + purpose, + signal, + markExternalCall: () => { + externalCallAttempted = true + }, + }) + if (!adapter || typeof adapter !== 'object') { + throw new Error(`${candidate.id}: createAdapter returned no ${purpose} adapter`) + } + return adapter + } + if (costUsd === 0) return create() + + const tags = memoryAdapterCreationCostTags(runDir, candidate.id, purpose) + const generation = costLedger.list({ tags }).length + const receipt = { + model: candidate.id, + inputTokens: 0, + outputTokens: 0, + actualCostUsd: costUsd, + } as const + const paid = await costLedger.runPaidCall({ + callId: memoryAdapterCreationCostCallId(candidate, purpose, generation), + channel: 'driver', + phase: `${costPhase}.${candidate.id}.adapter-${purpose}`, + actor: `agent-knowledge:memory-adapter:${candidate.id}`, + model: candidate.id, + tags, + maximumCharge: { externallyEnforcedMaximumUsd: costUsd }, + execute: create, + receipt: () => ({ + ...receipt, + actualCostUsd: externalCallAttempted ? costUsd : 0, + }), + receiptFromError: () => ({ + ...receipt, + actualCostUsd: externalCallAttempted ? costUsd : 0, + }), + }) + if (!paid.succeeded) throw paid.error + return paid.value +} + +function memoryAdapterCreationCostTags( + runDir: string, + candidateId: string, + purpose: 'execute' | 'recovery', +): Record { + return { + runDir: join(runDir, candidateId), + candidateId, + memoryAdapterCreation: purpose, + } +} + +function memoryAdapterCreationCostCallId( + candidate: MemoryAdapterBenchmarkCandidate, + purpose: 'execute' | 'recovery', + generation: number, +): string { + return stableId( + 'memory_adapter_creation_cost_call', + canonicalJson({ + purpose, + generation, + candidateId: candidate.id, + candidateRef: candidate.ref, + adapterCreationCostUsd: candidate.adapterCreationCostUsd ?? 0, + }), + ) +} + +function memoryAdapterBenchmarkCostByCandidate( + costLedger: CostLedgerHandle, + runDir: string, + candidates: readonly MemoryAdapterBenchmarkCandidate[], +): ReadonlyMap { + const candidateByRunDir = new Map( + candidates.map((candidate) => [join(runDir, candidate.id), candidate.id]), + ) + const totals = new Map() + for (const receipt of costLedger.list()) { + const candidateId = receipt.tags?.runDir + ? candidateByRunDir.get(receipt.tags.runDir) + : undefined + if (!candidateId) continue + totals.set(candidateId, (totals.get(candidateId) ?? 0) + receipt.costUsd) + } + return totals +} + +function renderMemoryAdapterRankingMarkdown( + rows: readonly MemoryAdapterBenchmarkRankingRow[], + totalCostUsd: number, + unrankedRecoveryCostUsd: number, +): string { + return [ + '# Memory Adapter Ranking', + '', + `- total cost: $${formatNumber(totalCostUsd)}`, + `- retired-candidate recovery cost: $${formatNumber(unrankedRecoveryCostUsd)}`, + '', + '| rank | candidate | adapter | cases | cells | failed | mean score | pass rate | cost |', + '| ---: | --- | --- | ---: | ---: | ---: | ---: | ---: | ---: |', + ...rows.map( + (row) => + `| ${row.rank} | ${row.label} | ${row.adapterId} | ${row.totalCases} | ${row.totalCells} | ${row.cellsFailed} | ${formatNumber(row.scoreMean)} | ${formatNumber(row.passRate)} | $${formatNumber(row.totalCostUsd)} |`, + ), + '', + ].join('\n') +} diff --git a/src/benchmarks/metrics.ts b/src/benchmarks/metrics.ts new file mode 100644 index 0000000..4308e19 --- /dev/null +++ b/src/benchmarks/metrics.ts @@ -0,0 +1,117 @@ +import type { CampaignResult } from '@tangle-network/agent-eval/campaign' + +import type { + KnowledgeBenchmarkDistribution, + KnowledgeBenchmarkReport, + KnowledgeBenchmarkScenario, + KnowledgeBenchmarkSliceSummary, +} from './types' + +import { formatNumber, mean } from './utils' + +export function summarizeKnowledgeBenchmarkCampaign(input: { + scenarios: readonly KnowledgeBenchmarkScenario[] + campaign: CampaignResult +}): KnowledgeBenchmarkReport { + const scenariosById = new Map(input.scenarios.map((scenario) => [scenario.id, scenario])) + const rows = input.campaign.cells.map((cell) => { + const score = Object.values(cell.judgeScores)[0] + const scenario = scenariosById.get(cell.scenarioId) + return { + cell, + scenario, + composite: score?.composite ?? 0, + passed: (score?.dimensions.passed ?? 0) >= 1, + dimensions: score?.dimensions ?? {}, + } + }) + const successful = rows.filter((row) => !row.cell.error) + return { + totalCases: input.scenarios.length, + totalCells: input.campaign.cells.length, + cellsFailed: input.campaign.aggregates.cellsFailed, + cellsCached: input.campaign.aggregates.cellsCached, + totalCostUsd: input.campaign.aggregates.totalCostUsd, + bySplit: summarizeSlices(successful, (row) => row.scenario?.splitTag ?? 'unknown'), + byFamily: summarizeSlices(successful, (row) => row.scenario?.family ?? 'unknown'), + byTaskKind: summarizeSlices(successful, (row) => row.scenario?.taskKind ?? 'unknown'), + dimensions: summarizeDimensions(successful.map((row) => row.dimensions)), + score: distribution(successful.map((row) => row.composite)), + } +} + +function summarizeDimensions( + rows: Array>, +): Record { + const values = new Map() + for (const row of rows) { + for (const [key, value] of Object.entries(row)) { + if (!Number.isFinite(value)) continue + const list = values.get(key) ?? [] + list.push(value) + values.set(key, list) + } + } + return Object.fromEntries([...values.entries()].map(([key, vals]) => [key, distribution(vals)])) +} + +function summarizeSlices( + rows: T[], + keyOf: (row: T) => string, +): Record { + const grouped = new Map() + for (const row of rows) { + const key = keyOf(row) + const list = grouped.get(key) ?? [] + list.push(row) + grouped.set(key, list) + } + return Object.fromEntries( + [...grouped.entries()].map(([key, list]) => { + const withShape = list as Array<{ composite: number; passed: boolean }> + return [ + key, + { + n: list.length, + meanScore: mean(withShape.map((row) => row.composite)), + passRate: mean(withShape.map((row) => (row.passed ? 1 : 0))), + score: distribution(withShape.map((row) => row.composite)), + }, + ] + }), + ) +} + +function distribution(values: readonly number[]): KnowledgeBenchmarkDistribution { + const finite = [...values].filter(Number.isFinite).sort((a, b) => a - b) + if (finite.length === 0) return { n: 0, min: 0, mean: 0, median: 0, p90: 0, max: 0 } + return { + n: finite.length, + min: finite[0]!, + mean: mean(finite), + median: percentile(finite, 0.5), + p90: percentile(finite, 0.9), + max: finite[finite.length - 1]!, + } +} + +function percentile(sortedValues: readonly number[], p: number): number { + if (sortedValues.length === 0) return 0 + const index = Math.min( + sortedValues.length - 1, + Math.max(0, Math.ceil(p * sortedValues.length) - 1), + ) + return sortedValues[index]! +} + +export function renderSliceTable(slices: Record): string { + const rows = Object.entries(slices).map( + ([key, slice]) => + `| ${key} | ${slice.n} | ${formatNumber(slice.meanScore)} | ${formatNumber(slice.passRate)} | ${formatNumber(slice.score.p90)} |`, + ) + return [ + '| slice | n | mean score | pass rate | score p90 |', + '| --- | ---: | ---: | ---: | ---: |', + ...(rows.length ? rows : ['| none | 0 | 0 | 0 | 0 |']), + ].join('\n') +} diff --git a/src/benchmarks/retrieval.ts b/src/benchmarks/retrieval.ts new file mode 100644 index 0000000..a03b071 --- /dev/null +++ b/src/benchmarks/retrieval.ts @@ -0,0 +1,95 @@ +import type { RetrievalGoldTarget } from '../retrieval-eval' + +import type { + BuildRetrievalBenchmarkCasesFromQrelsOptions, + KnowledgeRetrievalBenchmarkCase, + KnowledgeRetrievalBenchmarkQrel, +} from './types' + +import { compactObject, unique } from './utils' + +export function parseKnowledgeBenchmarkJsonl(text: string): T[] { + return text + .split(/\r?\n/) + .map((line) => line.trim()) + .filter(Boolean) + .map((line, index) => { + try { + return JSON.parse(line) as T + } catch (error) { + throw new Error(`invalid JSONL row ${index + 1}: ${(error as Error).message}`) + } + }) +} + +export function parseKnowledgeBenchmarkQrels(text: string): KnowledgeRetrievalBenchmarkQrel[] { + return text + .split(/\r?\n/) + .map((line) => line.trim()) + .filter((line) => line && !line.startsWith('#')) + .flatMap((line, index) => { + const parts = line.split(/\t|\s+/) + if (parts.length < 3) return [] + const [queryId, maybeZeroOrDocId, maybeDocIdOrScore, maybeScore] = parts + if (!queryId || !maybeZeroOrDocId || !maybeDocIdOrScore) return [] + if (queryId.toLowerCase() === 'qid' || queryId.toLowerCase() === 'query-id') return [] + const documentId = maybeScore === undefined ? maybeZeroOrDocId : maybeDocIdOrScore + const scoreText = maybeScore === undefined ? maybeDocIdOrScore : maybeScore + const score = Number(scoreText) + if (!documentId || !Number.isFinite(score)) { + throw new Error(`invalid qrels row ${index + 1}: expected query id, doc id, score`) + } + return [{ queryId, documentId, score }] + }) +} + +export function buildRetrievalBenchmarkCasesFromQrels( + options: BuildRetrievalBenchmarkCasesFromQrelsOptions, +): KnowledgeRetrievalBenchmarkCase[] { + const qrelsByQuery = new Map() + for (const qrel of options.qrels) { + if (qrel.score <= 0) continue + const list = qrelsByQuery.get(qrel.queryId) ?? [] + list.push(qrel) + qrelsByQuery.set(qrel.queryId, list) + } + + return options.queries.flatMap((query) => { + const qrels = qrelsByQuery.get(query.id) ?? [] + if (qrels.length === 0) return [] + const split = query.split ?? options.splitOf?.(query.id) + const expected = qrels.map((qrel) => + options.documentTarget + ? options.documentTarget(qrel.documentId, qrel) + : defaultDocumentTarget(qrel.documentId, options.targetKind ?? 'page'), + ) + return [ + compactObject({ + id: `${options.benchmarkId}:${query.id}`, + family: options.family, + taskKind: 'retrieval' as const, + query: query.text, + expected, + k: options.k, + split, + tags: unique([...(options.tags ?? []), ...(query.tags ?? []), ...(split ? [split] : [])]), + source: options.source, + metadata: query.metadata, + }) as KnowledgeRetrievalBenchmarkCase, + ] + }) +} + +function defaultDocumentTarget( + documentId: string, + targetKind: 'page' | 'page-path' | 'source', +): RetrievalGoldTarget { + switch (targetKind) { + case 'page': + return { kind: 'page', pageId: documentId } + case 'page-path': + return { kind: 'page-path', path: documentId } + case 'source': + return { kind: 'source', sourceId: documentId } + } +} diff --git a/src/benchmarks/scoring.ts b/src/benchmarks/scoring.ts new file mode 100644 index 0000000..f767073 --- /dev/null +++ b/src/benchmarks/scoring.ts @@ -0,0 +1,231 @@ +import { + type RetrievalEvalArtifact, + type RetrievalEvalScenario, + scoreRetrievalArtifact, +} from '../retrieval-eval' + +import type { + KnowledgeBenchmarkArtifact, + KnowledgeBenchmarkCase, + KnowledgeBenchmarkEvaluation, + KnowledgeClaimMatcher, + KnowledgeMemoryBenchmarkCase, + KnowledgeRetrievalBenchmarkCase, +} from './types' + +import { mean, unique } from './utils' + +import { isKnowledgeMemoryBenchmarkCase } from './validation' + +export function scoreKnowledgeBenchmarkArtifact( + testCase: KnowledgeBenchmarkCase, + artifact: TArtifact, +): KnowledgeBenchmarkEvaluation { + if (testCase.taskKind === 'retrieval') { + const retrievalArtifact = normalizeRetrievalArtifact(testCase, artifact) + const metrics = scoreRetrievalArtifact(retrievalArtifact, retrievalScenarioForCase(testCase)) + return { + score: metrics.recall, + passed: metrics.recall >= 1, + dimensions: { + recall: metrics.recall, + mrr: metrics.mrr, + ndcg: metrics.ndcg, + precision_at_k: metrics.precisionAtK, + expected_count: metrics.expectedCount, + matched_count: metrics.matchedCount, + }, + notes: `matched ${metrics.matchedCount}/${metrics.expectedCount}; first_hit_rank=${metrics.firstHitRank ?? 'none'}`, + raw: { matchedTargetIds: metrics.matchedTargetIds }, + } + } + if (isKnowledgeMemoryBenchmarkCase(testCase)) { + return scoreMemoryBenchmarkArtifact(testCase, artifact) + } + + const answerArtifact = artifact as KnowledgeBenchmarkArtifact + const text = answerArtifact.text ?? answerArtifact.answer ?? '' + const required = scoreClaims(text, testCase.requiredClaims ?? []) + const forbidden = scoreForbiddenClaims(text, testCase.forbiddenClaims ?? []) + const citation = scoreCitationRecall( + answerArtifact.citedSourceIds ?? [], + testCase.expectedSourceIds ?? [], + ) + const components = [ + required.totalWeight > 0 ? required.recall : undefined, + testCase.expectedSourceIds && testCase.expectedSourceIds.length > 0 ? citation : undefined, + forbidden.safe, + ].filter((value): value is number => value !== undefined) + const score = mean(components) + return { + score, + passed: score >= 1, + dimensions: { + claim_recall: required.recall, + citation_recall: citation, + hallucination_safe: forbidden.safe, + forbidden_claim_rate: forbidden.rate, + required_claim_count: required.total, + matched_claim_count: required.matched, + forbidden_claim_count: forbidden.total, + matched_forbidden_claim_count: forbidden.matched, + }, + notes: `required=${required.matched}/${required.total}; forbidden=${forbidden.matched}/${forbidden.total}; citation_recall=${citation.toFixed(3)}`, + raw: { + matchedRequiredClaimIds: required.matchedIds, + matchedForbiddenClaimIds: forbidden.matchedIds, + }, + } +} + +export function scoreMemoryBenchmarkArtifact( + testCase: KnowledgeMemoryBenchmarkCase, + artifact: TArtifact, +): KnowledgeBenchmarkEvaluation { + const memoryArtifact = artifact as KnowledgeBenchmarkArtifact + const text = [ + memoryArtifact.text, + memoryArtifact.answer, + ...(memoryArtifact.rememberedFacts ?? []), + ] + .filter((part): part is string => typeof part === 'string' && part.length > 0) + .join('\n') + const required = scoreClaims(text, testCase.requiredFacts ?? []) + const forbidden = scoreForbiddenClaims(text, testCase.forbiddenFacts ?? []) + const eventIds = unique([ + ...(memoryArtifact.citedEventIds ?? []), + ...(memoryArtifact.usedMemoryIds ?? []), + ]) + const eventRecall = scoreCitationRecall(eventIds, testCase.expectedEventIds ?? []) + const actorRecall = scoreCitationRecall( + memoryArtifact.actorIds ?? [], + testCase.expectedActorIds ?? [], + ) + const components = [ + required.totalWeight > 0 ? required.recall : undefined, + testCase.expectedEventIds && testCase.expectedEventIds.length > 0 ? eventRecall : undefined, + testCase.expectedActorIds && testCase.expectedActorIds.length > 0 ? actorRecall : undefined, + testCase.forbiddenFacts && testCase.forbiddenFacts.length > 0 ? forbidden.safe : undefined, + ].filter((value): value is number => value !== undefined) + const score = mean(components) + const dimensions: Record = {} + if (required.totalWeight > 0) { + dimensions.memory_fact_recall = required.recall + dimensions.memory_required_fact_count = required.total + dimensions.memory_matched_fact_count = required.matched + } + if (testCase.expectedEventIds && testCase.expectedEventIds.length > 0) { + dimensions.memory_event_recall = eventRecall + } + if (testCase.expectedActorIds && testCase.expectedActorIds.length > 0) { + dimensions.memory_actor_recall = actorRecall + } + if (testCase.forbiddenFacts && testCase.forbiddenFacts.length > 0) { + dimensions.memory_stale_safe = forbidden.safe + dimensions.memory_stale_rate = forbidden.rate + dimensions.memory_forbidden_fact_count = forbidden.total + dimensions.memory_matched_forbidden_fact_count = forbidden.matched + } + return { + score, + passed: score >= 1, + dimensions, + applicableDimensions: Object.keys(dimensions), + notes: `memory required=${required.matched}/${required.total}; stale=${forbidden.matched}/${forbidden.total}; event_recall=${eventRecall.toFixed(3)}; actor_recall=${actorRecall.toFixed(3)}`, + raw: { + matchedRequiredFactIds: required.matchedIds, + matchedForbiddenFactIds: forbidden.matchedIds, + citedEventIds: eventIds, + actorIds: memoryArtifact.actorIds ?? [], + }, + } +} + +function retrievalScenarioForCase( + testCase: KnowledgeRetrievalBenchmarkCase, +): RetrievalEvalScenario { + return { + id: testCase.id, + kind: 'retrieval-eval', + query: testCase.query, + expected: testCase.expected, + ...(testCase.k !== undefined ? { k: testCase.k } : {}), + } +} + +function normalizeRetrievalArtifact( + testCase: KnowledgeRetrievalBenchmarkCase, + artifact: TArtifact, +): RetrievalEvalArtifact { + const maybe = artifact as Partial & KnowledgeBenchmarkArtifact + const hits = maybe.hits ?? [] + if (Array.isArray(maybe.hits) && maybe.query && maybe.requestedK !== undefined) { + return maybe as RetrievalEvalArtifact + } + return { + config: {}, + query: testCase.query, + requestedK: testCase.k ?? Math.max(1, hits.length), + hits, + durationMs: maybe.durationMs ?? 0, + ...(maybe.costUsd !== undefined ? { costUsd: maybe.costUsd } : {}), + ...(maybe.metadata ? { metadata: maybe.metadata } : {}), + } +} + +function scoreClaims(text: string, claims: readonly KnowledgeClaimMatcher[]) { + let matched = 0 + let matchedWeight = 0 + let totalWeight = 0 + const matchedIds: string[] = [] + const haystack = text.toLowerCase() + for (const claim of claims) { + if ( + !claim.id.trim() || + claim.anyOf.length === 0 || + claim.anyOf.some((value) => !value.trim()) + ) { + throw new Error( + 'claim matchers require a non-empty id and at least one non-empty alternative', + ) + } + const weight = claim.weight ?? 1 + if (!Number.isFinite(weight) || weight <= 0) { + throw new Error(`claim matcher ${claim.id} weight must be a positive finite number`) + } + totalWeight += weight + if (claim.anyOf.some((fragment) => haystack.includes(fragment.toLowerCase()))) { + matched += 1 + matchedWeight += weight + matchedIds.push(claim.id) + } + } + return { + total: claims.length, + matched, + totalWeight, + recall: totalWeight === 0 ? 1 : matchedWeight / totalWeight, + matchedIds, + } +} + +function scoreForbiddenClaims(text: string, claims: readonly KnowledgeClaimMatcher[]) { + const matched = scoreClaims(text, claims) + return { + total: claims.length, + matched: matched.matched, + matchedIds: matched.matchedIds, + rate: claims.length === 0 ? 0 : matched.matched / claims.length, + safe: matched.matched === 0 ? 1 : 0, + } +} + +function scoreCitationRecall( + citedSourceIds: readonly string[], + expectedSourceIds: readonly string[], +): number { + if (expectedSourceIds.length === 0) return 1 + const cited = new Set(citedSourceIds) + const matched = expectedSourceIds.filter((sourceId) => cited.has(sourceId)).length + return matched / expectedSourceIds.length +} diff --git a/src/benchmarks/suite.ts b/src/benchmarks/suite.ts new file mode 100644 index 0000000..15f7f82 --- /dev/null +++ b/src/benchmarks/suite.ts @@ -0,0 +1,180 @@ +import { join } from 'node:path' + +import { canonicalJson } from '@tangle-network/agent-eval' + +import { + fsCampaignStorage, + type JudgeConfig, + type RunCampaignOptions, + runCampaign, +} from '@tangle-network/agent-eval/campaign' + +import { stableId } from '../ids' + +import { renderSliceTable, summarizeKnowledgeBenchmarkCampaign } from './metrics' + +import { scoreKnowledgeBenchmarkArtifact } from './scoring' + +import type { + KnowledgeBenchmarkArtifact, + KnowledgeBenchmarkCase, + KnowledgeBenchmarkReport, + KnowledgeBenchmarkScenario, + KnowledgeBenchmarkSplit, + RunKnowledgeBenchmarkSuiteOptions, + RunKnowledgeBenchmarkSuiteResult, +} from './types' + +import { compactObject, formatNumber, unique } from './utils' + +import { assertKnowledgeBenchmarkCases, assertNonEmptyBenchmarkString } from './validation' + +const KNOWLEDGE_BENCHMARK_IMPLEMENTATION_REF = 'agent-knowledge:benchmark-suite:v2' + +export async function runKnowledgeBenchmarkSuite( + options: RunKnowledgeBenchmarkSuiteOptions, +): Promise> { + assertKnowledgeBenchmarkCases(options.cases) + if (options.respondRef !== undefined) { + assertNonEmptyBenchmarkString(options.respondRef, 'knowledge benchmark respondRef') + } else if (options.resumable !== false) { + throw new Error('knowledge benchmark respondRef is required when resumable is enabled') + } + const storage = options.storage ?? fsCampaignStorage() + const costCeiling = options.costCeiling ?? options.costLedger?.costCeilingUsd ?? 0 + if (options.costLedger && options.costLedger.costCeilingUsd !== costCeiling) { + throw new Error('knowledge benchmark costCeiling must match the shared cost ledger ceiling') + } + const scenarios = buildKnowledgeBenchmarkScenarios(options.cases, options.splits) + const dispatch: RunCampaignOptions['dispatch'] = async ( + scenario, + context, + ) => { + const artifact = await options.respond({ case: scenario.case, scenario, context }) + return artifact + } + const campaign = await runCampaign({ + scenarios, + dispatch, + dispatchRef: stableId( + 'knowledge_benchmark', + canonicalJson({ + implementationRef: KNOWLEDGE_BENCHMARK_IMPLEMENTATION_REF, + respondRef: options.respondRef ?? 'non-resumable', + }), + ), + judges: [knowledgeBenchmarkJudge()], + runDir: options.runDir, + repo: options.repo, + seed: options.seed, + reps: options.reps, + resumable: options.resumable, + costCeiling, + costLedger: options.costLedger, + costPhase: options.costPhase, + maxConcurrency: options.maxConcurrency, + dispatchTimeoutMs: options.dispatchTimeoutMs, + expectUsage: options.expectUsage ?? 'off', + storage, + now: options.now, + }) + const report = summarizeKnowledgeBenchmarkCampaign({ scenarios, campaign }) + const reportJsonPath = join(campaign.runDir, 'knowledge-benchmark-report.json') + const reportMarkdownPath = join(campaign.runDir, 'knowledge-benchmark-report.md') + storage.write(reportJsonPath, `${JSON.stringify(report, null, 2)}\n`) + storage.write(reportMarkdownPath, renderKnowledgeBenchmarkReportMarkdown(report)) + return { + scenarios, + campaign, + report, + reportJsonPath, + reportMarkdownPath, + } +} + +export function renderKnowledgeBenchmarkReportMarkdown(report: KnowledgeBenchmarkReport): string { + return [ + '# Knowledge Benchmark Report', + '', + `- cases: ${report.totalCases}`, + `- cells: ${report.totalCells} total, ${report.cellsFailed} failed, ${report.cellsCached} cached`, + `- cost: $${formatNumber(report.totalCostUsd)}`, + `- score: mean ${formatNumber(report.score.mean)}, median ${formatNumber(report.score.median)}, p90 ${formatNumber(report.score.p90)}, n=${report.score.n}`, + '', + '## Task Kinds', + '', + renderSliceTable(report.byTaskKind), + '', + '## Splits', + '', + renderSliceTable(report.bySplit), + '', + '## Dimensions', + '', + '| dimension | n | mean | p90 |', + '| --- | ---: | ---: | ---: |', + ...Object.entries(report.dimensions) + .sort(([a], [b]) => a.localeCompare(b)) + .map( + ([key, dist]) => + `| ${key} | ${dist.n} | ${formatNumber(dist.mean)} | ${formatNumber(dist.p90)} |`, + ), + '', + ].join('\n') +} + +export function buildKnowledgeBenchmarkScenarios( + cases: readonly KnowledgeBenchmarkCase[], + splits?: readonly KnowledgeBenchmarkSplit[], +): KnowledgeBenchmarkScenario[] { + const splitSet = splits ? new Set(splits) : null + return cases.flatMap((testCase) => { + const splitTag = testCase.split ?? 'dev' + if (splitSet && !splitSet.has(splitTag)) return [] + return [ + compactObject({ + id: testCase.id, + kind: 'knowledge-benchmark' as const, + family: testCase.family, + taskKind: testCase.taskKind, + splitTag, + tags: unique([splitTag, ...(testCase.tags ?? [])]), + case: compactObject(testCase), + }) as KnowledgeBenchmarkScenario, + ] + }) +} + +export function knowledgeBenchmarkJudge(): JudgeConfig< + TArtifact, + KnowledgeBenchmarkScenario +> { + return { + name: 'knowledge-benchmark', + judgeVersion: 'agent-knowledge:knowledge-benchmark:v2', + dimensions: [ + { key: 'score', description: 'primary knowledge benchmark score' }, + { key: 'passed', description: '1 when the benchmark case passes' }, + { key: 'claim_recall', description: 'required claim coverage' }, + { key: 'citation_recall', description: 'expected citation/source coverage' }, + { key: 'hallucination_safe', description: '1 when no forbidden claim appears' }, + { key: 'memory_fact_recall', description: 'current memory fact coverage' }, + { key: 'memory_event_recall', description: 'expected memory event/source coverage' }, + { key: 'memory_stale_safe', description: '1 when obsolete memory is not reused' }, + { key: 'memory_actor_recall', description: 'expected speaker/user attribution coverage' }, + ], + appliesTo: (scenario) => scenario.kind === 'knowledge-benchmark', + score({ artifact, scenario }) { + const evaluation = scoreKnowledgeBenchmarkArtifact(scenario.case, artifact) + return { + dimensions: { + score: evaluation.score, + passed: evaluation.passed ? 1 : 0, + ...evaluation.dimensions, + }, + composite: evaluation.score, + notes: evaluation.notes, + } + }, + } +} diff --git a/src/benchmarks/types.ts b/src/benchmarks/types.ts new file mode 100644 index 0000000..871c87d --- /dev/null +++ b/src/benchmarks/types.ts @@ -0,0 +1,347 @@ +import type { + CampaignResult, + CampaignStorage, + CostLedgerHandle, + DispatchContext, + Scenario, +} from '@tangle-network/agent-eval/campaign' + +import type { AgentMemoryAcquireRunLease, AgentMemoryControllerMode } from '../memory/run-control' + +import type { AgentMemoryAdapter, AgentMemoryScope } from '../memory/types' + +import type { RetrievalGoldTarget, RetrievedKnowledgeHit } from '../retrieval-eval' + +export type KnowledgeBenchmarkTaskKind = + | 'retrieval' + | 'rag-answer' + | 'hallucination' + | 'kb-improvement' + | 'memory-ingest' + | 'memory-recall' + | 'memory-temporal' + | 'memory-update' + | 'memory-forgetting' + | 'memory-reasoning' + | 'memory-summarization' + | 'memory-recommendation' + | 'memory-multiparty' + +export type KnowledgeAnswerBenchmarkTaskKind = 'rag-answer' | 'hallucination' | 'kb-improvement' + +export type KnowledgeMemoryBenchmarkTaskKind = Exclude< + KnowledgeBenchmarkTaskKind, + 'retrieval' | KnowledgeAnswerBenchmarkTaskKind +> + +export type KnowledgeBenchmarkFamily = + | 'beir' + | 'mteb-retrieval' + | 'msmarco' + | 'trec-dl' + | 'miracl' + | 'lotte' + | 'bright' + | 'crag' + | 'hotpotqa' + | 'kilt' + | 'ragtruth' + | 'faithbench' + | 'locomo' + | 'longmemeval' + | 'longmemeval-v2' + | 'memora' + | 'memoryagentbench' + | 'memorybank' + | 'groupmembench' + | 'first-party' + | 'custom' + +export type KnowledgeBenchmarkSplit = 'search' | 'dev' | 'holdout' | string + +export interface KnowledgeBenchmarkSource { + name?: string + url?: string + version?: string + license?: string + citation?: string +} + +export interface KnowledgeBenchmarkSpec { + id: string + family: KnowledgeBenchmarkFamily + taskKind: KnowledgeBenchmarkTaskKind + primaryMetrics: readonly string[] + adapter: string + notes: string +} + +export interface KnowledgeBenchmarkCaseBase { + id: string + family: KnowledgeBenchmarkFamily | string + taskKind: KnowledgeBenchmarkTaskKind + split?: KnowledgeBenchmarkSplit + tags?: readonly string[] + source?: KnowledgeBenchmarkSource + metadata?: Record +} + +export interface KnowledgeRetrievalBenchmarkCase extends KnowledgeBenchmarkCaseBase { + taskKind: 'retrieval' + query: string + expected: RetrievalGoldTarget | readonly RetrievalGoldTarget[] + k?: number +} + +export interface KnowledgeClaimMatcher { + id: string + anyOf: readonly string[] + weight?: number +} + +export interface KnowledgeMemoryEvent { + id: string + text: string + actorId?: string + sessionId?: string + timestamp?: string + metadata?: Record +} + +export interface KnowledgeMemoryFactMatcher extends KnowledgeClaimMatcher { + sourceEventIds?: readonly string[] + validAt?: string + obsolete?: boolean +} + +export interface KnowledgeAnswerBenchmarkCase extends KnowledgeBenchmarkCaseBase { + taskKind: KnowledgeAnswerBenchmarkTaskKind + prompt: string + requiredClaims?: readonly KnowledgeClaimMatcher[] + forbiddenClaims?: readonly KnowledgeClaimMatcher[] + expectedSourceIds?: readonly string[] + referenceAnswer?: string +} + +export interface KnowledgeMemoryBenchmarkCase extends KnowledgeBenchmarkCaseBase { + taskKind: KnowledgeMemoryBenchmarkTaskKind + events: readonly KnowledgeMemoryEvent[] + prompt: string + requiredFacts?: readonly KnowledgeMemoryFactMatcher[] + forbiddenFacts?: readonly KnowledgeMemoryFactMatcher[] + expectedEventIds?: readonly string[] + expectedActorIds?: readonly string[] + referenceAnswer?: string +} + +export type KnowledgeBenchmarkCase = + | KnowledgeRetrievalBenchmarkCase + | KnowledgeAnswerBenchmarkCase + | KnowledgeMemoryBenchmarkCase + +export interface KnowledgeBenchmarkArtifact { + answer?: string + text?: string + hits?: readonly RetrievedKnowledgeHit[] + citedSourceIds?: readonly string[] + rememberedFacts?: readonly string[] + citedEventIds?: readonly string[] + usedMemoryIds?: readonly string[] + actorIds?: readonly string[] + /** Informational copy. Billable responders account through context.cost.runPaidCall. */ + costUsd?: number + durationMs?: number + metadata?: Record +} + +export interface KnowledgeBenchmarkEvaluation { + score: number + passed: boolean + dimensions: Record + /** Dimensions for which this case declared an actual target. */ + applicableDimensions?: readonly string[] + notes: string + raw: Record +} + +export interface KnowledgeBenchmarkScenario extends Scenario { + kind: 'knowledge-benchmark' + family: KnowledgeBenchmarkFamily | string + taskKind: KnowledgeBenchmarkTaskKind + splitTag: KnowledgeBenchmarkSplit + case: KnowledgeBenchmarkCase +} + +export type KnowledgeBenchmarkResponder = (input: { + case: KnowledgeBenchmarkCase + scenario: KnowledgeBenchmarkScenario + context: DispatchContext +}) => Promise | TArtifact + +export interface RunKnowledgeBenchmarkSuiteOptions { + cases: readonly KnowledgeBenchmarkCase[] + respond: KnowledgeBenchmarkResponder + /** Versioned identity for the model, prompt, retrieval, and runtime behavior. */ + respondRef?: string + runDir: string + splits?: readonly KnowledgeBenchmarkSplit[] + repo?: string + seed?: number + reps?: number + resumable?: boolean + costCeiling?: number + /** Shared across nested benchmark suites when an outer run owns spend. */ + costLedger?: CostLedgerHandle + costPhase?: string + maxConcurrency?: number + dispatchTimeoutMs?: number + expectUsage?: 'assert' | 'warn' | 'off' + storage?: CampaignStorage + now?: () => Date +} + +export interface KnowledgeBenchmarkDistribution { + n: number + min: number + mean: number + median: number + p90: number + max: number +} + +export interface KnowledgeBenchmarkSliceSummary { + n: number + meanScore: number + passRate: number + score: KnowledgeBenchmarkDistribution +} + +export interface KnowledgeBenchmarkReport { + totalCases: number + totalCells: number + cellsFailed: number + cellsCached: number + totalCostUsd: number + bySplit: Record + byFamily: Record + byTaskKind: Record + dimensions: Record + score: KnowledgeBenchmarkDistribution +} + +export interface RunKnowledgeBenchmarkSuiteResult { + scenarios: readonly KnowledgeBenchmarkScenario[] + campaign: CampaignResult + report: KnowledgeBenchmarkReport + reportJsonPath: string + reportMarkdownPath: string +} + +export interface MemoryAdapterBenchmarkCandidate { + id: string + /** Versioned adapter and configuration identity used by resumable caches. */ + ref: string + /** Expected adapter.id. Defaults to candidate id and permits lazy no-work resume. */ + adapterId?: string + label?: string + /** Local construction is free; call markExternalCall before billable provisioning or reconnects. */ + createAdapter: (input: { + purpose: 'execute' | 'recovery' + signal: AbortSignal + markExternalCall(): void + }) => AgentMemoryAdapter | Promise + /** Conservative charge for one billable adapter provisioning or reconnect call. */ + adapterCreationCostUsd?: number + searchLimit?: number + costUsdPerCase?: number + /** Conservative extra provider charge for recovering one interrupted case. */ + recoveryCostUsdPerAttempt?: number + scope?: AgentMemoryScope +} + +export interface RunMemoryAdapterBenchmarkOptions { + cases: readonly KnowledgeMemoryBenchmarkCase[] + candidates: readonly MemoryAdapterBenchmarkCandidate[] + /** Retired candidates retained only so interrupted scopes can be cleaned on resume. */ + recoveryCandidates?: readonly MemoryAdapterBenchmarkCandidate[] + runDir: string + storage?: CampaignStorage + repo?: string + seed?: number + reps?: number + resumable?: boolean + costCeiling?: number + /** Shared with nested benchmark suites so the dollar limit applies to the whole comparison. */ + costLedger?: CostLedgerHandle + costPhase?: string + maxConcurrency?: number + dispatchTimeoutMs?: number + cleanupTimeoutMs?: number + /** Refuse a damaged run with more unfinished attempts than this. Default 1000. */ + maxRecoveryAttempts?: number + /** Bound repeated provider cleanup after process crashes. Default 3 per attempt. */ + maxRecoveryRetriesPerAttempt?: number + expectUsage?: 'assert' | 'warn' | 'off' + now?: () => Date + /** Required with custom storage when all controllers are confined to one process. */ + controllerMode?: AgentMemoryControllerMode + /** Required for distributed controllers that share custom storage. */ + acquireRunLease?: AgentMemoryAcquireRunLease +} + +export interface MemoryAdapterBenchmarkRankingRow { + rank: number + candidateId: string + label: string + adapterId: string + scoreMean: number + passRate: number + totalCases: number + totalCells: number + cellsFailed: number + totalCostUsd: number + reportJsonPath: string + reportMarkdownPath: string + report: KnowledgeBenchmarkReport +} + +export interface RunMemoryAdapterBenchmarkResult { + rows: readonly MemoryAdapterBenchmarkRankingRow[] + totalCostUsd: number + /** Recovery spend for retired candidates, excluded from ranking rows but included in totalCostUsd. */ + unrankedRecoveryCostUsd: number + rankingJsonPath: string + rankingMarkdownPath: string + attemptLogPath: string + recoveryLogPath: string +} + +export interface KnowledgeRetrievalBenchmarkQuery { + id: string + text: string + split?: KnowledgeBenchmarkSplit + tags?: readonly string[] + metadata?: Record +} + +export interface KnowledgeRetrievalBenchmarkQrel { + queryId: string + documentId: string + score: number +} + +export interface BuildRetrievalBenchmarkCasesFromQrelsOptions { + benchmarkId: string + family: KnowledgeBenchmarkFamily | string + queries: readonly KnowledgeRetrievalBenchmarkQuery[] + qrels: readonly KnowledgeRetrievalBenchmarkQrel[] + source?: KnowledgeBenchmarkSource + tags?: readonly string[] + k?: number + targetKind?: 'page' | 'page-path' | 'source' + documentTarget?: ( + documentId: string, + qrel: KnowledgeRetrievalBenchmarkQrel, + ) => RetrievalGoldTarget + splitOf?: (queryId: string) => KnowledgeBenchmarkSplit +} diff --git a/src/benchmarks/utils.ts b/src/benchmarks/utils.ts new file mode 100644 index 0000000..b764642 --- /dev/null +++ b/src/benchmarks/utils.ts @@ -0,0 +1,28 @@ +export function mean(values: readonly number[]): number { + const finite = values.filter(Number.isFinite) + if (finite.length === 0) return 0 + return finite.reduce((sum, value) => sum + value, 0) / finite.length +} + +export function unique(values: readonly string[]): string[] { + return [...new Set(values.filter(Boolean))] +} + +export function formatNumber(value: number): string { + if (!Number.isFinite(value)) return '0' + return value.toFixed(value === 0 || Math.abs(value) >= 10 ? 0 : 3) +} + +export function normalizeUsd(value: number): number { + return Number(value.toFixed(12)) +} + +export function compactObject(value: unknown): unknown { + if (Array.isArray(value)) return value.map(compactObject) + if (!value || typeof value !== 'object') return value + return Object.fromEntries( + Object.entries(value as Record) + .filter(([, entry]) => entry !== undefined) + .map(([key, entry]) => [key, compactObject(entry)]), + ) +} diff --git a/src/benchmarks/validation.ts b/src/benchmarks/validation.ts new file mode 100644 index 0000000..6d2f0ff --- /dev/null +++ b/src/benchmarks/validation.ts @@ -0,0 +1,83 @@ +import type { + KnowledgeBenchmarkCase, + KnowledgeClaimMatcher, + KnowledgeMemoryBenchmarkCase, + KnowledgeMemoryFactMatcher, +} from './types' + +export function isKnowledgeMemoryBenchmarkCase( + testCase: KnowledgeBenchmarkCase, +): testCase is KnowledgeMemoryBenchmarkCase { + return testCase.taskKind.startsWith('memory-') +} + +export function assertKnowledgeBenchmarkCases(cases: readonly KnowledgeBenchmarkCase[]): void { + if (cases.length === 0) throw new Error('knowledge benchmark requires cases') + assertUniqueNonEmptyStrings( + cases.map((testCase) => testCase.id), + 'knowledge benchmark case id', + ) + for (const testCase of cases) { + if (typeof testCase.family !== 'string' || !testCase.family.trim()) { + throw new Error(`knowledge benchmark case ${testCase.id} requires a family`) + } + if (testCase.taskKind === 'retrieval') continue + if (isKnowledgeMemoryBenchmarkCase(testCase)) { + assertUniqueNonEmptyStrings( + testCase.events.map((event) => event.id), + `${testCase.id} memory event id`, + ) + for (const event of testCase.events) { + assertNonEmptyBenchmarkString(event.text, `${testCase.id} memory event ${event.id} text`) + } + assertClaimMatchers(testCase.requiredFacts ?? [], `${testCase.id} requiredFacts`) + assertClaimMatchers(testCase.forbiddenFacts ?? [], `${testCase.id} forbiddenFacts`) + assertUniqueNonEmptyStrings( + testCase.expectedEventIds ?? [], + `${testCase.id} expected event id`, + ) + assertUniqueNonEmptyStrings( + testCase.expectedActorIds ?? [], + `${testCase.id} expected actor id`, + ) + } else { + assertClaimMatchers(testCase.requiredClaims ?? [], `${testCase.id} requiredClaims`) + assertClaimMatchers(testCase.forbiddenClaims ?? [], `${testCase.id} forbiddenClaims`) + assertUniqueNonEmptyStrings(testCase.expectedSourceIds ?? [], `${testCase.id} source id`) + } + } +} + +function assertClaimMatchers(claims: readonly KnowledgeClaimMatcher[], label: string): void { + assertUniqueNonEmptyStrings( + claims.map((claim) => claim.id), + `${label} matcher id`, + ) + for (const claim of claims) { + if (claim.anyOf.length === 0) throw new Error(`${label} matcher ${claim.id} requires anyOf`) + assertUniqueNonEmptyStrings(claim.anyOf, `${label} matcher ${claim.id} anyOf`) + if (claim.weight !== undefined && (!Number.isFinite(claim.weight) || claim.weight <= 0)) { + throw new Error(`${label} matcher ${claim.id} weight must be a positive finite number`) + } + const sourceEventIds = (claim as Partial).sourceEventIds + if (sourceEventIds !== undefined) { + assertUniqueNonEmptyStrings(sourceEventIds, `${label} matcher ${claim.id} source event id`) + } + } +} + +export function assertUniqueNonEmptyStrings(values: readonly string[], label: string): void { + const seen = new Set() + for (const value of values) { + assertNonEmptyBenchmarkString(value, label) + if (seen.has(value)) throw new Error(`duplicate ${label}: ${value}`) + seen.add(value) + } +} + +export function assertNonEmptyBenchmarkString( + value: unknown, + label: string, +): asserts value is string { + if (typeof value !== 'string' || !value.trim()) throw new Error(`${label} must be non-empty`) +} diff --git a/src/memory/experiment.ts b/src/memory/experiment.ts index a2a9db0..71b9b73 100644 --- a/src/memory/experiment.ts +++ b/src/memory/experiment.ts @@ -1,1852 +1,21 @@ -import { randomUUID } from 'node:crypto' -import { join } from 'node:path' -import { canonicalJson } from '@tangle-network/agent-eval' -import { - type CampaignResult, - type CampaignStorage, - type CostLedgerHandle, - createRunCostLedger, - type DispatchContext, - fsCampaignStorage, - type JudgeConfig, - resolveRunDir, - runCampaign, - type Scenario, -} from '@tangle-network/agent-eval/campaign' -import { - type KnowledgeBenchmarkFamily, - type KnowledgeBenchmarkSplit, - type KnowledgeMemoryBenchmarkCase, - type KnowledgeMemoryBenchmarkTaskKind, - type KnowledgeMemoryEvent, - type KnowledgeMemoryFactMatcher, - scoreMemoryBenchmarkArtifact, -} from '../benchmarks/index' -import { stableId } from '../ids' -import { - appendAttemptJournalEvent, - assertNoInterruptedPaidCalls, - DEFAULT_MEMORY_RECOVERY_RETRIES_PER_ATTEMPT, - hasSettledPaidCall, - readActiveAttemptJournal, - reconcileInterruptedMemoryPaidCalls, - reserveRecoveryAttempts, -} from './attempt-log' -import { - type AgentMemoryBranch, - type AgentMemoryBranchSnapshot, - type AgentMemorySharingPolicy, - createAgentMemoryBranch, -} from './branch' -import { - createMemoryExecutionPool, - memoryRecoveryDelayMs, - releaseMemoryAdapterCreatedAfterAbort, - resolveMemoryCleanupTimeoutMs, - runBoundedMemoryLifecycle, - sleepForMemoryRecovery, -} from './lifecycle' -import { - type AgentMemoryAcquireRunLease, - type AgentMemoryControllerMode, - type AgentMemoryRunLease, - acquireAgentMemoryRunLease, - type OwnedAgentMemoryRunLease, -} from './run-control' -import type { AgentMemoryAdapter, AgentMemoryScope, AgentMemoryWriteInput } from './types' - -const MEMORY_EXPERIMENT_IMPLEMENTATION_REF = 'agent-knowledge:memory-experiment:v6' - -export interface AgentMemorySequenceProbe { - id: string - query: string - scope?: AgentMemoryScope - limit?: number - taskKind?: KnowledgeMemoryBenchmarkTaskKind - requiredFacts?: readonly KnowledgeMemoryFactMatcher[] - forbiddenFacts?: readonly KnowledgeMemoryFactMatcher[] - expectedEventIds?: readonly string[] - expectedActorIds?: readonly string[] - referenceAnswer?: string -} - -export interface AgentMemorySequenceStep { - id: string - instruction?: string - scope?: AgentMemoryScope - writes?: readonly AgentMemoryWriteInput[] - parallelWrites?: boolean - probes?: readonly AgentMemorySequenceProbe[] - parallelProbes?: boolean - metadata?: Record -} - -export interface AgentMemorySequence { - id: string - family: KnowledgeBenchmarkFamily | string - split?: KnowledgeBenchmarkSplit - steps: readonly AgentMemorySequenceStep[] - /** Exact scopes a runtime callback may write beyond scopes declared by steps. */ - cleanupScopes?: readonly AgentMemoryScope[] - tags?: readonly string[] - metadata?: Record -} - -export interface BuildAgentMemorySequencesFromBenchmarkCasesOptions { - memoryAgentId?: string - eventScope?: (input: { - event: KnowledgeMemoryEvent - case: KnowledgeMemoryBenchmarkCase - eventIndex: number - }) => AgentMemoryScope - probeScope?: (testCase: KnowledgeMemoryBenchmarkCase) => AgentMemoryScope -} - -export interface AgentMemoryExperimentCandidate { - id: string - label?: string - /** Change when provider configuration changes so cached cells cannot be reused. */ - ref: string - /** Local construction is free; call markExternalCall before billable provisioning or reconnects. */ - createAdapter(input: { - branchId: string - sequence: AgentMemorySequence - rep: number - seed: number - purpose: 'execute' | 'recovery' - signal: AbortSignal - markExternalCall(): void - }): AgentMemoryAdapter | null | Promise - policy?: AgentMemorySharingPolicy - baseScope?: AgentMemoryScope - /** Conservative external provider charge for one complete history. */ - externalCostUsdPerSequence?: number - /** Conservative extra provider charge when recovering one interrupted history. */ - externalRecoveryCostUsdPerAttempt?: number - /** Release resources and, when cleanupBranches is false, delete the isolated state. */ - disposeAdapter?(adapter: AgentMemoryAdapter): Promise -} - -export interface AgentMemorySequenceProbeResult { - id: string - stepId: string - query: string - score: number - passed: boolean - dimensions: Record - applicableDimensions: readonly string[] - notes: string - hitIds: readonly string[] -} - -export interface AgentMemorySequenceArtifact { - candidateId: string - sequenceId: string - score: number - passed: boolean - dimensions: Record - dimensionSampleCounts: Record - probes: readonly AgentMemorySequenceProbeResult[] - branchDigest: string - journalEntries: number - durationMs: number -} - -export interface AgentMemorySequenceScenario extends Scenario { - kind: 'agent-memory-sequence' - candidateId: string - sequenceId: string - sequence: AgentMemorySequence - seedGroup: string -} - -export interface AgentMemoryExperimentRankingRow { - rank: number - candidateId: string - label: string - scoreMean: number - passRate: number - totalSequences: number - totalCells: number - totalProbes: number - cellsFailed: number - totalCostUsd: number - durationMs: number - dimensions: Record -} - -export interface RunAgentMemoryExperimentOptions { - experimentId: string - /** Stable external branch namespace; distributed workers must use the same value. */ - experimentRunId?: string - sequences: readonly AgentMemorySequence[] - candidates: readonly AgentMemoryExperimentCandidate[] - /** Retired candidates retained only so interrupted branches can be cleaned on resume. */ - recoveryCandidates?: readonly AgentMemoryExperimentCandidate[] - runDir: string - executeStep?: (input: { - memory: AgentMemoryBranch - candidateId: string - sequence: AgentMemorySequence - step: AgentMemorySequenceStep - context: DispatchContext - }) => Promise - /** Required with executeStep; identify the runtime/profile behavior in cache keys. */ - executeStepRef?: string - onBranchSnapshot?: (input: { - candidateId: string - sequenceId: string - cellId: string - snapshot: AgentMemoryBranchSnapshot - }) => Promise | void - cleanupBranches?: boolean - storage?: CampaignStorage - repo?: string - seed?: number - reps?: number - resumable?: boolean - costCeiling?: number - /** Shared across nested experiments when an outer improvement run owns spend. */ - costLedger?: CostLedgerHandle - costPhase?: string - maxConcurrency?: number - dispatchTimeoutMs?: number - /** Total deadline for each provider cleanup, close, or recovery operation. */ - cleanupTimeoutMs?: number - /** Refuse a damaged run with more unfinished attempts than this. Default 1000. */ - maxRecoveryAttempts?: number - /** Bound repeated provider cleanup after process crashes. Default 3 per attempt. */ - maxRecoveryRetriesPerAttempt?: number - now?: () => Date - /** Required with custom storage when all controllers are confined to one process. */ - controllerMode?: AgentMemoryControllerMode - /** Required for distributed controllers that share custom storage. */ - acquireRunLease?: AgentMemoryAcquireRunLease -} - -export type AgentMemoryExperimentRunLease = AgentMemoryRunLease - -export interface AgentMemoryAttemptEvent { - schema: 2 - status: 'started' | 'cleaned' - branchId: string - candidateId: string - candidateRef: string - sequenceId: string - rep: number - seed: number - cleanupBranches: boolean - externalCostUsdPerSequence: number - externalRecoveryCostUsdPerAttempt: number - recordedAt: string - recovery: boolean -} - -export interface RunAgentMemoryExperimentResult { - campaign: CampaignResult - rows: readonly AgentMemoryExperimentRankingRow[] - totalCostUsd: number - /** Recovery spend for retired candidates, excluded from ranking rows but included in totalCostUsd. */ - unrankedRecoveryCostUsd: number - leaderCandidateId?: string - rankingJsonPath: string - rankingMarkdownPath: string - attemptLogPath: string - recoveryLogPath: string -} - -class AgentMemoryCleanupError extends AggregateError { - constructor(errors: Iterable, message: string) { - super(errors, message) - this.name = 'AgentMemoryCleanupError' - } -} - -type OwnedMemoryExperimentRunLease = OwnedAgentMemoryRunLease - -/** Converts existing ordered memory benchmark cases into executable histories. */ -export function buildAgentMemorySequencesFromBenchmarkCases( - cases: readonly KnowledgeMemoryBenchmarkCase[], - options: BuildAgentMemorySequencesFromBenchmarkCasesOptions = {}, -): AgentMemorySequence[] { - const memoryAgentId = options.memoryAgentId ?? 'benchmark-agent' - return cases.map((testCase) => ({ - id: testCase.id, - family: testCase.family, - ...(testCase.split !== undefined ? { split: testCase.split } : {}), - ...(testCase.tags !== undefined ? { tags: testCase.tags } : {}), - metadata: compactRecord({ - ...(testCase.metadata ?? {}), - taskKind: testCase.taskKind, - source: testCase.source, - }), - steps: [ - ...testCase.events.map((event, eventIndex) => ({ - id: `event:${event.id}`, - scope: compactScope( - options.eventScope?.({ event, case: testCase, eventIndex }) ?? { - agentId: memoryAgentId, - sessionId: testCase.id, - }, - ), - writes: [ - { - id: event.id, - kind: 'observation' as const, - text: event.text, - metadata: compactRecord({ - ...(event.metadata ?? {}), - eventId: event.id, - actorId: event.actorId, - sessionId: event.sessionId, - timestamp: event.timestamp, - }), - }, - ], - metadata: { eventIndex }, - })), - { - id: 'probe', - scope: compactScope( - options.probeScope?.(testCase) ?? { - agentId: memoryAgentId, - sessionId: testCase.id, - }, - ), - probes: [ - { - id: 'answer', - query: testCase.prompt, - taskKind: testCase.taskKind, - ...(testCase.requiredFacts !== undefined - ? { requiredFacts: testCase.requiredFacts } - : {}), - ...(testCase.forbiddenFacts !== undefined - ? { forbiddenFacts: testCase.forbiddenFacts } - : {}), - ...(testCase.expectedEventIds !== undefined - ? { expectedEventIds: testCase.expectedEventIds } - : {}), - ...(testCase.expectedActorIds !== undefined - ? { expectedActorIds: testCase.expectedActorIds } - : {}), - ...(testCase.referenceAnswer !== undefined - ? { referenceAnswer: testCase.referenceAnswer } - : {}), - }, - ], - }, - ], - })) -} - -/** Runs ordered, branch-isolated memory histories across candidate systems. */ -export async function runAgentMemoryExperiment( - options: RunAgentMemoryExperimentOptions, -): Promise { - assertNonEmptyString(options.experimentId, 'memory experiment experimentId') - assertNonEmptyString(options.runDir, 'memory experiment runDir') - if (options.experimentRunId !== undefined) { - assertNonEmptyString(options.experimentRunId, 'memory experiment experimentRunId') - } - if (options.sequences.length === 0) throw new Error('memory experiment requires sequences') - if (options.candidates.length === 0) throw new Error('memory experiment requires candidates') - if (options.executeStep && !options.executeStepRef) { - throw new Error('memory experiment executeStepRef is required when executeStep is configured') - } - assertUnique( - options.sequences.map((sequence) => sequence.id), - 'sequence', - ) - assertUnique( - [...options.candidates, ...(options.recoveryCandidates ?? [])].map((candidate) => candidate.id), - 'candidate', - ) - assertMemorySequences(options.sequences) - for (const candidate of [...options.candidates, ...(options.recoveryCandidates ?? [])]) { - if (options.cleanupBranches === false && !candidate.disposeAdapter) { - throw new Error( - `${candidate.id}: cleanupBranches=false requires disposeAdapter to delete isolated external state`, - ) - } - if ( - candidate.externalCostUsdPerSequence !== undefined && - (!Number.isFinite(candidate.externalCostUsdPerSequence) || - candidate.externalCostUsdPerSequence < 0) - ) { - throw new Error( - `${candidate.id}: externalCostUsdPerSequence must be a non-negative finite number`, - ) - } - if ( - candidate.externalRecoveryCostUsdPerAttempt !== undefined && - (!Number.isFinite(candidate.externalRecoveryCostUsdPerAttempt) || - candidate.externalRecoveryCostUsdPerAttempt < 0) - ) { - throw new Error( - `${candidate.id}: externalRecoveryCostUsdPerAttempt must be a non-negative finite number`, - ) - } - assertNonEmptyString(candidate.ref, `${candidate.id} ref`) - } - - const storage = options.storage ?? fsCampaignStorage() - const runDir = resolveRunDir(options.runDir, options.repo) - if (!storage.append) { - throw new Error('memory experiment requires CampaignStorage.append for durable attempt state') - } - resolveMemoryCleanupTimeoutMs(options.cleanupTimeoutMs, 'memory experiment') - const maxRecoveryAttempts = options.maxRecoveryAttempts ?? 1_000 - if (!Number.isSafeInteger(maxRecoveryAttempts) || maxRecoveryAttempts <= 0) { - throw new Error('memory experiment maxRecoveryAttempts must be a positive safe integer') - } - const maxRecoveryRetriesPerAttempt = - options.maxRecoveryRetriesPerAttempt ?? DEFAULT_MEMORY_RECOVERY_RETRIES_PER_ATTEMPT - if (!Number.isSafeInteger(maxRecoveryRetriesPerAttempt) || maxRecoveryRetriesPerAttempt <= 0) { - throw new Error( - 'memory experiment maxRecoveryRetriesPerAttempt must be a positive safe integer', - ) - } - storage.ensureDir(runDir) - const lease = await acquireAgentMemoryRunLease({ - experimentId: options.experimentId, - runDir, - storage, - customStorage: options.storage !== undefined, - lockFileName: 'memory-experiment.lock', - label: 'memory experiment', - controllerMode: options.controllerMode, - acquireRunLease: options.acquireRunLease, - }) - let result: RunAgentMemoryExperimentResult | undefined - let primaryError: unknown - try { - await lease.assertOwned() - result = await runOwnedAgentMemoryExperiment(options, storage, runDir, lease) - } catch (error) { - primaryError = error - } - let releaseError: unknown - try { - await lease.release() - } catch (error) { - releaseError = error - } - if (primaryError && releaseError) { - throw new AggregateError( - [primaryError, releaseError], - 'memory experiment failed and its controller lease could not be released', - ) - } - if (primaryError) throw primaryError - if (releaseError) throw releaseError - if (!result) throw new Error('memory experiment produced no result') - return result -} - -async function runOwnedAgentMemoryExperiment( - options: RunAgentMemoryExperimentOptions, - storage: CampaignStorage, - runDir: string, - lease: OwnedMemoryExperimentRunLease, -): Promise { - const runIdentity = stableId( - 'memory_run', - canonicalJson({ - experimentId: options.experimentId, - experimentRunId: options.experimentRunId ?? runDir, - }), - ) - const maxConcurrency = options.maxConcurrency ?? 2 - if (!Number.isSafeInteger(maxConcurrency) || maxConcurrency <= 0) { - throw new Error('memory experiment maxConcurrency must be a positive safe integer') - } - const executionPool = createMemoryExecutionPool(maxConcurrency) - const dispatchedExecutions: Promise[] = [] - const candidateById = new Map(options.candidates.map((candidate) => [candidate.id, candidate])) - const recoveryCandidateById = new Map( - [...options.candidates, ...(options.recoveryCandidates ?? [])].map((candidate) => [ - candidate.id, - candidate, - ]), - ) - const sequenceById = new Map(options.sequences.map((sequence) => [sequence.id, sequence])) - const scenarios = buildAgentMemorySequenceScenarios(options.sequences, options.candidates) - const attemptLogPath = join(runDir, 'memory-attempts.jsonl') - const recoveryLogPath = join(runDir, 'memory-recovery-attempts.jsonl') - const costCeiling = options.costCeiling ?? options.costLedger?.costCeilingUsd ?? 0 - const costLedger = - options.costLedger ?? - createRunCostLedger({ - storage, - runDir, - costCeilingUsd: costCeiling, - }) - if (costLedger.costCeilingUsd !== costCeiling) { - throw new Error('memory experiment costCeiling must match the shared cost ledger ceiling') - } - storage.ensureDir(runDir) - await recoverAbandonedMemoryAttempts({ - options, - storage, - runDir, - attemptLogPath, - candidateById: recoveryCandidateById, - sequenceById, - lease, - maxConcurrency, - costLedger, - maxRecoveryAttempts: options.maxRecoveryAttempts ?? 1_000, - recoveryLogPath, - maxRecoveryRetriesPerAttempt: - options.maxRecoveryRetriesPerAttempt ?? DEFAULT_MEMORY_RECOVERY_RETRIES_PER_ATTEMPT, - }) - await lease.assertOwned() - let campaign: CampaignResult | undefined - let campaignError: unknown - let settledExecutions: PromiseSettledResult[] = [] - try { - campaign = await runCampaign({ - scenarios, - dispatch: (scenario, context) => { - const candidate = candidateById.get(scenario.candidateId) - if (!candidate) throw new Error(`unknown memory candidate ${scenario.candidateId}`) - const operation = executionPool.run(() => - runSequenceCell({ - options, - candidate, - scenario, - context, - runIdentity, - storage, - attemptLogPath, - lease, - }), - ) - dispatchedExecutions.push(operation) - return operation - }, - dispatchRef: memoryExperimentDispatchRef(options), - judges: [agentMemorySequenceJudge()], - runDir, - storage, - seed: options.seed, - reps: options.reps, - resumable: options.resumable, - costCeiling, - costLedger, - costPhase: options.costPhase, - maxConcurrency, - dispatchTimeoutMs: options.dispatchTimeoutMs, - expectUsage: 'off', - now: options.now, - }) - } catch (error) { - campaignError = error - } finally { - settledExecutions = await Promise.allSettled(dispatchedExecutions) - } - const cleanupFailures = settledExecutions.flatMap((settled) => - settled.status === 'rejected' && settled.reason instanceof AgentMemoryCleanupError - ? [settled.reason] - : [], - ) - if (campaignError && cleanupFailures.length > 0) { - throw new AggregateError( - [campaignError, ...cleanupFailures], - 'memory experiment failed and provider cleanup also failed', - ) - } - if (campaignError) throw campaignError - if (cleanupFailures.length > 0) { - throw new AggregateError(cleanupFailures, 'memory experiment cleanup failed after dispatch') - } - if (!campaign) throw new Error('memory experiment produced no campaign result') - await lease.assertOwned() - const costByCandidate = memoryExperimentCostByCandidate( - costLedger, - campaign.runDir, - scenarios, - [...options.candidates, ...(options.recoveryCandidates ?? [])].map((candidate) => candidate.id), - ) - const unrankedRecoveryCostUsd = normalizeUsd( - (options.recoveryCandidates ?? []).reduce( - (sum, candidate) => sum + (costByCandidate.get(candidate.id) ?? 0), - 0, - ), - ) - const totalCostUsd = normalizeUsd( - [...costByCandidate.values()].reduce((sum, cost) => sum + cost, 0), - ) - const rows = rankAgentMemoryExperiment(options.candidates, scenarios, campaign, costByCandidate) - const rankingJsonPath = join(campaign.runDir, 'memory-experiment-ranking.json') - const rankingMarkdownPath = join(campaign.runDir, 'memory-experiment-ranking.md') - storage.write( - rankingJsonPath, - `${JSON.stringify( - { experimentId: options.experimentId, totalCostUsd, unrankedRecoveryCostUsd, rows }, - null, - 2, - )}\n`, - ) - storage.write( - rankingMarkdownPath, - renderAgentMemoryExperimentRanking(rows, totalCostUsd, unrankedRecoveryCostUsd), - ) - return { - campaign, - rows, - totalCostUsd, - unrankedRecoveryCostUsd, - leaderCandidateId: rows.find((row) => row.cellsFailed === 0)?.candidateId, - rankingJsonPath, - rankingMarkdownPath, - attemptLogPath, - recoveryLogPath, - } -} - -export function buildAgentMemorySequenceScenarios( - sequences: readonly AgentMemorySequence[], - candidates: readonly Pick[], -): AgentMemorySequenceScenario[] { - return candidates.flatMap((candidate) => - sequences.map((sequence) => ({ - id: `${stableId('candidate', candidate.id)}:${sequence.id}`, - kind: 'agent-memory-sequence' as const, - candidateId: candidate.id, - sequenceId: sequence.id, - sequence, - seedGroup: sequence.id, - tags: [...new Set([sequence.split ?? 'dev', ...(sequence.tags ?? []), candidate.id])], - })), - ) -} - -export function agentMemorySequenceJudge(): JudgeConfig< +export { + agentMemorySequenceJudge, + buildAgentMemorySequenceScenarios, + buildAgentMemorySequencesFromBenchmarkCases, +} from './experiment/cases' +export { runAgentMemoryExperiment } from './experiment/run' +export type { + AgentMemoryAttemptEvent, + AgentMemoryExperimentCandidate, + AgentMemoryExperimentRankingRow, + AgentMemoryExperimentRunLease, + AgentMemorySequence, AgentMemorySequenceArtifact, - AgentMemorySequenceScenario -> { - return { - name: 'agent-memory-sequence', - judgeVersion: 'agent-knowledge:memory-sequence:v2', - dimensions: [ - { key: 'score', description: 'mean memory probe score' }, - { key: 'passed', description: '1 when every memory probe passes' }, - { key: 'memory_fact_recall', description: 'current memory fact coverage' }, - { key: 'memory_event_recall', description: 'memory source event coverage' }, - { key: 'memory_actor_recall', description: 'memory actor attribution coverage' }, - { key: 'memory_stale_safe', description: '1 when obsolete memory is not reused' }, - ], - score({ artifact }) { - return { - composite: artifact.score, - dimensions: { - score: artifact.score, - passed: artifact.passed ? 1 : 0, - ...artifact.dimensions, - }, - notes: `${artifact.probes.filter((probe) => probe.passed).length}/${artifact.probes.length} probes passed`, - } - }, - } -} - -async function runSequenceCell(input: { - options: RunAgentMemoryExperimentOptions - candidate: AgentMemoryExperimentCandidate - scenario: AgentMemorySequenceScenario - context: DispatchContext - runIdentity: string - storage: CampaignStorage - attemptLogPath: string - lease: OwnedMemoryExperimentRunLease -}): Promise { - const { options, candidate, scenario, context, runIdentity, storage, attemptLogPath, lease } = - input - const cleanupBranches = options.cleanupBranches ?? true - const costUsd = candidate.externalCostUsdPerSequence ?? 0 - if (!Number.isFinite(costUsd) || costUsd < 0) { - throw new Error( - `${candidate.id}: externalCostUsdPerSequence must be a non-negative finite number`, - ) - } - if (!cleanupBranches && !candidate.disposeAdapter) { - throw new Error( - `${candidate.id}: cleanupBranches=false requires disposeAdapter to delete isolated external state`, - ) - } - const cleanupTimeoutMs = resolveMemoryCleanupTimeoutMs( - options.cleanupTimeoutMs, - `${candidate.id}: memory sequence`, - ) - - context.signal.throwIfAborted() - await lease.assertOwned() - const startedAt = Date.now() - const branchId = stableId( - 'memory_branch', - `${runIdentity}:${context.cellId}:${context.seed}:${randomUUID()}`, - ) - const attempt = memoryAttemptEvent({ - status: 'started', - branchId, - candidate, - sequence: scenario.sequence, - rep: context.rep, - seed: context.seed, - recovery: false, - cleanupBranches, - now: options.now, - }) - appendMemoryAttemptEvent(storage, attemptLogPath, attempt) - - let externalCallAttempted = false - const appendCleanedAttempt = (priorError?: unknown): void => { - try { - appendMemoryAttemptEvent(storage, attemptLogPath, { - ...attempt, - status: 'cleaned', - recordedAt: (options.now ?? (() => new Date()))().toISOString(), - }) - } catch (error) { - throw new AgentMemoryCleanupError( - [...(priorError ? [priorError] : []), error], - `${candidate.id}: memory branch cleanup could not be recorded`, - ) - } - } - const execute = async (): Promise => { - context.signal.throwIfAborted() - await lease.assertOwned() - let rawAdapter: AgentMemoryAdapter | undefined - let adapter: AgentMemoryAdapter | undefined - let memory: AgentMemoryBranch | undefined - let primaryError: unknown - let completedArtifact: AgentMemorySequenceArtifact | undefined - let finalClearStarted = false - let finalClearCompleted = false - try { - const created = await candidate.createAdapter({ - branchId, - sequence: scenario.sequence, - rep: context.rep, - seed: context.seed, - purpose: 'execute', - signal: context.signal, - markExternalCall: () => { - externalCallAttempted = true - }, - }) - if (!created) throw new Error(`${candidate.id}: createAdapter returned no execution adapter`) - rawAdapter = created - adapter = trackExternalMemoryCalls(created, () => { - externalCallAttempted = true - }) - await lease.assertOwned() - context.signal.throwIfAborted() - if (cleanupBranches && !adapter.clear) { - throw new Error( - `${candidate.id}: cleanupBranches requires an adapter with scoped clear support`, - ) - } - memory = createAgentMemoryBranch({ - adapter, - branchId, - lifetime: 'attempt', - policy: candidate.policy, - allowedWriteScopes: sequenceCleanupScopes(scenario.sequence), - baseScope: memoryExperimentBaseScope(options, candidate, scenario.sequenceId), - }) - const probes: AgentMemorySequenceProbeResult[] = [] - for (const step of scenario.sequence.steps) { - context.signal.throwIfAborted() - await lease.assertOwned() - await writeStep(memory, step) - await lease.assertOwned() - await options.executeStep?.({ - memory, - candidateId: candidate.id, - sequence: scenario.sequence, - step, - context, - }) - context.signal.throwIfAborted() - await lease.assertOwned() - const stepProbes = await probeStep(memory, scenario.sequence, step) - await lease.assertOwned() - probes.push(...stepProbes) - } - const snapshot = await memory.snapshot() - await lease.assertOwned() - await options.onBranchSnapshot?.({ - candidateId: candidate.id, - sequenceId: scenario.sequenceId, - cellId: context.cellId, - snapshot, - }) - await lease.assertOwned() - const dimensions = meanDimensions(probes.map((probe) => probe.dimensions)) - const dimensionSampleCounts = countDimensions( - probes.map((probe) => probe.applicableDimensions), - ) - const artifact: AgentMemorySequenceArtifact = { - candidateId: candidate.id, - sequenceId: scenario.sequenceId, - score: mean(probes.map((probe) => probe.score)), - passed: probes.length > 0 && probes.every((probe) => probe.passed), - dimensions, - dimensionSampleCounts, - probes, - branchDigest: snapshot.digest, - journalEntries: snapshot.journal.length, - durationMs: Math.max(0, Date.now() - startedAt), - } - if (cleanupBranches) { - finalClearStarted = true - await runBoundedMemoryLifecycle({ - operation: `${candidate.id}: final branch cleanup`, - timeoutMs: cleanupTimeoutMs, - resource: adapter, - run: () => clearSequenceScopes(memory!, scenario.sequence), - }) - finalClearCompleted = true - await lease.assertOwned() - } - completedArtifact = artifact - } catch (error) { - primaryError = error - } - const cleanupErrors: unknown[] = [] - let cleanupOwned = true - let ownershipError: unknown - try { - await lease.assertOwned() - } catch (error) { - cleanupOwned = false - ownershipError = error - } - if (finalClearStarted && !finalClearCompleted && primaryError) { - cleanupErrors.push(primaryError) - } - if ( - primaryError && - cleanupOwned && - !finalClearStarted && - memory && - cleanupBranches && - adapter?.clear - ) { - try { - await runBoundedMemoryLifecycle({ - operation: `${candidate.id}: failed branch cleanup`, - timeoutMs: cleanupTimeoutMs, - resource: adapter, - run: () => clearSequenceScopes(memory!, scenario.sequence), - }) - } catch (error) { - cleanupErrors.push(error) - } - } - if (adapter) { - try { - await runBoundedMemoryLifecycle({ - operation: `${candidate.id}: adapter close`, - timeoutMs: cleanupTimeoutMs, - resource: adapter, - run: async () => { - if (memory && cleanupOwned) await memory.close?.() - else { - if (cleanupOwned) await adapter!.flush?.() - await adapter!.close?.() - } - }, - }) - } catch (error) { - cleanupErrors.push(error) - } - if (cleanupOwned) { - try { - await runBoundedMemoryLifecycle({ - operation: `${candidate.id}: adapter disposal`, - timeoutMs: cleanupTimeoutMs, - resource: adapter, - run: () => { - if (candidate.disposeAdapter) externalCallAttempted = true - return candidate.disposeAdapter?.(rawAdapter!) - }, - }) - } catch (error) { - cleanupErrors.push(error) - } - } - } else { - cleanupErrors.push( - new Error(`${candidate.id}: adapter creation failed before cleanup could be confirmed`), - ) - } - if (cleanupOwned && cleanupErrors.length === 0) appendCleanedAttempt(primaryError) - if (!cleanupOwned && cleanupErrors.length === 0) { - if (primaryError) throw primaryError - throw ownershipError - } - if (cleanupErrors.length > 0) { - const primaryMessage = - primaryError instanceof Error ? primaryError.message : String(primaryError ?? '') - throw new AgentMemoryCleanupError( - [ - ...(primaryError && !cleanupErrors.includes(primaryError) ? [primaryError] : []), - ...(ownershipError ? [ownershipError] : []), - ...cleanupErrors, - ], - `${candidate.id}: memory branch cleanup failed${primaryMessage ? ` after: ${primaryMessage}` : ''}`, - ) - } - if (primaryError) throw primaryError - if (!completedArtifact) throw new Error(`${candidate.id}: memory sequence produced no result`) - return completedArtifact - } - - if (costUsd === 0) { - let artifact: AgentMemorySequenceArtifact | undefined - let error: unknown - try { - artifact = await execute() - } catch (caught) { - error = caught - } - if (error) throw error - if (!artifact) throw new Error(`${candidate.id}: memory sequence produced no result`) - return artifact - } - const receipt = { - model: candidate.id, - inputTokens: 0, - outputTokens: 0, - actualCostUsd: costUsd, - } as const - const paid = await context.cost.runPaidCall({ - callId: memoryAttemptCostCallId(attempt, 'execute', 0), - actor: `agent-knowledge:memory-experiment:${candidate.id}`, - model: candidate.id, - maximumCharge: { externallyEnforcedMaximumUsd: costUsd }, - execute, - receipt: () => receipt, - receiptFromError: () => ({ - ...receipt, - actualCostUsd: externalCallAttempted ? costUsd : 0, - }), - }) - if (!paid.succeeded) throw paid.error - return paid.value -} - -function trackExternalMemoryCalls( - adapter: AgentMemoryAdapter, - onExternalCall: () => void, -): AgentMemoryAdapter { - return { - id: adapter.id, - branchIsolation: adapter.branchIsolation, - search(query, options) { - onExternalCall() - return adapter.search.call(adapter, query, options) - }, - getContext(query, options) { - onExternalCall() - return adapter.getContext.call(adapter, query, options) - }, - write(input) { - onExternalCall() - return adapter.write.call(adapter, input) - }, - ...(adapter.clear - ? { - clear(scope?: AgentMemoryScope) { - onExternalCall() - return adapter.clear!.call(adapter, scope) - }, - } - : {}), - ...(adapter.flush - ? { - flush() { - onExternalCall() - return adapter.flush!.call(adapter) - }, - } - : {}), - ...(adapter.close - ? { - close() { - onExternalCall() - return adapter.close!.call(adapter) - }, - } - : {}), - } -} - -async function recoverAbandonedMemoryAttempts(input: { - options: RunAgentMemoryExperimentOptions - storage: CampaignStorage - runDir: string - attemptLogPath: string - candidateById: ReadonlyMap - sequenceById: ReadonlyMap - lease: OwnedMemoryExperimentRunLease - maxConcurrency: number - costLedger: CostLedgerHandle - maxRecoveryAttempts: number - recoveryLogPath: string - maxRecoveryRetriesPerAttempt: number -}): Promise { - let attempts = readActiveMemoryAttempts(input.storage, input.attemptLogPath) - if (attempts.length > input.maxRecoveryAttempts) { - throw new Error( - `memory experiment has ${attempts.length} unfinished attempts; maxRecoveryAttempts is ${input.maxRecoveryAttempts}`, - ) - } - for (const attempt of attempts) { - const candidate = input.candidateById.get(attempt.candidateId) - if (!candidate) { - throw new Error( - `cannot recover memory branch '${attempt.branchId}': candidate '${attempt.candidateId}' is missing; pass it in recoveryCandidates`, - ) - } - assertMemoryAttemptCandidateMatches(attempt, candidate) - if (!input.sequenceById.has(attempt.sequenceId)) { - throw new Error( - `cannot recover memory branch '${attempt.branchId}': sequence '${attempt.sequenceId}' is missing`, - ) - } - if ((input.options.cleanupBranches ?? true) !== attempt.cleanupBranches) { - throw new Error(`cannot recover memory branch '${attempt.branchId}': cleanupBranches changed`) - } - } - - reconcileInterruptedMemoryPaidCalls(input.costLedger) - assertNoInterruptedPaidCalls(input.costLedger, 'memory experiment recovery') - - for (const attempt of attempts) { - const candidate = input.candidateById.get(attempt.candidateId)! - const executionCostUsd = candidate.externalCostUsdPerSequence ?? 0 - if ( - executionCostUsd > 0 && - !hasSettledPaidCall(input.costLedger, memoryAttemptCostCallId(attempt, 'execute', 0)) - ) { - appendMemoryAttemptEvent(input.storage, input.attemptLogPath, { - ...attempt, - status: 'cleaned', - recovery: true, - recordedAt: (input.options.now ?? (() => new Date()))().toISOString(), - }) - } - } - attempts = readActiveMemoryAttempts(input.storage, input.attemptLogPath) - const recoveryGenerations = reserveRecoveryAttempts({ - storage: input.storage, - path: input.recoveryLogPath, - attemptIds: attempts.map((attempt) => attempt.branchId), - maxRetriesPerAttempt: input.maxRecoveryRetriesPerAttempt, - label: 'memory recovery attempt log', - now: input.options.now, - }) - const pool = createMemoryExecutionPool(input.maxConcurrency) - const settled = await Promise.allSettled( - attempts - .sort((left, right) => left.branchId.localeCompare(right.branchId)) - .map((attempt) => - pool.run(async () => { - await input.lease.assertOwned() - const candidate = input.candidateById.get(attempt.candidateId) - if (!candidate) { - throw new Error( - `cannot recover memory branch '${attempt.branchId}': candidate '${attempt.candidateId}' is missing; pass it in recoveryCandidates`, - ) - } - assertMemoryAttemptCandidateMatches(attempt, candidate) - const sequence = input.sequenceById.get(attempt.sequenceId) - if (!sequence) { - throw new Error( - `cannot recover memory branch '${attempt.branchId}': sequence '${attempt.sequenceId}' is missing`, - ) - } - if ((input.options.cleanupBranches ?? true) !== attempt.cleanupBranches) { - throw new Error( - `cannot recover memory branch '${attempt.branchId}': cleanupBranches changed`, - ) - } - const recoveryCostUsd = candidate.externalRecoveryCostUsdPerAttempt ?? 0 - const recoveryGeneration = recoveryGenerations.get(attempt.branchId) - if (recoveryGeneration === undefined) { - throw new Error(`missing recovery generation for memory branch '${attempt.branchId}'`) - } - let externalRecoveryAttempted = false - const recover = async (): Promise => { - await recoverMemoryAttempt({ - options: input.options, - candidate, - sequence, - attempt, - lease: input.lease, - onExternalCall: () => { - externalRecoveryAttempted = true - }, - }) - appendMemoryAttemptEvent(input.storage, input.attemptLogPath, { - ...attempt, - status: 'cleaned', - recovery: true, - recordedAt: (input.options.now ?? (() => new Date()))().toISOString(), - }) - } - if (recoveryCostUsd === 0) { - await recover() - } else { - const tags = memoryRecoveryCostTags(input.runDir, candidate.id, attempt.branchId) - const receipt = { - model: candidate.id, - inputTokens: 0, - outputTokens: 0, - actualCostUsd: recoveryCostUsd, - } as const - const paid = await input.costLedger.runPaidCall({ - callId: memoryAttemptCostCallId(attempt, 'recovery', recoveryGeneration), - channel: 'driver', - phase: `${input.options.costPhase ?? 'memory.experiment'}.recovery`, - actor: `agent-knowledge:memory-recovery:${candidate.id}`, - model: candidate.id, - tags, - maximumCharge: { externallyEnforcedMaximumUsd: recoveryCostUsd }, - execute: recover, - receipt: () => ({ - ...receipt, - actualCostUsd: externalRecoveryAttempted ? recoveryCostUsd : 0, - }), - receiptFromError: () => ({ - ...receipt, - actualCostUsd: externalRecoveryAttempted ? recoveryCostUsd : 0, - }), - }) - if (!paid.succeeded) throw paid.error - } - }), - ), - ) - const failures = settled.flatMap((result) => - result.status === 'rejected' ? [result.reason] : [], - ) - if (failures.length === 1) throw failures[0] - if (failures.length > 1) { - throw new AggregateError(failures, 'multiple abandoned memory branches failed recovery') - } -} - -async function recoverMemoryAttempt(input: { - options: RunAgentMemoryExperimentOptions - candidate: AgentMemoryExperimentCandidate - sequence: AgentMemorySequence - attempt: AgentMemoryAttemptEvent - lease: OwnedMemoryExperimentRunLease - onExternalCall(): void -}): Promise { - const { options, candidate, sequence, attempt, lease, onExternalCall } = input - const cleanupBranches = attempt.cleanupBranches - const cleanupTimeoutMs = resolveMemoryCleanupTimeoutMs( - options.cleanupTimeoutMs, - `${candidate.id}: abandoned branch recovery`, - ) - let rawAdapter: AgentMemoryAdapter | undefined - let adapter: AgentMemoryAdapter | undefined - let memory: AgentMemoryBranch | undefined - let primaryError: unknown - try { - const abortController = new AbortController() - const creation = Promise.resolve().then(() => - candidate.createAdapter({ - branchId: attempt.branchId, - sequence, - rep: attempt.rep, - seed: attempt.seed, - purpose: 'recovery', - signal: abortController.signal, - markExternalCall: onExternalCall, - }), - ) - releaseMemoryAdapterCreatedAfterAbort({ - creation, - signal: abortController.signal, - dispose: candidate.disposeAdapter - ? async (created) => { - onExternalCall() - await candidate.disposeAdapter?.(created) - } - : undefined, - }) - const recovered = await runBoundedMemoryLifecycle({ - operation: `${candidate.id}: recovery adapter creation`, - timeoutMs: cleanupTimeoutMs, - abortController, - run: () => creation, - }) - await lease.assertOwned() - if (recovered === null) return - rawAdapter = recovered - adapter = trackExternalMemoryCalls(recovered, onExternalCall) - const recoveryDelayMs = memoryRecoveryDelayMs(adapter) - await sleepForMemoryRecovery( - recoveryDelayMs, - () => lease.assertOwned(), - cleanupTimeoutMs, - `${candidate.id}: abandoned branch recovery visibility wait`, - ) - if (cleanupBranches) { - if (!adapter.clear) { - throw new Error( - `${candidate.id}: abandoned branch recovery requires an adapter with scoped clear support`, - ) - } - memory = createAgentMemoryBranch({ - adapter, - branchId: attempt.branchId, - lifetime: 'attempt', - policy: candidate.policy, - allowedWriteScopes: sequenceCleanupScopes(sequence), - baseScope: memoryExperimentBaseScope(options, candidate, sequence.id), - }) - await runBoundedMemoryLifecycle({ - operation: `${candidate.id}: abandoned branch cleanup`, - timeoutMs: cleanupTimeoutMs, - resource: adapter, - run: () => clearSequenceScopes(memory!, sequence), - }) - await lease.assertOwned() - } - } catch (error) { - primaryError = error - } - - const cleanupErrors: unknown[] = [] - let cleanupOwned = true - try { - await lease.assertOwned() - } catch (error) { - cleanupOwned = false - cleanupErrors.push(error) - } - if (adapter) { - try { - await runBoundedMemoryLifecycle({ - operation: `${candidate.id}: recovery adapter close`, - timeoutMs: cleanupTimeoutMs, - resource: adapter, - run: async () => { - if (memory && cleanupOwned) { - await memory.close?.() - } else { - if (cleanupOwned && adapter!.flush) { - await adapter!.flush() - } - await adapter!.close?.() - } - }, - }) - } catch (error) { - cleanupErrors.push(error) - } - if (cleanupOwned) { - try { - if (candidate.disposeAdapter) onExternalCall() - await runBoundedMemoryLifecycle({ - operation: `${candidate.id}: recovery adapter disposal`, - timeoutMs: cleanupTimeoutMs, - resource: adapter, - run: () => candidate.disposeAdapter?.(rawAdapter!), - }) - } catch (error) { - cleanupErrors.push(error) - } - } - } else { - cleanupErrors.push( - new Error( - `${candidate.id}: recovery adapter creation failed before cleanup could be confirmed`, - ), - ) - } - if (!cleanupOwned || primaryError || cleanupErrors.length > 0) { - throw new AgentMemoryCleanupError( - [...(primaryError ? [primaryError] : []), ...cleanupErrors], - `${candidate.id}: abandoned memory branch '${attempt.branchId}' recovery failed`, - ) - } -} - -function memoryAttemptCostCallId( - attempt: AgentMemoryAttemptEvent, - purpose: 'execute' | 'recovery', - generation: number, -): string { - return stableId( - 'memory_cost_call', - canonicalJson({ - purpose, - generation, - branchId: attempt.branchId, - candidateId: attempt.candidateId, - candidateRef: attempt.candidateRef, - externalCostUsdPerSequence: attempt.externalCostUsdPerSequence, - externalRecoveryCostUsdPerAttempt: attempt.externalRecoveryCostUsdPerAttempt, - sequenceId: attempt.sequenceId, - rep: attempt.rep, - seed: attempt.seed, - }), - ) -} - -function memoryRecoveryCostTags( - runDir: string, - candidateId: string, - branchId: string, -): Record { - return { - runDir, - candidateId, - branchId, - memoryRecovery: 'attempt', - } -} - -function memoryAttemptEvent(input: { - status: AgentMemoryAttemptEvent['status'] - branchId: string - candidate: AgentMemoryExperimentCandidate - sequence: AgentMemorySequence - rep: number - seed: number - recovery: boolean - cleanupBranches?: boolean - now?: () => Date -}): AgentMemoryAttemptEvent { - return { - schema: 2, - status: input.status, - branchId: input.branchId, - candidateId: input.candidate.id, - candidateRef: input.candidate.ref, - sequenceId: input.sequence.id, - rep: input.rep, - seed: input.seed, - cleanupBranches: input.cleanupBranches ?? true, - externalCostUsdPerSequence: input.candidate.externalCostUsdPerSequence ?? 0, - externalRecoveryCostUsdPerAttempt: input.candidate.externalRecoveryCostUsdPerAttempt ?? 0, - recordedAt: (input.now ?? (() => new Date()))().toISOString(), - recovery: input.recovery, - } -} - -function appendMemoryAttemptEvent( - storage: CampaignStorage, - path: string, - event: AgentMemoryAttemptEvent, -): void { - appendAttemptJournalEvent({ - storage, - path, - event, - label: 'memory attempt log', - }) -} - -function readActiveMemoryAttempts( - storage: CampaignStorage, - path: string, -): AgentMemoryAttemptEvent[] { - return readActiveAttemptJournal({ - storage, - path, - label: 'memory attempt log', - parse: parseMemoryAttemptEvent, - id: (event) => event.branchId, - sameAttempt: sameMemoryAttempt, - }) -} - -function parseMemoryAttemptEvent( - value: unknown, - path: string, - line: number, -): AgentMemoryAttemptEvent { - const event = value as Partial | null - const valid = - typeof event === 'object' && - event !== null && - event.schema === 2 && - (event.status === 'started' || event.status === 'cleaned') && - typeof event.branchId === 'string' && - event.branchId.length > 0 && - typeof event.candidateId === 'string' && - event.candidateId.length > 0 && - typeof event.candidateRef === 'string' && - event.candidateRef.length > 0 && - typeof event.sequenceId === 'string' && - event.sequenceId.length > 0 && - Number.isSafeInteger(event.rep) && - Number.isSafeInteger(event.seed) && - typeof event.cleanupBranches === 'boolean' && - typeof event.externalCostUsdPerSequence === 'number' && - Number.isFinite(event.externalCostUsdPerSequence) && - event.externalCostUsdPerSequence >= 0 && - typeof event.externalRecoveryCostUsdPerAttempt === 'number' && - Number.isFinite(event.externalRecoveryCostUsdPerAttempt) && - event.externalRecoveryCostUsdPerAttempt >= 0 && - typeof event.recordedAt === 'string' && - !Number.isNaN(Date.parse(event.recordedAt)) && - typeof event.recovery === 'boolean' - if (!valid) throw new Error(`invalid memory attempt event in '${path}' line ${line}`) - return value as AgentMemoryAttemptEvent -} - -function sameMemoryAttempt(left: AgentMemoryAttemptEvent, right: AgentMemoryAttemptEvent): boolean { - return ( - left.branchId === right.branchId && - left.candidateId === right.candidateId && - left.candidateRef === right.candidateRef && - left.sequenceId === right.sequenceId && - left.rep === right.rep && - left.seed === right.seed && - left.cleanupBranches === right.cleanupBranches && - left.externalCostUsdPerSequence === right.externalCostUsdPerSequence && - left.externalRecoveryCostUsdPerAttempt === right.externalRecoveryCostUsdPerAttempt - ) -} - -function assertMemoryAttemptCandidateMatches( - attempt: AgentMemoryAttemptEvent, - candidate: AgentMemoryExperimentCandidate, -): void { - if (candidate.ref !== attempt.candidateRef) { - throw new Error( - `cannot recover memory branch '${attempt.branchId}': candidate ref changed from '${attempt.candidateRef}' to '${candidate.ref}'`, - ) - } - const executionCost = candidate.externalCostUsdPerSequence ?? 0 - const recoveryCost = candidate.externalRecoveryCostUsdPerAttempt ?? 0 - if ( - executionCost !== attempt.externalCostUsdPerSequence || - recoveryCost !== attempt.externalRecoveryCostUsdPerAttempt - ) { - throw new Error( - `cannot recover memory branch '${attempt.branchId}': candidate cost settings changed; start a new run or restore the recorded costs`, - ) - } -} - -async function writeStep(memory: AgentMemoryBranch, step: AgentMemorySequenceStep): Promise { - const writes = (step.writes ?? []).map((write) => ({ - ...write, - scope: mergeScopes(step.scope, write.scope), - })) - if (step.parallelWrites) { - await Promise.all(writes.map((write) => memory.write(write))) - return - } - for (const write of writes) await memory.write(write) -} - -async function probeStep( - memory: AgentMemoryBranch, - sequence: AgentMemorySequence, - step: AgentMemorySequenceStep, -): Promise { - const run = async (probe: AgentMemorySequenceProbe): Promise => { - const scope = mergeScopes(step.scope, probe.scope) - const context = await memory.getContext(probe.query, { scope, limit: probe.limit }) - const artifact = { - answer: context.text, - rememberedFacts: context.hits.map((hit) => hit.text), - citedEventIds: uniqueStrings([ - ...context.hits.map((hit) => hit.metadata?.eventId), - ...context.hits.flatMap((hit) => stringArray(hit.metadata?.eventIds)), - ]), - usedMemoryIds: context.hits.map((hit) => hit.id), - actorIds: uniqueStrings([ - ...context.hits.map((hit) => hit.metadata?.actorId), - ...context.hits.flatMap((hit) => stringArray(hit.metadata?.actorIds)), - ]), - } - const evaluation = scoreMemoryBenchmarkArtifact( - { - id: `${sequence.id}:${step.id}:${probe.id}`, - family: sequence.family, - taskKind: probe.taskKind ?? 'memory-recall', - split: sequence.split, - events: [], - prompt: probe.query, - requiredFacts: - probe.requiredFacts && probe.requiredFacts.length > 0 - ? probe.requiredFacts - : probe.referenceAnswer - ? [{ id: `${probe.id}:reference`, anyOf: [probe.referenceAnswer] }] - : undefined, - forbiddenFacts: probe.forbiddenFacts, - expectedEventIds: probe.expectedEventIds, - expectedActorIds: probe.expectedActorIds, - referenceAnswer: probe.referenceAnswer, - }, - artifact, - ) - return { - id: probe.id, - stepId: step.id, - query: probe.query, - score: evaluation.score, - passed: evaluation.passed, - dimensions: evaluation.dimensions, - applicableDimensions: evaluation.applicableDimensions ?? Object.keys(evaluation.dimensions), - notes: evaluation.notes, - hitIds: context.hits.map((hit) => hit.id), - } - } - if (step.parallelProbes === false) { - const results: AgentMemorySequenceProbeResult[] = [] - for (const probe of step.probes ?? []) results.push(await run(probe)) - return results - } - return Promise.all((step.probes ?? []).map(run)) -} - -function rankAgentMemoryExperiment( - candidates: readonly AgentMemoryExperimentCandidate[], - scenarios: readonly AgentMemorySequenceScenario[], - campaign: CampaignResult, - costByCandidate: ReadonlyMap, -): AgentMemoryExperimentRankingRow[] { - const scenarioById = new Map(scenarios.map((scenario) => [scenario.id, scenario])) - const rows = candidates.map((candidate): AgentMemoryExperimentRankingRow => { - const candidateScenarios = scenarios.filter((scenario) => scenario.candidateId === candidate.id) - const cells = campaign.cells.filter( - (cell) => scenarioById.get(cell.scenarioId)?.candidateId === candidate.id, - ) - const successful = cells.filter((cell) => !cell.error && cell.artifact) - const dimensionRows = successful.map((cell) => cell.artifact.dimensions) - return { - rank: 0, - candidateId: candidate.id, - label: candidate.label ?? candidate.id, - scoreMean: mean( - cells.map((cell) => (!cell.error && cell.artifact ? cell.artifact.score : 0)), - ), - passRate: mean(cells.map((cell) => (!cell.error && cell.artifact?.passed ? 1 : 0))), - totalSequences: candidateScenarios.length, - totalCells: cells.length, - totalProbes: successful.reduce((sum, cell) => sum + cell.artifact.probes.length, 0), - cellsFailed: cells.filter((cell) => Boolean(cell.error)).length, - totalCostUsd: normalizeUsd(costByCandidate.get(candidate.id) ?? 0), - durationMs: cells.reduce((sum, cell) => sum + cell.durationMs, 0), - dimensions: meanDimensions(dimensionRows), - } - }) - return rows - .sort( - (a, b) => - Number(a.cellsFailed > 0) - Number(b.cellsFailed > 0) || - b.scoreMean - a.scoreMean || - b.passRate - a.passRate || - a.totalCostUsd - b.totalCostUsd || - a.candidateId.localeCompare(b.candidateId), - ) - .map((row, index) => ({ ...row, rank: index + 1 })) -} - -function memoryExperimentCostByCandidate( - costLedger: CostLedgerHandle, - runDir: string, - scenarios: readonly AgentMemorySequenceScenario[], - candidateIdsInput: readonly string[], -): ReadonlyMap { - const candidateByScenario = new Map( - scenarios.map((scenario) => [scenario.id, scenario.candidateId]), - ) - const candidateIds = new Set(candidateIdsInput) - const totals = new Map() - for (const receipt of costLedger.list()) { - if (receipt.tags?.runDir !== runDir) continue - const scenarioCandidate = receipt.tags.scenarioId - ? candidateByScenario.get(receipt.tags.scenarioId) - : undefined - const recoveryCandidate = - receipt.tags.memoryRecovery === 'attempt' && receipt.tags.candidateId - ? receipt.tags.candidateId - : undefined - const candidateId = scenarioCandidate ?? recoveryCandidate - if (!candidateId || !candidateIds.has(candidateId)) continue - totals.set(candidateId, (totals.get(candidateId) ?? 0) + receipt.costUsd) - } - return totals -} - -function renderAgentMemoryExperimentRanking( - rows: readonly AgentMemoryExperimentRankingRow[], - totalCostUsd: number, - unrankedRecoveryCostUsd: number, -): string { - return [ - '# Agent Memory Experiment', - '', - `- total cost: $${format(totalCostUsd)}`, - `- retired-candidate recovery cost: $${format(unrankedRecoveryCostUsd)}`, - '', - '| rank | candidate | sequences | cells | probes | failed | score | pass rate | cost | duration ms |', - '| ---: | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |', - ...rows.map( - (row) => - `| ${row.rank} | ${row.label} | ${row.totalSequences} | ${row.totalCells} | ${row.totalProbes} | ${row.cellsFailed} | ${format(row.scoreMean)} | ${format(row.passRate)} | $${format(row.totalCostUsd)} | ${format(row.durationMs)} |`, - ), - '', - ].join('\n') -} - -function memoryExperimentDispatchRef(options: RunAgentMemoryExperimentOptions): string { - return stableId( - 'memory_experiment', - canonicalJson({ - implementationRef: MEMORY_EXPERIMENT_IMPLEMENTATION_REF, - experimentId: options.experimentId, - experimentRunId: options.experimentRunId ?? null, - executeStepRef: options.executeStepRef ?? 'fixtures', - cleanupBranches: options.cleanupBranches ?? true, - candidates: options.candidates - .map((candidate) => ({ - id: candidate.id, - ref: candidate.ref, - policy: candidate.policy ?? null, - baseScope: candidate.baseScope ?? null, - externalCostUsdPerSequence: candidate.externalCostUsdPerSequence ?? 0, - externalRecoveryCostUsdPerAttempt: candidate.externalRecoveryCostUsdPerAttempt ?? 0, - })) - .sort((a, b) => a.id.localeCompare(b.id)), - }), - ) -} - -function meanDimensions(rows: readonly Record[]): Record { - const values = new Map() - for (const row of rows) { - for (const [key, value] of Object.entries(row)) { - if (!Number.isFinite(value)) continue - const bucket = values.get(key) ?? [] - bucket.push(value) - values.set(key, bucket) - } - } - return Object.fromEntries([...values].map(([key, bucket]) => [key, mean(bucket)])) -} - -function countDimensions(rows: readonly (readonly string[])[]): Record { - const counts = new Map() - for (const row of rows) { - for (const key of new Set(row)) counts.set(key, (counts.get(key) ?? 0) + 1) - } - return Object.fromEntries(counts) -} - -function mean(values: readonly number[]): number { - return values.length === 0 ? 0 : values.reduce((sum, value) => sum + value, 0) / values.length -} - -function mergeScopes(base?: AgentMemoryScope, extra?: AgentMemoryScope): AgentMemoryScope { - return { - ...(base ?? {}), - ...(extra ?? {}), - tags: { ...(base?.tags ?? {}), ...(extra?.tags ?? {}) }, - } -} - -function memoryExperimentBaseScope( - options: Pick, - candidate: Pick, - sequenceId: string, -): AgentMemoryScope { - return mergeScopes(candidate.baseScope, { - tags: { - memoryExperimentId: options.experimentId, - memoryCandidateId: candidate.id, - memorySequenceId: sequenceId, - }, - }) -} - -function sequenceCleanupScopes(sequence: AgentMemorySequence): AgentMemoryScope[] { - const scopes = new Map() - for (const scope of sequence.cleanupScopes ?? []) { - const normalized = normalizeCleanupScope(scope) - scopes.set(JSON.stringify(normalized), normalized) - } - for (const step of sequence.steps) { - const candidates = [ - ...(step.scope ? [step.scope] : []), - ...(step.writes ?? []).map((write) => mergeScopes(step.scope, write.scope)), - ...(step.probes ?? []).map((probe) => mergeScopes(step.scope, probe.scope)), - ] - for (const scope of candidates) { - const normalized = normalizeCleanupScope(scope) - scopes.set(JSON.stringify(normalized), normalized) - } - } - return [...scopes.values()] -} - -async function clearSequenceScopes( - memory: AgentMemoryBranch, - sequence: AgentMemorySequence, -): Promise { - for (const scope of sequenceCleanupScopes(sequence)) await memory.clear?.(scope) -} - -function assertMemorySequences(sequences: readonly AgentMemorySequence[]): void { - for (const sequence of sequences) { - assertNonEmptyString(sequence.family, `memory experiment sequence ${sequence.id} family`) - if (sequence.steps.length === 0) { - throw new Error(`memory experiment sequence ${sequence.id} has no steps`) - } - assertUnique( - sequence.steps.map((step) => step.id), - `step in sequence ${sequence.id}`, - ) - let probeCount = 0 - for (const step of sequence.steps) { - for (const write of step.writes ?? []) { - assertNonEmptyString(write.text, `memory experiment write in ${sequence.id}/${step.id}`) - if (write.id !== undefined) { - assertNonEmptyString(write.id, `memory experiment write id in ${sequence.id}/${step.id}`) - } - } - assertUnique( - (step.probes ?? []).map((probe) => probe.id), - `probe in sequence ${sequence.id} step ${step.id}`, - ) - for (const probe of step.probes ?? []) { - probeCount += 1 - assertNonEmptyString( - probe.query, - `memory experiment probe query ${sequence.id}/${step.id}/${probe.id}`, - ) - if (probe.limit !== undefined && (!Number.isSafeInteger(probe.limit) || probe.limit <= 0)) { - throw new Error( - `memory experiment probe limit ${sequence.id}/${step.id}/${probe.id} must be a positive safe integer`, - ) - } - assertMemoryFactMatchers( - probe.requiredFacts ?? [], - `${sequence.id}/${step.id}/${probe.id} requiredFacts`, - ) - assertMemoryFactMatchers( - probe.forbiddenFacts ?? [], - `${sequence.id}/${step.id}/${probe.id} forbiddenFacts`, - ) - assertStringList(probe.expectedEventIds, `${sequence.id}/${step.id}/${probe.id} event ids`) - assertStringList(probe.expectedActorIds, `${sequence.id}/${step.id}/${probe.id} actor ids`) - if (probe.referenceAnswer !== undefined) { - assertNonEmptyString( - probe.referenceAnswer, - `memory experiment reference answer ${sequence.id}/${step.id}/${probe.id}`, - ) - } - const hasTarget = - (probe.requiredFacts?.length ?? 0) > 0 || - (probe.forbiddenFacts?.length ?? 0) > 0 || - (probe.expectedEventIds?.length ?? 0) > 0 || - (probe.expectedActorIds?.length ?? 0) > 0 || - Boolean(probe.referenceAnswer?.trim()) - if (!hasTarget) { - throw new Error( - `memory experiment probe ${sequence.id}/${step.id}/${probe.id} has no measurable target`, - ) - } - } - } - if (probeCount === 0) { - throw new Error(`memory experiment sequence ${sequence.id} has no probes`) - } - } -} - -function normalizeCleanupScope(scope: AgentMemoryScope): AgentMemoryScope { - const normalized = compactScope(scope) - if (normalized.tags && Object.keys(normalized.tags).length === 0) { - delete normalized.tags - } - return normalized -} - -function compactScope(scope: AgentMemoryScope): AgentMemoryScope { - return Object.fromEntries( - Object.entries(scope).filter(([, value]) => value !== undefined), - ) as AgentMemoryScope -} - -function compactRecord(record: Record): Record { - return Object.fromEntries(Object.entries(record).filter(([, value]) => value !== undefined)) -} - -function uniqueStrings(values: readonly unknown[]): string[] { - return [...new Set(values.filter((value): value is string => typeof value === 'string'))] -} - -function stringArray(value: unknown): string[] { - return Array.isArray(value) - ? value.filter((entry): entry is string => typeof entry === 'string') - : [] -} - -function assertUnique(values: readonly string[], label: string): void { - const seen = new Set() - for (const value of values) { - assertNonEmptyString(value, `memory experiment ${label} id`) - if (seen.has(value)) throw new Error(`duplicate memory experiment ${label} id: ${value}`) - seen.add(value) - } -} - -function assertMemoryFactMatchers( - matchers: readonly KnowledgeMemoryFactMatcher[], - label: string, -): void { - assertUnique( - matchers.map((matcher) => matcher.id), - `${label} matcher`, - ) - for (const matcher of matchers) { - if (matcher.anyOf.length === 0) { - throw new Error(`memory experiment ${label} matcher ${matcher.id} requires anyOf`) - } - assertStringList(matcher.anyOf, `${label} matcher ${matcher.id} anyOf`) - assertStringList(matcher.sourceEventIds, `${label} matcher ${matcher.id} source event ids`) - if (matcher.weight !== undefined && (!Number.isFinite(matcher.weight) || matcher.weight <= 0)) { - throw new Error( - `memory experiment ${label} matcher ${matcher.id} weight must be a positive finite number`, - ) - } - } -} - -function assertStringList(values: readonly string[] | undefined, label: string): void { - if (values === undefined) return - assertUnique(values, label) -} - -function assertNonEmptyString(value: unknown, label: string): asserts value is string { - if (typeof value !== 'string' || value.trim().length === 0) { - throw new Error(`${label} must be a non-empty string`) - } -} - -function format(value: number): string { - return Number.isFinite(value) ? value.toFixed(4) : '0.0000' -} - -function normalizeUsd(value: number): number { - return Number(value.toFixed(12)) -} + AgentMemorySequenceProbe, + AgentMemorySequenceProbeResult, + AgentMemorySequenceScenario, + AgentMemorySequenceStep, + BuildAgentMemorySequencesFromBenchmarkCasesOptions, + RunAgentMemoryExperimentOptions, + RunAgentMemoryExperimentResult, +} from './experiment/types' diff --git a/src/memory/experiment/cases.ts b/src/memory/experiment/cases.ts new file mode 100644 index 0000000..941934e --- /dev/null +++ b/src/memory/experiment/cases.ts @@ -0,0 +1,133 @@ +/** Converts existing ordered memory benchmark cases into executable histories. */ +import type { JudgeConfig } from '@tangle-network/agent-eval/campaign' +import type { KnowledgeMemoryBenchmarkCase } from '../../benchmarks/index' +import { stableId } from '../../ids' +import type { + AgentMemoryExperimentCandidate, + AgentMemorySequence, + AgentMemorySequenceArtifact, + AgentMemorySequenceScenario, + BuildAgentMemorySequencesFromBenchmarkCasesOptions, +} from './types' +import { compactRecord, compactScope } from './validation' + +export function buildAgentMemorySequencesFromBenchmarkCases( + cases: readonly KnowledgeMemoryBenchmarkCase[], + options: BuildAgentMemorySequencesFromBenchmarkCasesOptions = {}, +): AgentMemorySequence[] { + const memoryAgentId = options.memoryAgentId ?? 'benchmark-agent' + return cases.map((testCase) => ({ + id: testCase.id, + family: testCase.family, + ...(testCase.split !== undefined ? { split: testCase.split } : {}), + ...(testCase.tags !== undefined ? { tags: testCase.tags } : {}), + metadata: compactRecord({ + ...(testCase.metadata ?? {}), + taskKind: testCase.taskKind, + source: testCase.source, + }), + steps: [ + ...testCase.events.map((event, eventIndex) => ({ + id: `event:${event.id}`, + scope: compactScope( + options.eventScope?.({ event, case: testCase, eventIndex }) ?? { + agentId: memoryAgentId, + sessionId: testCase.id, + }, + ), + writes: [ + { + id: event.id, + kind: 'observation' as const, + text: event.text, + metadata: compactRecord({ + ...(event.metadata ?? {}), + eventId: event.id, + actorId: event.actorId, + sessionId: event.sessionId, + timestamp: event.timestamp, + }), + }, + ], + metadata: { eventIndex }, + })), + { + id: 'probe', + scope: compactScope( + options.probeScope?.(testCase) ?? { + agentId: memoryAgentId, + sessionId: testCase.id, + }, + ), + probes: [ + { + id: 'answer', + query: testCase.prompt, + taskKind: testCase.taskKind, + ...(testCase.requiredFacts !== undefined + ? { requiredFacts: testCase.requiredFacts } + : {}), + ...(testCase.forbiddenFacts !== undefined + ? { forbiddenFacts: testCase.forbiddenFacts } + : {}), + ...(testCase.expectedEventIds !== undefined + ? { expectedEventIds: testCase.expectedEventIds } + : {}), + ...(testCase.expectedActorIds !== undefined + ? { expectedActorIds: testCase.expectedActorIds } + : {}), + ...(testCase.referenceAnswer !== undefined + ? { referenceAnswer: testCase.referenceAnswer } + : {}), + }, + ], + }, + ], + })) +} + +export function buildAgentMemorySequenceScenarios( + sequences: readonly AgentMemorySequence[], + candidates: readonly Pick[], +): AgentMemorySequenceScenario[] { + return candidates.flatMap((candidate) => + sequences.map((sequence) => ({ + id: `${stableId('candidate', candidate.id)}:${sequence.id}`, + kind: 'agent-memory-sequence' as const, + candidateId: candidate.id, + sequenceId: sequence.id, + sequence, + seedGroup: sequence.id, + tags: [...new Set([sequence.split ?? 'dev', ...(sequence.tags ?? []), candidate.id])], + })), + ) +} + +export function agentMemorySequenceJudge(): JudgeConfig< + AgentMemorySequenceArtifact, + AgentMemorySequenceScenario +> { + return { + name: 'agent-memory-sequence', + judgeVersion: 'agent-knowledge:memory-sequence:v2', + dimensions: [ + { key: 'score', description: 'mean memory probe score' }, + { key: 'passed', description: '1 when every memory probe passes' }, + { key: 'memory_fact_recall', description: 'current memory fact coverage' }, + { key: 'memory_event_recall', description: 'memory source event coverage' }, + { key: 'memory_actor_recall', description: 'memory actor attribution coverage' }, + { key: 'memory_stale_safe', description: '1 when obsolete memory is not reused' }, + ], + score({ artifact }) { + return { + composite: artifact.score, + dimensions: { + score: artifact.score, + passed: artifact.passed ? 1 : 0, + ...artifact.dimensions, + }, + notes: `${artifact.probes.filter((probe) => probe.passed).length}/${artifact.probes.length} probes passed`, + } + }, + } +} diff --git a/src/memory/experiment/cell.ts b/src/memory/experiment/cell.ts new file mode 100644 index 0000000..585410c --- /dev/null +++ b/src/memory/experiment/cell.ts @@ -0,0 +1,392 @@ +import { randomUUID } from 'node:crypto' +import type { CampaignStorage, DispatchContext } from '@tangle-network/agent-eval/campaign' +import { scoreMemoryBenchmarkArtifact } from '../../benchmarks/index' +import { stableId } from '../../ids' +import { type AgentMemoryBranch, createAgentMemoryBranch } from '../branch' +import { resolveMemoryCleanupTimeoutMs, runBoundedMemoryLifecycle } from '../lifecycle' +import type { AgentMemoryAdapter } from '../types' +import { countDimensions, mean, meanDimensions } from './metrics' +import { appendMemoryAttemptEvent, memoryAttemptCostCallId, memoryAttemptEvent } from './recovery' +import { + AgentMemoryCleanupError, + clearSequenceScopes, + memoryExperimentBaseScope, + mergeScopes, + sequenceCleanupScopes, + trackExternalMemoryCalls, +} from './runtime' +import type { + AgentMemoryExperimentCandidate, + AgentMemorySequence, + AgentMemorySequenceArtifact, + AgentMemorySequenceProbe, + AgentMemorySequenceProbeResult, + AgentMemorySequenceScenario, + AgentMemorySequenceStep, + OwnedMemoryExperimentRunLease, + RunAgentMemoryExperimentOptions, +} from './types' +import { stringArray, uniqueStrings } from './validation' + +export async function runSequenceCell(input: { + options: RunAgentMemoryExperimentOptions + candidate: AgentMemoryExperimentCandidate + scenario: AgentMemorySequenceScenario + context: DispatchContext + runIdentity: string + storage: CampaignStorage + attemptLogPath: string + lease: OwnedMemoryExperimentRunLease +}): Promise { + const { options, candidate, scenario, context, runIdentity, storage, attemptLogPath, lease } = + input + const cleanupBranches = options.cleanupBranches ?? true + const costUsd = candidate.externalCostUsdPerSequence ?? 0 + if (!Number.isFinite(costUsd) || costUsd < 0) { + throw new Error( + `${candidate.id}: externalCostUsdPerSequence must be a non-negative finite number`, + ) + } + if (!cleanupBranches && !candidate.disposeAdapter) { + throw new Error( + `${candidate.id}: cleanupBranches=false requires disposeAdapter to delete isolated external state`, + ) + } + const cleanupTimeoutMs = resolveMemoryCleanupTimeoutMs( + options.cleanupTimeoutMs, + `${candidate.id}: memory sequence`, + ) + + context.signal.throwIfAborted() + await lease.assertOwned() + const startedAt = Date.now() + const branchId = stableId( + 'memory_branch', + `${runIdentity}:${context.cellId}:${context.seed}:${randomUUID()}`, + ) + const attempt = memoryAttemptEvent({ + status: 'started', + branchId, + candidate, + sequence: scenario.sequence, + rep: context.rep, + seed: context.seed, + recovery: false, + cleanupBranches, + now: options.now, + }) + appendMemoryAttemptEvent(storage, attemptLogPath, attempt) + + let externalCallAttempted = false + const appendCleanedAttempt = (priorError?: unknown): void => { + try { + appendMemoryAttemptEvent(storage, attemptLogPath, { + ...attempt, + status: 'cleaned', + recordedAt: (options.now ?? (() => new Date()))().toISOString(), + }) + } catch (error) { + throw new AgentMemoryCleanupError( + [...(priorError ? [priorError] : []), error], + `${candidate.id}: memory branch cleanup could not be recorded`, + ) + } + } + const execute = async (): Promise => { + context.signal.throwIfAborted() + await lease.assertOwned() + let rawAdapter: AgentMemoryAdapter | undefined + let adapter: AgentMemoryAdapter | undefined + let memory: AgentMemoryBranch | undefined + let primaryError: unknown + let completedArtifact: AgentMemorySequenceArtifact | undefined + let finalClearStarted = false + let finalClearCompleted = false + try { + const created = await candidate.createAdapter({ + branchId, + sequence: scenario.sequence, + rep: context.rep, + seed: context.seed, + purpose: 'execute', + signal: context.signal, + markExternalCall: () => { + externalCallAttempted = true + }, + }) + if (!created) throw new Error(`${candidate.id}: createAdapter returned no execution adapter`) + rawAdapter = created + adapter = trackExternalMemoryCalls(created, () => { + externalCallAttempted = true + }) + await lease.assertOwned() + context.signal.throwIfAborted() + if (cleanupBranches && !adapter.clear) { + throw new Error( + `${candidate.id}: cleanupBranches requires an adapter with scoped clear support`, + ) + } + memory = createAgentMemoryBranch({ + adapter, + branchId, + lifetime: 'attempt', + policy: candidate.policy, + allowedWriteScopes: sequenceCleanupScopes(scenario.sequence), + baseScope: memoryExperimentBaseScope(options, candidate, scenario.sequenceId), + }) + const probes: AgentMemorySequenceProbeResult[] = [] + for (const step of scenario.sequence.steps) { + context.signal.throwIfAborted() + await lease.assertOwned() + await writeStep(memory, step) + await lease.assertOwned() + await options.executeStep?.({ + memory, + candidateId: candidate.id, + sequence: scenario.sequence, + step, + context, + }) + context.signal.throwIfAborted() + await lease.assertOwned() + const stepProbes = await probeStep(memory, scenario.sequence, step) + await lease.assertOwned() + probes.push(...stepProbes) + } + const snapshot = await memory.snapshot() + await lease.assertOwned() + await options.onBranchSnapshot?.({ + candidateId: candidate.id, + sequenceId: scenario.sequenceId, + cellId: context.cellId, + snapshot, + }) + await lease.assertOwned() + const dimensions = meanDimensions(probes.map((probe) => probe.dimensions)) + const dimensionSampleCounts = countDimensions( + probes.map((probe) => probe.applicableDimensions), + ) + const artifact: AgentMemorySequenceArtifact = { + candidateId: candidate.id, + sequenceId: scenario.sequenceId, + score: mean(probes.map((probe) => probe.score)), + passed: probes.length > 0 && probes.every((probe) => probe.passed), + dimensions, + dimensionSampleCounts, + probes, + branchDigest: snapshot.digest, + journalEntries: snapshot.journal.length, + durationMs: Math.max(0, Date.now() - startedAt), + } + if (cleanupBranches) { + finalClearStarted = true + await runBoundedMemoryLifecycle({ + operation: `${candidate.id}: final branch cleanup`, + timeoutMs: cleanupTimeoutMs, + resource: adapter, + run: () => clearSequenceScopes(memory!, scenario.sequence), + }) + finalClearCompleted = true + await lease.assertOwned() + } + completedArtifact = artifact + } catch (error) { + primaryError = error + } + const cleanupErrors: unknown[] = [] + let cleanupOwned = true + let ownershipError: unknown + try { + await lease.assertOwned() + } catch (error) { + cleanupOwned = false + ownershipError = error + } + if (finalClearStarted && !finalClearCompleted && primaryError) { + cleanupErrors.push(primaryError) + } + if ( + primaryError && + cleanupOwned && + !finalClearStarted && + memory && + cleanupBranches && + adapter?.clear + ) { + try { + await runBoundedMemoryLifecycle({ + operation: `${candidate.id}: failed branch cleanup`, + timeoutMs: cleanupTimeoutMs, + resource: adapter, + run: () => clearSequenceScopes(memory!, scenario.sequence), + }) + } catch (error) { + cleanupErrors.push(error) + } + } + if (adapter) { + try { + await runBoundedMemoryLifecycle({ + operation: `${candidate.id}: adapter close`, + timeoutMs: cleanupTimeoutMs, + resource: adapter, + run: async () => { + if (memory && cleanupOwned) await memory.close?.() + else { + if (cleanupOwned) await adapter!.flush?.() + await adapter!.close?.() + } + }, + }) + } catch (error) { + cleanupErrors.push(error) + } + if (cleanupOwned) { + try { + await runBoundedMemoryLifecycle({ + operation: `${candidate.id}: adapter disposal`, + timeoutMs: cleanupTimeoutMs, + resource: adapter, + run: () => { + if (candidate.disposeAdapter) externalCallAttempted = true + return candidate.disposeAdapter?.(rawAdapter!) + }, + }) + } catch (error) { + cleanupErrors.push(error) + } + } + } else { + cleanupErrors.push( + new Error(`${candidate.id}: adapter creation failed before cleanup could be confirmed`), + ) + } + if (cleanupOwned && cleanupErrors.length === 0) appendCleanedAttempt(primaryError) + if (!cleanupOwned && cleanupErrors.length === 0) { + if (primaryError) throw primaryError + throw ownershipError + } + if (cleanupErrors.length > 0) { + const primaryMessage = + primaryError instanceof Error ? primaryError.message : String(primaryError ?? '') + throw new AgentMemoryCleanupError( + [ + ...(primaryError && !cleanupErrors.includes(primaryError) ? [primaryError] : []), + ...(ownershipError ? [ownershipError] : []), + ...cleanupErrors, + ], + `${candidate.id}: memory branch cleanup failed${primaryMessage ? ` after: ${primaryMessage}` : ''}`, + ) + } + if (primaryError) throw primaryError + if (!completedArtifact) throw new Error(`${candidate.id}: memory sequence produced no result`) + return completedArtifact + } + + if (costUsd === 0) { + let artifact: AgentMemorySequenceArtifact | undefined + let error: unknown + try { + artifact = await execute() + } catch (caught) { + error = caught + } + if (error) throw error + if (!artifact) throw new Error(`${candidate.id}: memory sequence produced no result`) + return artifact + } + const receipt = { + model: candidate.id, + inputTokens: 0, + outputTokens: 0, + actualCostUsd: costUsd, + } as const + const paid = await context.cost.runPaidCall({ + callId: memoryAttemptCostCallId(attempt, 'execute', 0), + actor: `agent-knowledge:memory-experiment:${candidate.id}`, + model: candidate.id, + maximumCharge: { externallyEnforcedMaximumUsd: costUsd }, + execute, + receipt: () => receipt, + receiptFromError: () => ({ + ...receipt, + actualCostUsd: externalCallAttempted ? costUsd : 0, + }), + }) + if (!paid.succeeded) throw paid.error + return paid.value +} + +async function writeStep(memory: AgentMemoryBranch, step: AgentMemorySequenceStep): Promise { + const writes = (step.writes ?? []).map((write) => ({ + ...write, + scope: mergeScopes(step.scope, write.scope), + })) + if (step.parallelWrites) { + await Promise.all(writes.map((write) => memory.write(write))) + return + } + for (const write of writes) await memory.write(write) +} + +async function probeStep( + memory: AgentMemoryBranch, + sequence: AgentMemorySequence, + step: AgentMemorySequenceStep, +): Promise { + const run = async (probe: AgentMemorySequenceProbe): Promise => { + const scope = mergeScopes(step.scope, probe.scope) + const context = await memory.getContext(probe.query, { scope, limit: probe.limit }) + const artifact = { + answer: context.text, + rememberedFacts: context.hits.map((hit) => hit.text), + citedEventIds: uniqueStrings([ + ...context.hits.map((hit) => hit.metadata?.eventId), + ...context.hits.flatMap((hit) => stringArray(hit.metadata?.eventIds)), + ]), + usedMemoryIds: context.hits.map((hit) => hit.id), + actorIds: uniqueStrings([ + ...context.hits.map((hit) => hit.metadata?.actorId), + ...context.hits.flatMap((hit) => stringArray(hit.metadata?.actorIds)), + ]), + } + const evaluation = scoreMemoryBenchmarkArtifact( + { + id: `${sequence.id}:${step.id}:${probe.id}`, + family: sequence.family, + taskKind: probe.taskKind ?? 'memory-recall', + split: sequence.split, + events: [], + prompt: probe.query, + requiredFacts: + probe.requiredFacts && probe.requiredFacts.length > 0 + ? probe.requiredFacts + : probe.referenceAnswer + ? [{ id: `${probe.id}:reference`, anyOf: [probe.referenceAnswer] }] + : undefined, + forbiddenFacts: probe.forbiddenFacts, + expectedEventIds: probe.expectedEventIds, + expectedActorIds: probe.expectedActorIds, + referenceAnswer: probe.referenceAnswer, + }, + artifact, + ) + return { + id: probe.id, + stepId: step.id, + query: probe.query, + score: evaluation.score, + passed: evaluation.passed, + dimensions: evaluation.dimensions, + applicableDimensions: evaluation.applicableDimensions ?? Object.keys(evaluation.dimensions), + notes: evaluation.notes, + hitIds: context.hits.map((hit) => hit.id), + } + } + if (step.parallelProbes === false) { + const results: AgentMemorySequenceProbeResult[] = [] + for (const probe of step.probes ?? []) results.push(await run(probe)) + return results + } + return Promise.all((step.probes ?? []).map(run)) +} diff --git a/src/memory/experiment/metrics.ts b/src/memory/experiment/metrics.ts new file mode 100644 index 0000000..77afdef --- /dev/null +++ b/src/memory/experiment/metrics.ts @@ -0,0 +1,131 @@ +import type { CampaignResult, CostLedgerHandle } from '@tangle-network/agent-eval/campaign' +import type { + AgentMemoryExperimentCandidate, + AgentMemoryExperimentRankingRow, + AgentMemorySequenceArtifact, + AgentMemorySequenceScenario, +} from './types' + +export function rankAgentMemoryExperiment( + candidates: readonly AgentMemoryExperimentCandidate[], + scenarios: readonly AgentMemorySequenceScenario[], + campaign: CampaignResult, + costByCandidate: ReadonlyMap, +): AgentMemoryExperimentRankingRow[] { + const scenarioById = new Map(scenarios.map((scenario) => [scenario.id, scenario])) + const rows = candidates.map((candidate): AgentMemoryExperimentRankingRow => { + const candidateScenarios = scenarios.filter((scenario) => scenario.candidateId === candidate.id) + const cells = campaign.cells.filter( + (cell) => scenarioById.get(cell.scenarioId)?.candidateId === candidate.id, + ) + const successful = cells.filter((cell) => !cell.error && cell.artifact) + const dimensionRows = successful.map((cell) => cell.artifact.dimensions) + return { + rank: 0, + candidateId: candidate.id, + label: candidate.label ?? candidate.id, + scoreMean: mean( + cells.map((cell) => (!cell.error && cell.artifact ? cell.artifact.score : 0)), + ), + passRate: mean(cells.map((cell) => (!cell.error && cell.artifact?.passed ? 1 : 0))), + totalSequences: candidateScenarios.length, + totalCells: cells.length, + totalProbes: successful.reduce((sum, cell) => sum + cell.artifact.probes.length, 0), + cellsFailed: cells.filter((cell) => Boolean(cell.error)).length, + totalCostUsd: normalizeUsd(costByCandidate.get(candidate.id) ?? 0), + durationMs: cells.reduce((sum, cell) => sum + cell.durationMs, 0), + dimensions: meanDimensions(dimensionRows), + } + }) + return rows + .sort( + (a, b) => + Number(a.cellsFailed > 0) - Number(b.cellsFailed > 0) || + b.scoreMean - a.scoreMean || + b.passRate - a.passRate || + a.totalCostUsd - b.totalCostUsd || + a.candidateId.localeCompare(b.candidateId), + ) + .map((row, index) => ({ ...row, rank: index + 1 })) +} + +export function memoryExperimentCostByCandidate( + costLedger: CostLedgerHandle, + runDir: string, + scenarios: readonly AgentMemorySequenceScenario[], + candidateIdsInput: readonly string[], +): ReadonlyMap { + const candidateByScenario = new Map( + scenarios.map((scenario) => [scenario.id, scenario.candidateId]), + ) + const candidateIds = new Set(candidateIdsInput) + const totals = new Map() + for (const receipt of costLedger.list()) { + if (receipt.tags?.runDir !== runDir) continue + const scenarioCandidate = receipt.tags.scenarioId + ? candidateByScenario.get(receipt.tags.scenarioId) + : undefined + const recoveryCandidate = + receipt.tags.memoryRecovery === 'attempt' && receipt.tags.candidateId + ? receipt.tags.candidateId + : undefined + const candidateId = scenarioCandidate ?? recoveryCandidate + if (!candidateId || !candidateIds.has(candidateId)) continue + totals.set(candidateId, (totals.get(candidateId) ?? 0) + receipt.costUsd) + } + return totals +} + +export function renderAgentMemoryExperimentRanking( + rows: readonly AgentMemoryExperimentRankingRow[], + totalCostUsd: number, + unrankedRecoveryCostUsd: number, +): string { + return [ + '# Agent Memory Experiment', + '', + `- total cost: $${format(totalCostUsd)}`, + `- retired-candidate recovery cost: $${format(unrankedRecoveryCostUsd)}`, + '', + '| rank | candidate | sequences | cells | probes | failed | score | pass rate | cost | duration ms |', + '| ---: | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |', + ...rows.map( + (row) => + `| ${row.rank} | ${row.label} | ${row.totalSequences} | ${row.totalCells} | ${row.totalProbes} | ${row.cellsFailed} | ${format(row.scoreMean)} | ${format(row.passRate)} | $${format(row.totalCostUsd)} | ${format(row.durationMs)} |`, + ), + '', + ].join('\n') +} + +export function meanDimensions(rows: readonly Record[]): Record { + const values = new Map() + for (const row of rows) { + for (const [key, value] of Object.entries(row)) { + if (!Number.isFinite(value)) continue + const bucket = values.get(key) ?? [] + bucket.push(value) + values.set(key, bucket) + } + } + return Object.fromEntries([...values].map(([key, bucket]) => [key, mean(bucket)])) +} + +export function countDimensions(rows: readonly (readonly string[])[]): Record { + const counts = new Map() + for (const row of rows) { + for (const key of new Set(row)) counts.set(key, (counts.get(key) ?? 0) + 1) + } + return Object.fromEntries(counts) +} + +export function mean(values: readonly number[]): number { + return values.length === 0 ? 0 : values.reduce((sum, value) => sum + value, 0) / values.length +} + +export function format(value: number): string { + return Number.isFinite(value) ? value.toFixed(4) : '0.0000' +} + +export function normalizeUsd(value: number): number { + return Number(value.toFixed(12)) +} diff --git a/src/memory/experiment/recovery.ts b/src/memory/experiment/recovery.ts new file mode 100644 index 0000000..9ea2f20 --- /dev/null +++ b/src/memory/experiment/recovery.ts @@ -0,0 +1,491 @@ +import { canonicalJson } from '@tangle-network/agent-eval' +import type { CampaignStorage, CostLedgerHandle } from '@tangle-network/agent-eval/campaign' +import { stableId } from '../../ids' +import { + appendAttemptJournalEvent, + assertNoInterruptedPaidCalls, + hasSettledPaidCall, + readActiveAttemptJournal, + reconcileInterruptedMemoryPaidCalls, + reserveRecoveryAttempts, +} from '../attempt-log' +import { type AgentMemoryBranch, createAgentMemoryBranch } from '../branch' +import { + createMemoryExecutionPool, + memoryRecoveryDelayMs, + releaseMemoryAdapterCreatedAfterAbort, + resolveMemoryCleanupTimeoutMs, + runBoundedMemoryLifecycle, + sleepForMemoryRecovery, +} from '../lifecycle' +import type { AgentMemoryAdapter } from '../types' +import { + AgentMemoryCleanupError, + clearSequenceScopes, + memoryExperimentBaseScope, + sequenceCleanupScopes, + trackExternalMemoryCalls, +} from './runtime' +import type { + AgentMemoryAttemptEvent, + AgentMemoryExperimentCandidate, + AgentMemorySequence, + OwnedMemoryExperimentRunLease, + RunAgentMemoryExperimentOptions, +} from './types' + +export async function recoverAbandonedMemoryAttempts(input: { + options: RunAgentMemoryExperimentOptions + storage: CampaignStorage + runDir: string + attemptLogPath: string + candidateById: ReadonlyMap + sequenceById: ReadonlyMap + lease: OwnedMemoryExperimentRunLease + maxConcurrency: number + costLedger: CostLedgerHandle + maxRecoveryAttempts: number + recoveryLogPath: string + maxRecoveryRetriesPerAttempt: number +}): Promise { + let attempts = readActiveMemoryAttempts(input.storage, input.attemptLogPath) + if (attempts.length > input.maxRecoveryAttempts) { + throw new Error( + `memory experiment has ${attempts.length} unfinished attempts; maxRecoveryAttempts is ${input.maxRecoveryAttempts}`, + ) + } + for (const attempt of attempts) { + const candidate = input.candidateById.get(attempt.candidateId) + if (!candidate) { + throw new Error( + `cannot recover memory branch '${attempt.branchId}': candidate '${attempt.candidateId}' is missing; pass it in recoveryCandidates`, + ) + } + assertMemoryAttemptCandidateMatches(attempt, candidate) + if (!input.sequenceById.has(attempt.sequenceId)) { + throw new Error( + `cannot recover memory branch '${attempt.branchId}': sequence '${attempt.sequenceId}' is missing`, + ) + } + if ((input.options.cleanupBranches ?? true) !== attempt.cleanupBranches) { + throw new Error(`cannot recover memory branch '${attempt.branchId}': cleanupBranches changed`) + } + } + + reconcileInterruptedMemoryPaidCalls(input.costLedger) + assertNoInterruptedPaidCalls(input.costLedger, 'memory experiment recovery') + + for (const attempt of attempts) { + const candidate = input.candidateById.get(attempt.candidateId)! + const executionCostUsd = candidate.externalCostUsdPerSequence ?? 0 + if ( + executionCostUsd > 0 && + !hasSettledPaidCall(input.costLedger, memoryAttemptCostCallId(attempt, 'execute', 0)) + ) { + appendMemoryAttemptEvent(input.storage, input.attemptLogPath, { + ...attempt, + status: 'cleaned', + recovery: true, + recordedAt: (input.options.now ?? (() => new Date()))().toISOString(), + }) + } + } + attempts = readActiveMemoryAttempts(input.storage, input.attemptLogPath) + const recoveryGenerations = reserveRecoveryAttempts({ + storage: input.storage, + path: input.recoveryLogPath, + attemptIds: attempts.map((attempt) => attempt.branchId), + maxRetriesPerAttempt: input.maxRecoveryRetriesPerAttempt, + label: 'memory recovery attempt log', + now: input.options.now, + }) + const pool = createMemoryExecutionPool(input.maxConcurrency) + const settled = await Promise.allSettled( + attempts + .sort((left, right) => left.branchId.localeCompare(right.branchId)) + .map((attempt) => + pool.run(async () => { + await input.lease.assertOwned() + const candidate = input.candidateById.get(attempt.candidateId) + if (!candidate) { + throw new Error( + `cannot recover memory branch '${attempt.branchId}': candidate '${attempt.candidateId}' is missing; pass it in recoveryCandidates`, + ) + } + assertMemoryAttemptCandidateMatches(attempt, candidate) + const sequence = input.sequenceById.get(attempt.sequenceId) + if (!sequence) { + throw new Error( + `cannot recover memory branch '${attempt.branchId}': sequence '${attempt.sequenceId}' is missing`, + ) + } + if ((input.options.cleanupBranches ?? true) !== attempt.cleanupBranches) { + throw new Error( + `cannot recover memory branch '${attempt.branchId}': cleanupBranches changed`, + ) + } + const recoveryCostUsd = candidate.externalRecoveryCostUsdPerAttempt ?? 0 + const recoveryGeneration = recoveryGenerations.get(attempt.branchId) + if (recoveryGeneration === undefined) { + throw new Error(`missing recovery generation for memory branch '${attempt.branchId}'`) + } + let externalRecoveryAttempted = false + const recover = async (): Promise => { + await recoverMemoryAttempt({ + options: input.options, + candidate, + sequence, + attempt, + lease: input.lease, + onExternalCall: () => { + externalRecoveryAttempted = true + }, + }) + appendMemoryAttemptEvent(input.storage, input.attemptLogPath, { + ...attempt, + status: 'cleaned', + recovery: true, + recordedAt: (input.options.now ?? (() => new Date()))().toISOString(), + }) + } + if (recoveryCostUsd === 0) { + await recover() + } else { + const tags = memoryRecoveryCostTags(input.runDir, candidate.id, attempt.branchId) + const receipt = { + model: candidate.id, + inputTokens: 0, + outputTokens: 0, + actualCostUsd: recoveryCostUsd, + } as const + const paid = await input.costLedger.runPaidCall({ + callId: memoryAttemptCostCallId(attempt, 'recovery', recoveryGeneration), + channel: 'driver', + phase: `${input.options.costPhase ?? 'memory.experiment'}.recovery`, + actor: `agent-knowledge:memory-recovery:${candidate.id}`, + model: candidate.id, + tags, + maximumCharge: { externallyEnforcedMaximumUsd: recoveryCostUsd }, + execute: recover, + receipt: () => ({ + ...receipt, + actualCostUsd: externalRecoveryAttempted ? recoveryCostUsd : 0, + }), + receiptFromError: () => ({ + ...receipt, + actualCostUsd: externalRecoveryAttempted ? recoveryCostUsd : 0, + }), + }) + if (!paid.succeeded) throw paid.error + } + }), + ), + ) + const failures = settled.flatMap((result) => + result.status === 'rejected' ? [result.reason] : [], + ) + if (failures.length === 1) throw failures[0] + if (failures.length > 1) { + throw new AggregateError(failures, 'multiple abandoned memory branches failed recovery') + } +} + +async function recoverMemoryAttempt(input: { + options: RunAgentMemoryExperimentOptions + candidate: AgentMemoryExperimentCandidate + sequence: AgentMemorySequence + attempt: AgentMemoryAttemptEvent + lease: OwnedMemoryExperimentRunLease + onExternalCall(): void +}): Promise { + const { options, candidate, sequence, attempt, lease, onExternalCall } = input + const cleanupBranches = attempt.cleanupBranches + const cleanupTimeoutMs = resolveMemoryCleanupTimeoutMs( + options.cleanupTimeoutMs, + `${candidate.id}: abandoned branch recovery`, + ) + let rawAdapter: AgentMemoryAdapter | undefined + let adapter: AgentMemoryAdapter | undefined + let memory: AgentMemoryBranch | undefined + let primaryError: unknown + try { + const abortController = new AbortController() + const creation = Promise.resolve().then(() => + candidate.createAdapter({ + branchId: attempt.branchId, + sequence, + rep: attempt.rep, + seed: attempt.seed, + purpose: 'recovery', + signal: abortController.signal, + markExternalCall: onExternalCall, + }), + ) + releaseMemoryAdapterCreatedAfterAbort({ + creation, + signal: abortController.signal, + dispose: candidate.disposeAdapter + ? async (created) => { + onExternalCall() + await candidate.disposeAdapter?.(created) + } + : undefined, + }) + const recovered = await runBoundedMemoryLifecycle({ + operation: `${candidate.id}: recovery adapter creation`, + timeoutMs: cleanupTimeoutMs, + abortController, + run: () => creation, + }) + await lease.assertOwned() + if (recovered === null) return + rawAdapter = recovered + adapter = trackExternalMemoryCalls(recovered, onExternalCall) + const recoveryDelayMs = memoryRecoveryDelayMs(adapter) + await sleepForMemoryRecovery( + recoveryDelayMs, + () => lease.assertOwned(), + cleanupTimeoutMs, + `${candidate.id}: abandoned branch recovery visibility wait`, + ) + if (cleanupBranches) { + if (!adapter.clear) { + throw new Error( + `${candidate.id}: abandoned branch recovery requires an adapter with scoped clear support`, + ) + } + memory = createAgentMemoryBranch({ + adapter, + branchId: attempt.branchId, + lifetime: 'attempt', + policy: candidate.policy, + allowedWriteScopes: sequenceCleanupScopes(sequence), + baseScope: memoryExperimentBaseScope(options, candidate, sequence.id), + }) + await runBoundedMemoryLifecycle({ + operation: `${candidate.id}: abandoned branch cleanup`, + timeoutMs: cleanupTimeoutMs, + resource: adapter, + run: () => clearSequenceScopes(memory!, sequence), + }) + await lease.assertOwned() + } + } catch (error) { + primaryError = error + } + + const cleanupErrors: unknown[] = [] + let cleanupOwned = true + try { + await lease.assertOwned() + } catch (error) { + cleanupOwned = false + cleanupErrors.push(error) + } + if (adapter) { + try { + await runBoundedMemoryLifecycle({ + operation: `${candidate.id}: recovery adapter close`, + timeoutMs: cleanupTimeoutMs, + resource: adapter, + run: async () => { + if (memory && cleanupOwned) { + await memory.close?.() + } else { + if (cleanupOwned && adapter!.flush) { + await adapter!.flush() + } + await adapter!.close?.() + } + }, + }) + } catch (error) { + cleanupErrors.push(error) + } + if (cleanupOwned) { + try { + if (candidate.disposeAdapter) onExternalCall() + await runBoundedMemoryLifecycle({ + operation: `${candidate.id}: recovery adapter disposal`, + timeoutMs: cleanupTimeoutMs, + resource: adapter, + run: () => candidate.disposeAdapter?.(rawAdapter!), + }) + } catch (error) { + cleanupErrors.push(error) + } + } + } else { + cleanupErrors.push( + new Error( + `${candidate.id}: recovery adapter creation failed before cleanup could be confirmed`, + ), + ) + } + if (!cleanupOwned || primaryError || cleanupErrors.length > 0) { + throw new AgentMemoryCleanupError( + [...(primaryError ? [primaryError] : []), ...cleanupErrors], + `${candidate.id}: abandoned memory branch '${attempt.branchId}' recovery failed`, + ) + } +} + +export function memoryAttemptCostCallId( + attempt: AgentMemoryAttemptEvent, + purpose: 'execute' | 'recovery', + generation: number, +): string { + return stableId( + 'memory_cost_call', + canonicalJson({ + purpose, + generation, + branchId: attempt.branchId, + candidateId: attempt.candidateId, + candidateRef: attempt.candidateRef, + externalCostUsdPerSequence: attempt.externalCostUsdPerSequence, + externalRecoveryCostUsdPerAttempt: attempt.externalRecoveryCostUsdPerAttempt, + sequenceId: attempt.sequenceId, + rep: attempt.rep, + seed: attempt.seed, + }), + ) +} + +function memoryRecoveryCostTags( + runDir: string, + candidateId: string, + branchId: string, +): Record { + return { + runDir, + candidateId, + branchId, + memoryRecovery: 'attempt', + } +} + +export function memoryAttemptEvent(input: { + status: AgentMemoryAttemptEvent['status'] + branchId: string + candidate: AgentMemoryExperimentCandidate + sequence: AgentMemorySequence + rep: number + seed: number + recovery: boolean + cleanupBranches?: boolean + now?: () => Date +}): AgentMemoryAttemptEvent { + return { + schema: 2, + status: input.status, + branchId: input.branchId, + candidateId: input.candidate.id, + candidateRef: input.candidate.ref, + sequenceId: input.sequence.id, + rep: input.rep, + seed: input.seed, + cleanupBranches: input.cleanupBranches ?? true, + externalCostUsdPerSequence: input.candidate.externalCostUsdPerSequence ?? 0, + externalRecoveryCostUsdPerAttempt: input.candidate.externalRecoveryCostUsdPerAttempt ?? 0, + recordedAt: (input.now ?? (() => new Date()))().toISOString(), + recovery: input.recovery, + } +} + +export function appendMemoryAttemptEvent( + storage: CampaignStorage, + path: string, + event: AgentMemoryAttemptEvent, +): void { + appendAttemptJournalEvent({ + storage, + path, + event, + label: 'memory attempt log', + }) +} + +function readActiveMemoryAttempts( + storage: CampaignStorage, + path: string, +): AgentMemoryAttemptEvent[] { + return readActiveAttemptJournal({ + storage, + path, + label: 'memory attempt log', + parse: parseMemoryAttemptEvent, + id: (event) => event.branchId, + sameAttempt: sameMemoryAttempt, + }) +} + +function parseMemoryAttemptEvent( + value: unknown, + path: string, + line: number, +): AgentMemoryAttemptEvent { + const event = value as Partial | null + const valid = + typeof event === 'object' && + event !== null && + event.schema === 2 && + (event.status === 'started' || event.status === 'cleaned') && + typeof event.branchId === 'string' && + event.branchId.length > 0 && + typeof event.candidateId === 'string' && + event.candidateId.length > 0 && + typeof event.candidateRef === 'string' && + event.candidateRef.length > 0 && + typeof event.sequenceId === 'string' && + event.sequenceId.length > 0 && + Number.isSafeInteger(event.rep) && + Number.isSafeInteger(event.seed) && + typeof event.cleanupBranches === 'boolean' && + typeof event.externalCostUsdPerSequence === 'number' && + Number.isFinite(event.externalCostUsdPerSequence) && + event.externalCostUsdPerSequence >= 0 && + typeof event.externalRecoveryCostUsdPerAttempt === 'number' && + Number.isFinite(event.externalRecoveryCostUsdPerAttempt) && + event.externalRecoveryCostUsdPerAttempt >= 0 && + typeof event.recordedAt === 'string' && + !Number.isNaN(Date.parse(event.recordedAt)) && + typeof event.recovery === 'boolean' + if (!valid) throw new Error(`invalid memory attempt event in '${path}' line ${line}`) + return value as AgentMemoryAttemptEvent +} + +function sameMemoryAttempt(left: AgentMemoryAttemptEvent, right: AgentMemoryAttemptEvent): boolean { + return ( + left.branchId === right.branchId && + left.candidateId === right.candidateId && + left.candidateRef === right.candidateRef && + left.sequenceId === right.sequenceId && + left.rep === right.rep && + left.seed === right.seed && + left.cleanupBranches === right.cleanupBranches && + left.externalCostUsdPerSequence === right.externalCostUsdPerSequence && + left.externalRecoveryCostUsdPerAttempt === right.externalRecoveryCostUsdPerAttempt + ) +} + +function assertMemoryAttemptCandidateMatches( + attempt: AgentMemoryAttemptEvent, + candidate: AgentMemoryExperimentCandidate, +): void { + if (candidate.ref !== attempt.candidateRef) { + throw new Error( + `cannot recover memory branch '${attempt.branchId}': candidate ref changed from '${attempt.candidateRef}' to '${candidate.ref}'`, + ) + } + const executionCost = candidate.externalCostUsdPerSequence ?? 0 + const recoveryCost = candidate.externalRecoveryCostUsdPerAttempt ?? 0 + if ( + executionCost !== attempt.externalCostUsdPerSequence || + recoveryCost !== attempt.externalRecoveryCostUsdPerAttempt + ) { + throw new Error( + `cannot recover memory branch '${attempt.branchId}': candidate cost settings changed; start a new run or restore the recorded costs`, + ) + } +} diff --git a/src/memory/experiment/run.ts b/src/memory/experiment/run.ts new file mode 100644 index 0000000..92ffccf --- /dev/null +++ b/src/memory/experiment/run.ts @@ -0,0 +1,323 @@ +import { join } from 'node:path' +import { canonicalJson } from '@tangle-network/agent-eval' +import { + type CampaignResult, + type CampaignStorage, + createRunCostLedger, + fsCampaignStorage, + resolveRunDir, + runCampaign, +} from '@tangle-network/agent-eval/campaign' +import { stableId } from '../../ids' +import { DEFAULT_MEMORY_RECOVERY_RETRIES_PER_ATTEMPT } from '../attempt-log' +import { createMemoryExecutionPool, resolveMemoryCleanupTimeoutMs } from '../lifecycle' +import { acquireAgentMemoryRunLease } from '../run-control' +import { agentMemorySequenceJudge, buildAgentMemorySequenceScenarios } from './cases' +import { runSequenceCell } from './cell' +import { + memoryExperimentCostByCandidate, + normalizeUsd, + rankAgentMemoryExperiment, + renderAgentMemoryExperimentRanking, +} from './metrics' +import { recoverAbandonedMemoryAttempts } from './recovery' +import { AgentMemoryCleanupError } from './runtime' +import type { + AgentMemorySequenceArtifact, + AgentMemorySequenceScenario, + OwnedMemoryExperimentRunLease, + RunAgentMemoryExperimentOptions, + RunAgentMemoryExperimentResult, +} from './types' +import { assertMemorySequences, assertNonEmptyString, assertUnique } from './validation' + +const MEMORY_EXPERIMENT_IMPLEMENTATION_REF = 'agent-knowledge:memory-experiment:v6' + +/** Runs ordered, branch-isolated memory histories across candidate systems. */ +export async function runAgentMemoryExperiment( + options: RunAgentMemoryExperimentOptions, +): Promise { + assertNonEmptyString(options.experimentId, 'memory experiment experimentId') + assertNonEmptyString(options.runDir, 'memory experiment runDir') + if (options.experimentRunId !== undefined) { + assertNonEmptyString(options.experimentRunId, 'memory experiment experimentRunId') + } + if (options.sequences.length === 0) throw new Error('memory experiment requires sequences') + if (options.candidates.length === 0) throw new Error('memory experiment requires candidates') + if (options.executeStep && !options.executeStepRef) { + throw new Error('memory experiment executeStepRef is required when executeStep is configured') + } + assertUnique( + options.sequences.map((sequence) => sequence.id), + 'sequence', + ) + assertUnique( + [...options.candidates, ...(options.recoveryCandidates ?? [])].map((candidate) => candidate.id), + 'candidate', + ) + assertMemorySequences(options.sequences) + for (const candidate of [...options.candidates, ...(options.recoveryCandidates ?? [])]) { + if (options.cleanupBranches === false && !candidate.disposeAdapter) { + throw new Error( + `${candidate.id}: cleanupBranches=false requires disposeAdapter to delete isolated external state`, + ) + } + if ( + candidate.externalCostUsdPerSequence !== undefined && + (!Number.isFinite(candidate.externalCostUsdPerSequence) || + candidate.externalCostUsdPerSequence < 0) + ) { + throw new Error( + `${candidate.id}: externalCostUsdPerSequence must be a non-negative finite number`, + ) + } + if ( + candidate.externalRecoveryCostUsdPerAttempt !== undefined && + (!Number.isFinite(candidate.externalRecoveryCostUsdPerAttempt) || + candidate.externalRecoveryCostUsdPerAttempt < 0) + ) { + throw new Error( + `${candidate.id}: externalRecoveryCostUsdPerAttempt must be a non-negative finite number`, + ) + } + assertNonEmptyString(candidate.ref, `${candidate.id} ref`) + } + + const storage = options.storage ?? fsCampaignStorage() + const runDir = resolveRunDir(options.runDir, options.repo) + if (!storage.append) { + throw new Error('memory experiment requires CampaignStorage.append for durable attempt state') + } + resolveMemoryCleanupTimeoutMs(options.cleanupTimeoutMs, 'memory experiment') + const maxRecoveryAttempts = options.maxRecoveryAttempts ?? 1_000 + if (!Number.isSafeInteger(maxRecoveryAttempts) || maxRecoveryAttempts <= 0) { + throw new Error('memory experiment maxRecoveryAttempts must be a positive safe integer') + } + const maxRecoveryRetriesPerAttempt = + options.maxRecoveryRetriesPerAttempt ?? DEFAULT_MEMORY_RECOVERY_RETRIES_PER_ATTEMPT + if (!Number.isSafeInteger(maxRecoveryRetriesPerAttempt) || maxRecoveryRetriesPerAttempt <= 0) { + throw new Error( + 'memory experiment maxRecoveryRetriesPerAttempt must be a positive safe integer', + ) + } + storage.ensureDir(runDir) + const lease = await acquireAgentMemoryRunLease({ + experimentId: options.experimentId, + runDir, + storage, + customStorage: options.storage !== undefined, + lockFileName: 'memory-experiment.lock', + label: 'memory experiment', + controllerMode: options.controllerMode, + acquireRunLease: options.acquireRunLease, + }) + let result: RunAgentMemoryExperimentResult | undefined + let primaryError: unknown + try { + await lease.assertOwned() + result = await runOwnedAgentMemoryExperiment(options, storage, runDir, lease) + } catch (error) { + primaryError = error + } + let releaseError: unknown + try { + await lease.release() + } catch (error) { + releaseError = error + } + if (primaryError && releaseError) { + throw new AggregateError( + [primaryError, releaseError], + 'memory experiment failed and its controller lease could not be released', + ) + } + if (primaryError) throw primaryError + if (releaseError) throw releaseError + if (!result) throw new Error('memory experiment produced no result') + return result +} + +async function runOwnedAgentMemoryExperiment( + options: RunAgentMemoryExperimentOptions, + storage: CampaignStorage, + runDir: string, + lease: OwnedMemoryExperimentRunLease, +): Promise { + const runIdentity = stableId( + 'memory_run', + canonicalJson({ + experimentId: options.experimentId, + experimentRunId: options.experimentRunId ?? runDir, + }), + ) + const maxConcurrency = options.maxConcurrency ?? 2 + if (!Number.isSafeInteger(maxConcurrency) || maxConcurrency <= 0) { + throw new Error('memory experiment maxConcurrency must be a positive safe integer') + } + const executionPool = createMemoryExecutionPool(maxConcurrency) + const dispatchedExecutions: Promise[] = [] + const candidateById = new Map(options.candidates.map((candidate) => [candidate.id, candidate])) + const recoveryCandidateById = new Map( + [...options.candidates, ...(options.recoveryCandidates ?? [])].map((candidate) => [ + candidate.id, + candidate, + ]), + ) + const sequenceById = new Map(options.sequences.map((sequence) => [sequence.id, sequence])) + const scenarios = buildAgentMemorySequenceScenarios(options.sequences, options.candidates) + const attemptLogPath = join(runDir, 'memory-attempts.jsonl') + const recoveryLogPath = join(runDir, 'memory-recovery-attempts.jsonl') + const costCeiling = options.costCeiling ?? options.costLedger?.costCeilingUsd ?? 0 + const costLedger = + options.costLedger ?? + createRunCostLedger({ + storage, + runDir, + costCeilingUsd: costCeiling, + }) + if (costLedger.costCeilingUsd !== costCeiling) { + throw new Error('memory experiment costCeiling must match the shared cost ledger ceiling') + } + storage.ensureDir(runDir) + await recoverAbandonedMemoryAttempts({ + options, + storage, + runDir, + attemptLogPath, + candidateById: recoveryCandidateById, + sequenceById, + lease, + maxConcurrency, + costLedger, + maxRecoveryAttempts: options.maxRecoveryAttempts ?? 1_000, + recoveryLogPath, + maxRecoveryRetriesPerAttempt: + options.maxRecoveryRetriesPerAttempt ?? DEFAULT_MEMORY_RECOVERY_RETRIES_PER_ATTEMPT, + }) + await lease.assertOwned() + let campaign: CampaignResult | undefined + let campaignError: unknown + let settledExecutions: PromiseSettledResult[] = [] + try { + campaign = await runCampaign({ + scenarios, + dispatch: (scenario, context) => { + const candidate = candidateById.get(scenario.candidateId) + if (!candidate) throw new Error(`unknown memory candidate ${scenario.candidateId}`) + const operation = executionPool.run(() => + runSequenceCell({ + options, + candidate, + scenario, + context, + runIdentity, + storage, + attemptLogPath, + lease, + }), + ) + dispatchedExecutions.push(operation) + return operation + }, + dispatchRef: memoryExperimentDispatchRef(options), + judges: [agentMemorySequenceJudge()], + runDir, + storage, + seed: options.seed, + reps: options.reps, + resumable: options.resumable, + costCeiling, + costLedger, + costPhase: options.costPhase, + maxConcurrency, + dispatchTimeoutMs: options.dispatchTimeoutMs, + expectUsage: 'off', + now: options.now, + }) + } catch (error) { + campaignError = error + } finally { + settledExecutions = await Promise.allSettled(dispatchedExecutions) + } + const cleanupFailures = settledExecutions.flatMap((settled) => + settled.status === 'rejected' && settled.reason instanceof AgentMemoryCleanupError + ? [settled.reason] + : [], + ) + if (campaignError && cleanupFailures.length > 0) { + throw new AggregateError( + [campaignError, ...cleanupFailures], + 'memory experiment failed and provider cleanup also failed', + ) + } + if (campaignError) throw campaignError + if (cleanupFailures.length > 0) { + throw new AggregateError(cleanupFailures, 'memory experiment cleanup failed after dispatch') + } + if (!campaign) throw new Error('memory experiment produced no campaign result') + await lease.assertOwned() + const costByCandidate = memoryExperimentCostByCandidate( + costLedger, + campaign.runDir, + scenarios, + [...options.candidates, ...(options.recoveryCandidates ?? [])].map((candidate) => candidate.id), + ) + const unrankedRecoveryCostUsd = normalizeUsd( + (options.recoveryCandidates ?? []).reduce( + (sum, candidate) => sum + (costByCandidate.get(candidate.id) ?? 0), + 0, + ), + ) + const totalCostUsd = normalizeUsd( + [...costByCandidate.values()].reduce((sum, cost) => sum + cost, 0), + ) + const rows = rankAgentMemoryExperiment(options.candidates, scenarios, campaign, costByCandidate) + const rankingJsonPath = join(campaign.runDir, 'memory-experiment-ranking.json') + const rankingMarkdownPath = join(campaign.runDir, 'memory-experiment-ranking.md') + storage.write( + rankingJsonPath, + `${JSON.stringify( + { experimentId: options.experimentId, totalCostUsd, unrankedRecoveryCostUsd, rows }, + null, + 2, + )}\n`, + ) + storage.write( + rankingMarkdownPath, + renderAgentMemoryExperimentRanking(rows, totalCostUsd, unrankedRecoveryCostUsd), + ) + return { + campaign, + rows, + totalCostUsd, + unrankedRecoveryCostUsd, + leaderCandidateId: rows.find((row) => row.cellsFailed === 0)?.candidateId, + rankingJsonPath, + rankingMarkdownPath, + attemptLogPath, + recoveryLogPath, + } +} + +function memoryExperimentDispatchRef(options: RunAgentMemoryExperimentOptions): string { + return stableId( + 'memory_experiment', + canonicalJson({ + implementationRef: MEMORY_EXPERIMENT_IMPLEMENTATION_REF, + experimentId: options.experimentId, + experimentRunId: options.experimentRunId ?? null, + executeStepRef: options.executeStepRef ?? 'fixtures', + cleanupBranches: options.cleanupBranches ?? true, + candidates: options.candidates + .map((candidate) => ({ + id: candidate.id, + ref: candidate.ref, + policy: candidate.policy ?? null, + baseScope: candidate.baseScope ?? null, + externalCostUsdPerSequence: candidate.externalCostUsdPerSequence ?? 0, + externalRecoveryCostUsdPerAttempt: candidate.externalRecoveryCostUsdPerAttempt ?? 0, + })) + .sort((a, b) => a.id.localeCompare(b.id)), + }), + ) +} diff --git a/src/memory/experiment/runtime.ts b/src/memory/experiment/runtime.ts new file mode 100644 index 0000000..81aea45 --- /dev/null +++ b/src/memory/experiment/runtime.ts @@ -0,0 +1,110 @@ +import type { AgentMemoryBranch } from '../branch' +import type { AgentMemoryAdapter, AgentMemoryScope } from '../types' +import type { + AgentMemoryExperimentCandidate, + AgentMemorySequence, + RunAgentMemoryExperimentOptions, +} from './types' +import { normalizeCleanupScope } from './validation' + +export class AgentMemoryCleanupError extends AggregateError { + constructor(errors: Iterable, message: string) { + super(errors, message) + this.name = 'AgentMemoryCleanupError' + } +} + +export function trackExternalMemoryCalls( + adapter: AgentMemoryAdapter, + onExternalCall: () => void, +): AgentMemoryAdapter { + return { + id: adapter.id, + branchIsolation: adapter.branchIsolation, + search(query, options) { + onExternalCall() + return adapter.search.call(adapter, query, options) + }, + getContext(query, options) { + onExternalCall() + return adapter.getContext.call(adapter, query, options) + }, + write(input) { + onExternalCall() + return adapter.write.call(adapter, input) + }, + ...(adapter.clear + ? { + clear(scope?: AgentMemoryScope) { + onExternalCall() + return adapter.clear!.call(adapter, scope) + }, + } + : {}), + ...(adapter.flush + ? { + flush() { + onExternalCall() + return adapter.flush!.call(adapter) + }, + } + : {}), + ...(adapter.close + ? { + close() { + onExternalCall() + return adapter.close!.call(adapter) + }, + } + : {}), + } +} + +export function mergeScopes(base?: AgentMemoryScope, extra?: AgentMemoryScope): AgentMemoryScope { + return { + ...(base ?? {}), + ...(extra ?? {}), + tags: { ...(base?.tags ?? {}), ...(extra?.tags ?? {}) }, + } +} + +export function memoryExperimentBaseScope( + options: Pick, + candidate: Pick, + sequenceId: string, +): AgentMemoryScope { + return mergeScopes(candidate.baseScope, { + tags: { + memoryExperimentId: options.experimentId, + memoryCandidateId: candidate.id, + memorySequenceId: sequenceId, + }, + }) +} + +export function sequenceCleanupScopes(sequence: AgentMemorySequence): AgentMemoryScope[] { + const scopes = new Map() + for (const scope of sequence.cleanupScopes ?? []) { + const normalized = normalizeCleanupScope(scope) + scopes.set(JSON.stringify(normalized), normalized) + } + for (const step of sequence.steps) { + const candidates = [ + ...(step.scope ? [step.scope] : []), + ...(step.writes ?? []).map((write) => mergeScopes(step.scope, write.scope)), + ...(step.probes ?? []).map((probe) => mergeScopes(step.scope, probe.scope)), + ] + for (const scope of candidates) { + const normalized = normalizeCleanupScope(scope) + scopes.set(JSON.stringify(normalized), normalized) + } + } + return [...scopes.values()] +} + +export async function clearSequenceScopes( + memory: AgentMemoryBranch, + sequence: AgentMemorySequence, +): Promise { + for (const scope of sequenceCleanupScopes(sequence)) await memory.clear?.(scope) +} diff --git a/src/memory/experiment/types.ts b/src/memory/experiment/types.ts new file mode 100644 index 0000000..5e150fd --- /dev/null +++ b/src/memory/experiment/types.ts @@ -0,0 +1,227 @@ +import type { + CampaignResult, + CampaignStorage, + CostLedgerHandle, + DispatchContext, + Scenario, +} from '@tangle-network/agent-eval/campaign' +import type { + KnowledgeBenchmarkFamily, + KnowledgeBenchmarkSplit, + KnowledgeMemoryBenchmarkCase, + KnowledgeMemoryBenchmarkTaskKind, + KnowledgeMemoryEvent, + KnowledgeMemoryFactMatcher, +} from '../../benchmarks/index' +import type { + AgentMemoryBranch, + AgentMemoryBranchSnapshot, + AgentMemorySharingPolicy, +} from '../branch' +import type { + AgentMemoryAcquireRunLease, + AgentMemoryControllerMode, + AgentMemoryRunLease, + OwnedAgentMemoryRunLease, +} from '../run-control' +import type { AgentMemoryAdapter, AgentMemoryScope, AgentMemoryWriteInput } from '../types' + +export interface AgentMemorySequenceProbe { + id: string + query: string + scope?: AgentMemoryScope + limit?: number + taskKind?: KnowledgeMemoryBenchmarkTaskKind + requiredFacts?: readonly KnowledgeMemoryFactMatcher[] + forbiddenFacts?: readonly KnowledgeMemoryFactMatcher[] + expectedEventIds?: readonly string[] + expectedActorIds?: readonly string[] + referenceAnswer?: string +} + +export interface AgentMemorySequenceStep { + id: string + instruction?: string + scope?: AgentMemoryScope + writes?: readonly AgentMemoryWriteInput[] + parallelWrites?: boolean + probes?: readonly AgentMemorySequenceProbe[] + parallelProbes?: boolean + metadata?: Record +} + +export interface AgentMemorySequence { + id: string + family: KnowledgeBenchmarkFamily | string + split?: KnowledgeBenchmarkSplit + steps: readonly AgentMemorySequenceStep[] + /** Exact scopes a runtime callback may write beyond scopes declared by steps. */ + cleanupScopes?: readonly AgentMemoryScope[] + tags?: readonly string[] + metadata?: Record +} + +export interface BuildAgentMemorySequencesFromBenchmarkCasesOptions { + memoryAgentId?: string + eventScope?: (input: { + event: KnowledgeMemoryEvent + case: KnowledgeMemoryBenchmarkCase + eventIndex: number + }) => AgentMemoryScope + probeScope?: (testCase: KnowledgeMemoryBenchmarkCase) => AgentMemoryScope +} + +export interface AgentMemoryExperimentCandidate { + id: string + label?: string + /** Change when provider configuration changes so cached cells cannot be reused. */ + ref: string + /** Local construction is free; call markExternalCall before billable provisioning or reconnects. */ + createAdapter(input: { + branchId: string + sequence: AgentMemorySequence + rep: number + seed: number + purpose: 'execute' | 'recovery' + signal: AbortSignal + markExternalCall(): void + }): AgentMemoryAdapter | null | Promise + policy?: AgentMemorySharingPolicy + baseScope?: AgentMemoryScope + /** Conservative external provider charge for one complete history. */ + externalCostUsdPerSequence?: number + /** Conservative extra provider charge when recovering one interrupted history. */ + externalRecoveryCostUsdPerAttempt?: number + /** Release resources and, when cleanupBranches is false, delete the isolated state. */ + disposeAdapter?(adapter: AgentMemoryAdapter): Promise +} + +export interface AgentMemorySequenceProbeResult { + id: string + stepId: string + query: string + score: number + passed: boolean + dimensions: Record + applicableDimensions: readonly string[] + notes: string + hitIds: readonly string[] +} + +export interface AgentMemorySequenceArtifact { + candidateId: string + sequenceId: string + score: number + passed: boolean + dimensions: Record + dimensionSampleCounts: Record + probes: readonly AgentMemorySequenceProbeResult[] + branchDigest: string + journalEntries: number + durationMs: number +} + +export interface AgentMemorySequenceScenario extends Scenario { + kind: 'agent-memory-sequence' + candidateId: string + sequenceId: string + sequence: AgentMemorySequence + seedGroup: string +} + +export interface AgentMemoryExperimentRankingRow { + rank: number + candidateId: string + label: string + scoreMean: number + passRate: number + totalSequences: number + totalCells: number + totalProbes: number + cellsFailed: number + totalCostUsd: number + durationMs: number + dimensions: Record +} + +export interface RunAgentMemoryExperimentOptions { + experimentId: string + /** Stable external branch namespace; distributed workers must use the same value. */ + experimentRunId?: string + sequences: readonly AgentMemorySequence[] + candidates: readonly AgentMemoryExperimentCandidate[] + /** Retired candidates retained only so interrupted branches can be cleaned on resume. */ + recoveryCandidates?: readonly AgentMemoryExperimentCandidate[] + runDir: string + executeStep?: (input: { + memory: AgentMemoryBranch + candidateId: string + sequence: AgentMemorySequence + step: AgentMemorySequenceStep + context: DispatchContext + }) => Promise + /** Required with executeStep; identify the runtime/profile behavior in cache keys. */ + executeStepRef?: string + onBranchSnapshot?: (input: { + candidateId: string + sequenceId: string + cellId: string + snapshot: AgentMemoryBranchSnapshot + }) => Promise | void + cleanupBranches?: boolean + storage?: CampaignStorage + repo?: string + seed?: number + reps?: number + resumable?: boolean + costCeiling?: number + /** Shared across nested experiments when an outer improvement run owns spend. */ + costLedger?: CostLedgerHandle + costPhase?: string + maxConcurrency?: number + dispatchTimeoutMs?: number + /** Total deadline for each provider cleanup, close, or recovery operation. */ + cleanupTimeoutMs?: number + /** Refuse a damaged run with more unfinished attempts than this. Default 1000. */ + maxRecoveryAttempts?: number + /** Bound repeated provider cleanup after process crashes. Default 3 per attempt. */ + maxRecoveryRetriesPerAttempt?: number + now?: () => Date + /** Required with custom storage when all controllers are confined to one process. */ + controllerMode?: AgentMemoryControllerMode + /** Required for distributed controllers that share custom storage. */ + acquireRunLease?: AgentMemoryAcquireRunLease +} + +export type AgentMemoryExperimentRunLease = AgentMemoryRunLease + +export interface AgentMemoryAttemptEvent { + schema: 2 + status: 'started' | 'cleaned' + branchId: string + candidateId: string + candidateRef: string + sequenceId: string + rep: number + seed: number + cleanupBranches: boolean + externalCostUsdPerSequence: number + externalRecoveryCostUsdPerAttempt: number + recordedAt: string + recovery: boolean +} + +export interface RunAgentMemoryExperimentResult { + campaign: CampaignResult + rows: readonly AgentMemoryExperimentRankingRow[] + totalCostUsd: number + /** Recovery spend for retired candidates, excluded from ranking rows but included in totalCostUsd. */ + unrankedRecoveryCostUsd: number + leaderCandidateId?: string + rankingJsonPath: string + rankingMarkdownPath: string + attemptLogPath: string + recoveryLogPath: string +} + +export type OwnedMemoryExperimentRunLease = OwnedAgentMemoryRunLease diff --git a/src/memory/experiment/validation.ts b/src/memory/experiment/validation.ts new file mode 100644 index 0000000..6da7924 --- /dev/null +++ b/src/memory/experiment/validation.ts @@ -0,0 +1,141 @@ +import type { KnowledgeMemoryFactMatcher } from '../../benchmarks/index' +import type { AgentMemoryScope } from '../types' +import type { AgentMemorySequence } from './types' + +export function assertMemorySequences(sequences: readonly AgentMemorySequence[]): void { + for (const sequence of sequences) { + assertNonEmptyString(sequence.family, `memory experiment sequence ${sequence.id} family`) + if (sequence.steps.length === 0) { + throw new Error(`memory experiment sequence ${sequence.id} has no steps`) + } + assertUnique( + sequence.steps.map((step) => step.id), + `step in sequence ${sequence.id}`, + ) + let probeCount = 0 + for (const step of sequence.steps) { + for (const write of step.writes ?? []) { + assertNonEmptyString(write.text, `memory experiment write in ${sequence.id}/${step.id}`) + if (write.id !== undefined) { + assertNonEmptyString(write.id, `memory experiment write id in ${sequence.id}/${step.id}`) + } + } + assertUnique( + (step.probes ?? []).map((probe) => probe.id), + `probe in sequence ${sequence.id} step ${step.id}`, + ) + for (const probe of step.probes ?? []) { + probeCount += 1 + assertNonEmptyString( + probe.query, + `memory experiment probe query ${sequence.id}/${step.id}/${probe.id}`, + ) + if (probe.limit !== undefined && (!Number.isSafeInteger(probe.limit) || probe.limit <= 0)) { + throw new Error( + `memory experiment probe limit ${sequence.id}/${step.id}/${probe.id} must be a positive safe integer`, + ) + } + assertMemoryFactMatchers( + probe.requiredFacts ?? [], + `${sequence.id}/${step.id}/${probe.id} requiredFacts`, + ) + assertMemoryFactMatchers( + probe.forbiddenFacts ?? [], + `${sequence.id}/${step.id}/${probe.id} forbiddenFacts`, + ) + assertStringList(probe.expectedEventIds, `${sequence.id}/${step.id}/${probe.id} event ids`) + assertStringList(probe.expectedActorIds, `${sequence.id}/${step.id}/${probe.id} actor ids`) + if (probe.referenceAnswer !== undefined) { + assertNonEmptyString( + probe.referenceAnswer, + `memory experiment reference answer ${sequence.id}/${step.id}/${probe.id}`, + ) + } + const hasTarget = + (probe.requiredFacts?.length ?? 0) > 0 || + (probe.forbiddenFacts?.length ?? 0) > 0 || + (probe.expectedEventIds?.length ?? 0) > 0 || + (probe.expectedActorIds?.length ?? 0) > 0 || + Boolean(probe.referenceAnswer?.trim()) + if (!hasTarget) { + throw new Error( + `memory experiment probe ${sequence.id}/${step.id}/${probe.id} has no measurable target`, + ) + } + } + } + if (probeCount === 0) { + throw new Error(`memory experiment sequence ${sequence.id} has no probes`) + } + } +} + +export function normalizeCleanupScope(scope: AgentMemoryScope): AgentMemoryScope { + const normalized = compactScope(scope) + if (normalized.tags && Object.keys(normalized.tags).length === 0) { + delete normalized.tags + } + return normalized +} + +export function compactScope(scope: AgentMemoryScope): AgentMemoryScope { + return Object.fromEntries( + Object.entries(scope).filter(([, value]) => value !== undefined), + ) as AgentMemoryScope +} + +export function compactRecord(record: Record): Record { + return Object.fromEntries(Object.entries(record).filter(([, value]) => value !== undefined)) +} + +export function uniqueStrings(values: readonly unknown[]): string[] { + return [...new Set(values.filter((value): value is string => typeof value === 'string'))] +} + +export function stringArray(value: unknown): string[] { + return Array.isArray(value) + ? value.filter((entry): entry is string => typeof entry === 'string') + : [] +} + +export function assertUnique(values: readonly string[], label: string): void { + const seen = new Set() + for (const value of values) { + assertNonEmptyString(value, `memory experiment ${label} id`) + if (seen.has(value)) throw new Error(`duplicate memory experiment ${label} id: ${value}`) + seen.add(value) + } +} + +function assertMemoryFactMatchers( + matchers: readonly KnowledgeMemoryFactMatcher[], + label: string, +): void { + assertUnique( + matchers.map((matcher) => matcher.id), + `${label} matcher`, + ) + for (const matcher of matchers) { + if (matcher.anyOf.length === 0) { + throw new Error(`memory experiment ${label} matcher ${matcher.id} requires anyOf`) + } + assertStringList(matcher.anyOf, `${label} matcher ${matcher.id} anyOf`) + assertStringList(matcher.sourceEventIds, `${label} matcher ${matcher.id} source event ids`) + if (matcher.weight !== undefined && (!Number.isFinite(matcher.weight) || matcher.weight <= 0)) { + throw new Error( + `memory experiment ${label} matcher ${matcher.id} weight must be a positive finite number`, + ) + } + } +} + +function assertStringList(values: readonly string[] | undefined, label: string): void { + if (values === undefined) return + assertUnique(values, label) +} + +export function assertNonEmptyString(value: unknown, label: string): asserts value is string { + if (typeof value !== 'string' || value.trim().length === 0) { + throw new Error(`${label} must be a non-empty string`) + } +} diff --git a/src/memory/holdout.ts b/src/memory/holdout.ts index f142e8b..d1a8b89 100644 --- a/src/memory/holdout.ts +++ b/src/memory/holdout.ts @@ -1,137 +1,30 @@ import { randomUUID } from 'node:crypto' -import { mulberry32 } from '@tangle-network/agent-eval' -import type { OffPolicyTrajectory } from '@tangle-network/agent-eval/rl' -import { sha256 } from '../ids' -import type { AgentMemoryHit, AgentMemoryScope } from './types' - -// Randomized retrieval holdout (epsilon-dropout) for per-item treatment-effect logging. -// Default-off: nothing in this module runs unless a consumer passes a RetrievalHoldoutConfig. -// The library never does I/O here; persistence is the consumer's job via onEvent. -// Design + estimator + sample-size analysis: research repo, -// projects/probabilistic-agent-optimization/notes/2026-07-03-DRAFT-o3-holdout-design.md (O3 / EXP-007). - -export interface RetrievalHoldoutConfig { - /** Per-session probability that one eligible watchlist item is suppressed. 0 logs the full schema without ever dropping. */ - epsilon: number - /** Item ids eligible for suppression. Empty or absent means no item can ever be dropped. */ - watchlist?: string[] - /** Ties every event to the exact epsilon/watchlist in force, for audit and replay. */ - configVersion?: string - /** Copied onto every event so multi-adapter logs stay attributable. */ - adapterId?: string - /** Corpus/store version stamp; an edited item under the same id is a different treatment. */ - corpusVersion?: string - /** - * Emit plaintext sessionId and scope on events. Default false: events carry only - * sessionIdHash/scopeHash, so PII-bearing identifiers (tenantId/userId/tags) never reach a - * consumer-controlled sink unless the consumer explicitly owns that decision. Note that - * replaying assignment draws from logs alone needs the plaintext sessionId, so - * privacy-default logs require the consumer's own sessionId mapping for replay audits. - */ - includePlaintextIdentifiers?: boolean - /** - * Cap on tracked sessions per experiment config in the sticky wrapper's registry. - * Exists so tests can exercise eviction; production should keep the default (10,000). - */ - maxTrackedSessions?: number - /** - * Uniform-[0,1) generator keyed by a string. Defaults to a sha256-derived deterministic - * generator so every assignment is replayable from the logged keys alone (design rule D5). - */ - rng?: (key: string) => number - /** Receives one event per retrieval call, INCLUDING no-drop calls: control-arm membership is half the data. */ - onEvent: (event: RetrievalHoldoutEvent) => void -} - -export interface RetrievalHoldoutEligibleItem { - id: string - /** 1-based position in the post-filter hit list. */ - rank: number - score?: number - kind: string - /** sha256(hit.text) prefix; effects are estimated per (id, contentHash) pair. */ - contentHash: string -} -export interface RetrievalHoldoutEvent { - v: 1 - eventId: string - ts: string - adapterId?: string - /** Plaintext session id — emitted ONLY when config.includePlaintextIdentifiers is true. */ - sessionId?: string - /** Consumer-supplied experiment/outcome join id (scope.tags.taskId); deliberately plaintext. */ - taskId?: string - /** 1-based call counter within the session; 0 when the call is outside session randomization. */ - callIndex: number - /** - * sha256(sessionId) prefix — the default privacy-preserving session join key AND the seed-key - * reference for the assignment draws (previously named rngKey; identical derivation, deduped). - */ - sessionIdHash?: string - queryHash?: string - /** Verbatim scope — emitted ONLY when config.includePlaintextIdentifiers is true (PII risk). */ - scope?: AgentMemoryScope - /** sha256 prefix of the canonical-JSON scope (keys sorted, undefined stripped). */ - scopeHash?: string - config: { epsilon: number; watchlist: string[]; configVersion?: string } - /** - * Value-hash of the experiment-defining knobs, sha256({epsilon, sorted watchlist}) prefix. - * The estimator groups events by it; the sticky-session registry is keyed by it. - */ - configHash: string - /** - * False when no sessionId is available or the adapter answered without retrieval - * (see bypassReason), so the fraction-under-experiment denominator stays honest. - */ - holdoutEligible: boolean - /** Present only on adapter paths that bypassed retrieval, where no suppression could apply. */ - bypassReason?: RetrievalHoldoutBypassReason - /** The full post-filter eligibility set E, logged on every call (control arm + interference probes). */ - eligible: RetrievalHoldoutEligibleItem[] - /** Ids in watchlist ∩ E, in eligibility order. */ - watchlistEligible: string[] - sessionHoldout: boolean - /** The session's sticky drop target once drawn; distinguishes "target absent from E" from "not yet drawn". */ - sessionTargetId: string | null - /** The item suppressed in THIS call, or null. */ - droppedId: string | null - /** 1/|watchlist ∩ E| recorded at draw time; the exact inverse-propensity weight input. */ - pickPropensity: number | null - /** epsilon * pickPropensity, recorded at draw time so analysis never re-derives assignment probabilities. */ - dropPropensity: number | null - deliveredIds: string[] - corpusVersion?: string -} - -export interface RetrievalHoldoutSessionState { - sessionId: string - /** Calls observed so far in this session. */ - callCount: number - sessionHoldout: boolean - /** Sticky drop target; drawn once at the first call whose eligibility set intersects the watchlist. */ - targetId: string | null - pickPropensity: number | null -} +import { mulberry32 } from '@tangle-network/agent-eval' -export interface RetrievalHoldoutCallContext { - sessionId?: string - taskId?: string - /** Raw query; only its sha256 prefix is logged. */ - query?: string - scope?: AgentMemoryScope - /** State returned by the previous call of this session; threading it is what makes suppression sticky. */ - session?: RetrievalHoldoutSessionState -} +import type { OffPolicyTrajectory } from '@tangle-network/agent-eval/rl' -export interface RetrievalHoldoutResult { - delivered: AgentMemoryHit[] - event: RetrievalHoldoutEvent - session?: RetrievalHoldoutSessionState -} +import { sha256 } from '../ids' -/** Adapter context paths that answer without retrieval, so no holdout draw can happen. */ -export type RetrievalHoldoutBypassReason = 'short-term-context' | 'raw-string-context' +import type { + AgentMemoryHit, + RetrievalHoldoutBypassReason, + RetrievalHoldoutCallContext, + RetrievalHoldoutConfig, + RetrievalHoldoutEvent, + RetrievalHoldoutResult, + RetrievalHoldoutSessionState, +} from './types' + +export type { + RetrievalHoldoutBypassReason, + RetrievalHoldoutCallContext, + RetrievalHoldoutConfig, + RetrievalHoldoutEligibleItem, + RetrievalHoldoutEvent, + RetrievalHoldoutResult, + RetrievalHoldoutSessionState, +} from './types' /** * Deterministic uniform [0,1) for assignment draws. The sha256 key derivation is ours — it makes @@ -353,6 +246,7 @@ export function emitRetrievalHoldoutBypass( // shows up in the log as a mixed-exposure session (same sessionIdHash, different sessionTargetId, // callIndex restarting at 1), which analysis excludes and counts. const DEFAULT_MAX_TRACKED_SESSIONS = 10_000 + // Keyed by configHash VALUE, not config object identity: callers building options inline pass a // fresh config object per call (the natural adapter pattern), and identity-keying would silently // reset callIndex and re-draw the "sticky" target mid-session — one session logged as diff --git a/src/memory/improvement.ts b/src/memory/improvement.ts index 1e6029c..770648a 100644 --- a/src/memory/improvement.ts +++ b/src/memory/improvement.ts @@ -1,1249 +1,12 @@ -import { join } from 'node:path' -import { canonicalJson } from '@tangle-network/agent-eval' -import { - type CampaignStorage, - type CostLedgerHandle, - campaignLineageStore, - createRunCostLedger, - fsCampaignStorage, - type Governor, - type GovernorContext, - type GovernorOp, - type HeldoutSignificance, - type HeldoutSignificanceOptions, - heldoutSignificance, - type Lineage, - type LineageStore, - type MutableSurface, - memLineageStore, - type PairedHoldout, - resolveRunDir, - runLineageLoop, - type Scenario, - type SurfaceProposer, - surfaceHash, -} from '@tangle-network/agent-eval/campaign' -import { - appendDurableJournalEvent, - assertNoInterruptedPaidCalls, - reconcileInterruptedRunPaidCalls, -} from './attempt-log' -import type { - AgentMemoryExperimentCandidate, - AgentMemorySequence, - AgentMemorySequenceArtifact, - AgentMemorySequenceProbe, - RunAgentMemoryExperimentOptions, - RunAgentMemoryExperimentResult, -} from './experiment' -import { runAgentMemoryExperiment } from './experiment' -import { runBoundedMemoryLifecycle } from './lifecycle' -import { - type AgentMemoryAcquireRunLease, - type AgentMemoryControllerMode, - type AgentMemoryRunLease, - acquireAgentMemoryRunLease, - type OwnedAgentMemoryRunLease, -} from './run-control' - -export interface AgentMemoryImprovementSeed { - config: TConfig - track: string - vision?: string - proposer: string -} - -export interface AgentMemoryDimensionComparison { - dimension: string - n: number - expectedN: number - measured: boolean - meanDelta: number - low: number - high: number - tolerance: number - regressed: boolean -} - -export interface AgentMemoryPromotionDecision { - status: 'promote' | 'hold' | 'no-change' - reasons: readonly string[] - baselineScore: number - winnerScore: number - lift: number - significance?: HeldoutSignificance - criticalDimensions: readonly AgentMemoryDimensionComparison[] -} - -export interface AgentMemoryActivation { - id: string - status: - | 'not-eligible' - | 'not-configured' - | 'pending' - | 'activated' - | 'recovered' - | 'already-activated' - journalPath: string -} - -export interface AgentMemoryActivationDriver { - /** Change whenever activation behavior or the external target changes. */ - ref: string - /** Return the exact currently active configuration. */ - readCurrent(): Promise - /** Atomically replace expectedConfig with config, or fail on a concurrent change. */ - compareAndSet(input: { - activationId: string - expectedConfig: TConfig - expectedSurfaceHash: string - config: TConfig - surfaceHash: string - decision: AgentMemoryPromotionDecision - lineage: Lineage - holdout: RunAgentMemoryExperimentResult - }): Promise -} - -interface AgentMemoryActivationEvent { - schema: 1 - status: 'prepared' | 'activated' - activationId: string - experimentId: string - activationRef: string - baselineSurfaceHash: string - winnerSurfaceHash: string - holdoutManifestHash: string - recordedAt: string - outcome?: 'applied' | 'recovered' | 'already-current' -} - -interface AgentMemoryActivationJournalState { - prepared: boolean - activated?: AgentMemoryActivationEvent -} - -export type AgentMemoryImprovementRunLease = AgentMemoryRunLease - -export interface AgentMemoryGovernor { - decide( - context: GovernorContext & { - costLedger: CostLedgerHandle - costPhase: string - }, - ): GovernorOp | Promise -} - -export interface RunAgentMemoryImprovementOptions { - experimentId: string - trainSequences: readonly AgentMemorySequence[] - holdoutSequences: readonly AgentMemorySequence[] - /** First entry is the current baseline; remaining entries seed independent search tracks. */ - seeds: readonly AgentMemoryImprovementSeed[] - createCandidate(input: { - config: TConfig - candidateId: string - surfaceHash: string - }): - | Omit - | Promise> - proposer: SurfaceProposer - /** Optional proposer implementations keyed by seed and branch proposer labels. */ - proposers?: Readonly> - /** Stable version or commit for the candidate factory, proposer, and governor. */ - improvementRef: string - governor?: AgentMemoryGovernor - budget: { maxSteps: number } - populationSize?: number - candidateConcurrency?: number - sequenceConcurrency?: number - runDir: string - repo?: string - storage?: CampaignStorage - lineageStore?: LineageStore - /** Required with custom storage when all controllers are confined to one process. */ - controllerMode?: AgentMemoryControllerMode - /** Required for distributed controllers using custom storage. Worker concurrency is independent. */ - acquireRunLease?: AgentMemoryAcquireRunLease - seed?: number - reps?: number - resumable?: boolean - dispatchTimeoutMs?: number - cleanupTimeoutMs?: number - maxRecoveryAttempts?: number - maxRecoveryRetriesPerAttempt?: number - maxTotalCostUsd?: number - executeStep?: RunAgentMemoryExperimentOptions['executeStep'] - executeStepRef?: string - onBranchSnapshot?: RunAgentMemoryExperimentOptions['onBranchSnapshot'] - cleanupBranches?: boolean - serializeConfig?: (config: TConfig) => string - parseConfig?: (surface: string) => TConfig - significance?: HeldoutSignificanceOptions - criticalDimensions?: readonly string[] - criticalDimensionTolerance?: number - minHoldoutScore?: number - activation?: AgentMemoryActivationDriver - activationTimeoutMs?: number - now?: () => Date -} - -export interface RunAgentMemoryImprovementResult { - lineage: Lineage - baselineConfig: TConfig - winnerConfig: TConfig - baselineSurface: string - winnerSurface: string - baselineSurfaceHash: string - winnerSurfaceHash: string - decision: AgentMemoryPromotionDecision - activation: AgentMemoryActivation - holdout?: RunAgentMemoryExperimentResult - totalCostUsd: number - resultJsonPath: string -} - -interface MemoryConfigScenario extends Scenario { - kind: 'agent-memory-config-search' - sequenceId: string -} - -const DEFAULT_CRITICAL_DIMENSIONS = [ - 'memory_stale_safe', - 'memory_actor_recall', - 'memory_event_recall', -] as const -const MEMORY_IMPROVEMENT_IMPLEMENTATION_REF = 'agent-knowledge:memory-improvement:v2' - -/** Searches branchable memory configurations and activates only a fresh holdout win. */ -export async function runAgentMemoryImprovement( - options: RunAgentMemoryImprovementOptions, -): Promise> { - if ('onPromote' in options) { - throw new Error( - 'memory improvement onPromote was removed; use activation.readCurrent and activation.compareAndSet', - ) - } - if (options.seeds.length === 0) throw new Error('memory improvement requires seed configs') - if (options.trainSequences.length === 0) { - throw new Error('memory improvement requires training sequences') - } - if (options.holdoutSequences.length === 0) { - throw new Error('memory improvement requires holdout sequences') - } - const trainIds = new Set(options.trainSequences.map((sequence) => sequence.id)) - const overlap = options.holdoutSequences - .map((sequence) => sequence.id) - .filter((id) => trainIds.has(id)) - if (overlap.length > 0) { - throw new Error(`memory improvement train/holdout overlap: ${overlap.join(', ')}`) - } - const trainFingerprints = new Map( - options.trainSequences.map((sequence) => [memorySequenceFingerprint(sequence), sequence.id]), - ) - const duplicateHistories = options.holdoutSequences.flatMap((sequence) => { - const trainId = trainFingerprints.get(memorySequenceFingerprint(sequence)) - return trainId ? [`${trainId}/${sequence.id}`] : [] - }) - if (duplicateHistories.length > 0) { - throw new Error( - `memory improvement train/holdout histories duplicate content: ${duplicateHistories.join(', ')}`, - ) - } - if (typeof options.improvementRef !== 'string' || !options.improvementRef.trim()) { - throw new Error('memory improvement improvementRef must be a non-empty string') - } - assertMemoryImprovementOptions(options) - const storage = options.storage ?? fsCampaignStorage() - const runDir = resolveRunDir(options.runDir, options.repo) - storage.ensureDir(runDir) - const lease = await acquireAgentMemoryRunLease({ - experimentId: options.experimentId, - runDir, - storage, - customStorage: options.storage !== undefined, - lockFileName: 'memory-improvement.lock', - label: 'memory improvement', - controllerMode: options.controllerMode, - acquireRunLease: options.acquireRunLease, - }) - let result: RunAgentMemoryImprovementResult | undefined - let primaryError: unknown - try { - result = await runAgentMemoryImprovementOwned(options, storage, runDir, lease) - } catch (error) { - primaryError = error - } - let releaseError: unknown - try { - await lease.release() - } catch (error) { - releaseError = error - } - if (primaryError && releaseError) { - throw new AggregateError( - [primaryError, releaseError], - 'memory improvement failed and its controller lease could not be released', - ) - } - if (primaryError) throw primaryError - if (releaseError) throw releaseError - if (!result) throw new Error('memory improvement produced no result') - return result -} - -async function runAgentMemoryImprovementOwned( - options: RunAgentMemoryImprovementOptions, - storage: CampaignStorage, - runDir: string, - lease: OwnedRunLease, -): Promise> { - await lease.assertOwned() - const serializeRaw = options.serializeConfig ?? ((config: TConfig) => canonicalJson(config)) - const parseRaw = options.parseConfig ?? ((surface: string) => JSON.parse(surface) as TConfig) - const serialize = (config: TConfig): string => serializeMemoryConfig(serializeRaw, config) - const parse = (surface: string): TConfig => parseMemoryConfig(parseRaw, surface) - for (const seed of options.seeds) assertMemoryConfigRoundTrip(seed.config, serialize, parse) - if (options.activation && !storage.append) { - throw new Error('memory activation requires CampaignStorage.append') - } - if (options.resumable !== false && !options.lineageStore && !storage.append) { - throw new Error('resumable memory improvement requires CampaignStorage.append') - } - assertMemoryImprovementIdentity(options, storage, runDir, serialize) - const costLedger = createRunCostLedger({ - storage, - runDir, - costCeilingUsd: options.maxTotalCostUsd ?? 0, - }) - reconcileInterruptedRunPaidCalls(costLedger, 'memory improvement run') - assertNoInterruptedPaidCalls(costLedger, 'memory improvement recovery') - const trainScenarios: MemoryConfigScenario[] = options.trainSequences.map((sequence) => ({ - id: sequence.id, - kind: 'agent-memory-config-search', - sequenceId: sequence.id, - })) - const evaluations = new Map>() - - const evaluateSurface = async ( - surface: MutableSurface, - ): Promise<{ score: number; scoreVector: number[] }> => { - await lease.assertOwned() - const text = requireStringSurface(surface) - const config = parse(text) - const canonicalSurface = serialize(config) - const hash = surfaceHash(canonicalSurface) - let pending = evaluations.get(hash) - if (!pending) { - pending = (async () => { - const candidate = await buildCandidate(options, config, hash, `search-${hash}`) - await lease.assertOwned() - const experiment = await runAgentMemoryExperiment({ - ...experimentOptions(options, costLedger, storage, lease), - experimentId: `${options.experimentId}:search:${hash}`, - sequences: options.trainSequences, - candidates: [candidate], - runDir: join(runDir, 'search', hash), - costPhase: `memory.search.${hash}`, - }) - await lease.assertOwned() - return experiment - })() - evaluations.set(hash, pending) - void pending.catch(() => evaluations.delete(hash)) - } - const result = await pending - await lease.assertOwned() - const row = result.rows[0] - if (!row || row.cellsFailed > 0) { - throw new Error(`${hash}: memory candidate did not complete every training cell`) - } - return { - score: row.scoreMean, - scoreVector: sequenceScores(result, options.trainSequences, row.candidateId), - } - } - - const lineageStore = - options.lineageStore ?? - (options.resumable === false - ? memLineageStore() - : campaignLineageStore(storage, join(runDir, 'lineage.jsonl'))) - const lineageOptions = { - seeds: options.seeds.map((seed) => ({ - surface: serialize(seed.config), - track: seed.track, - proposer: seed.proposer, - ...(seed.vision !== undefined ? { vision: seed.vision } : {}), - })), - scenarios: trainScenarios, - proposer: withCostContext(options.proposer, costLedger, lease, 'default'), - proposers: options.proposers - ? Object.fromEntries( - Object.entries(options.proposers).map(([name, proposer]) => [ - name, - withCostContext(proposer, costLedger, lease, name), - ]), - ) - : undefined, - scoreSurface: evaluateSurface, - governor: options.governor - ? withGovernorCostContext(options.governor, costLedger, lease) - : undefined, - budget: { - ...options.budget, - maxNodes: options.seeds.length + options.budget.maxSteps, - }, - store: lineageStore, - populationSize: options.populationSize, - candidateConcurrency: options.candidateConcurrency, - } - const search = await runLineageLoop(lineageOptions) - await lease.assertOwned() - const best = search.best - if (!best) throw new Error('memory improvement produced no measured config') - - const baselineSurface = serialize(options.seeds[0]!.config) - const baselineHash = surfaceHash(baselineSurface) - const winnerConfig = parse(requireStringSurface(best.surface)) - const winnerSurface = serialize(winnerConfig) - const winnerHash = surfaceHash(winnerSurface) - const baselineConfig = parse(baselineSurface) - - let holdout: RunAgentMemoryExperimentResult | undefined - let decision: AgentMemoryPromotionDecision - if (winnerHash === baselineHash) { - const baselineMeasurement = await evaluateSurface(baselineSurface) - decision = { - status: 'no-change', - reasons: ['search did not find a config better than the baseline'], - baselineScore: baselineMeasurement.score, - winnerScore: baselineMeasurement.score, - lift: 0, - criticalDimensions: [], - } - } else { - await lease.assertOwned() - const [baselineCandidate, winnerCandidate] = await Promise.all([ - buildCandidate(options, baselineConfig, baselineHash, 'baseline'), - buildCandidate(options, winnerConfig, winnerHash, 'winner'), - ]) - await lease.assertOwned() - holdout = await runAgentMemoryExperiment({ - ...experimentOptions(options, costLedger, storage, lease), - experimentId: `${options.experimentId}:holdout`, - sequences: options.holdoutSequences, - candidates: [baselineCandidate, winnerCandidate], - runDir: join(runDir, 'holdout'), - costPhase: 'memory.holdout', - }) - decision = decidePromotion({ - options, - result: holdout, - baselineId: baselineCandidate.id, - winnerId: winnerCandidate.id, - }) - } - - const resultJsonPath = join(runDir, 'memory-improvement-result.json') - const activationRef = options.activation?.ref ?? 'not-configured' - const activationId = `memory-activation-${surfaceHash( - canonicalJson({ - experimentId: options.experimentId, - improvementRef: options.improvementRef, - activationRef, - baselineSurfaceHash: baselineHash, - winnerSurfaceHash: winnerHash, - holdoutManifestHash: holdout?.campaign.manifestHash ?? null, - promotionPolicy: normalizedPromotionPolicy(options), - }), - )}` - const activationJournalDir = join(runDir, 'activations') - const activationJournalPath = join(activationJournalDir, `${activationId}.jsonl`) - const activationEligible = decision.status === 'promote' && holdout !== undefined - const activationEventIdentity = holdout - ? { - schema: 1 as const, - activationId, - experimentId: options.experimentId, - activationRef, - baselineSurfaceHash: baselineHash, - winnerSurfaceHash: winnerHash, - holdoutManifestHash: holdout.campaign.manifestHash, - } - : undefined - const activationJournal = - activationEligible && activationEventIdentity - ? readMemoryActivationJournal(storage, activationJournalPath, activationEventIdentity) - : { prepared: false } - const activation: AgentMemoryActivation = { - id: activationId, - status: !activationEligible - ? 'not-eligible' - : activationJournal.activated - ? 'already-activated' - : options.activation - ? 'pending' - : 'not-configured', - journalPath: activationJournalPath, - } - const result = { - lineage: search.lineage, - baselineConfig, - winnerConfig, - baselineSurface, - winnerSurface, - baselineSurfaceHash: baselineHash, - winnerSurfaceHash: winnerHash, - decision, - activation, - ...(holdout ? { holdout } : {}), - totalCostUsd: costLedger.summary().totalCostUsd, - resultJsonPath, - } satisfies RunAgentMemoryImprovementResult - await lease.assertOwned() - writeMemoryImprovementResult(storage, options.experimentId, result) - if ( - activationEligible && - !activationJournal.activated && - activationEventIdentity && - holdout && - options.activation - ) { - const activationDriver = options.activation - const activationTimeoutMs = options.activationTimeoutMs ?? 60_000 - const hadPreparedEvent = activationJournal.prepared - if (!hadPreparedEvent) { - await lease.assertOwned() - storage.ensureDir(activationJournalDir) - appendMemoryActivationEvent(storage, activationJournalPath, { - ...activationEventIdentity, - status: 'prepared', - recordedAt: (options.now ?? (() => new Date()))().toISOString(), - }) - } - - await lease.assertOwned() - const currentConfig = await runBoundedMemoryLifecycle({ - operation: `${activationDriver.ref}: read current memory configuration`, - timeoutMs: activationTimeoutMs, - run: () => activationDriver.readCurrent(), - }) - await lease.assertOwned() - const currentHash = surfaceHash(serialize(currentConfig)) - if (currentHash !== baselineHash && currentHash !== winnerHash) { - throw new Error( - `memory activation target '${activationDriver.ref}' changed concurrently; expected '${baselineHash}' or '${winnerHash}', found '${currentHash}'`, - ) - } - - let outcome: NonNullable - if (currentHash === winnerHash) { - outcome = hadPreparedEvent ? 'recovered' : 'already-current' - activation.status = 'recovered' - } else { - let compareError: unknown - try { - await runBoundedMemoryLifecycle({ - operation: `${activationDriver.ref}: activate memory configuration`, - timeoutMs: activationTimeoutMs, - run: () => - activationDriver.compareAndSet({ - activationId, - expectedConfig: baselineConfig, - expectedSurfaceHash: baselineHash, - config: winnerConfig, - surfaceHash: winnerHash, - decision, - lineage: search.lineage, - holdout, - }), - }) - } catch (error) { - compareError = error - } - await lease.assertOwned() - - let observedConfig: TConfig - try { - observedConfig = await runBoundedMemoryLifecycle({ - operation: `${activationDriver.ref}: confirm memory configuration`, - timeoutMs: activationTimeoutMs, - run: () => activationDriver.readCurrent(), - }) - } catch (error) { - if (compareError) { - throw new AggregateError( - [compareError, error], - `memory activation '${activationId}' failed and its live state could not be confirmed`, - ) - } - throw error - } - await lease.assertOwned() - const observedHash = surfaceHash(serialize(observedConfig)) - if (observedHash !== winnerHash) { - const mismatch = new Error( - `memory activation '${activationId}' did not install the measured winner; found '${observedHash}'`, - ) - if (compareError) { - throw new AggregateError( - [compareError, mismatch], - `memory activation '${activationId}' failed without applying the measured winner`, - ) - } - throw mismatch - } - outcome = compareError ? 'recovered' : 'applied' - activation.status = compareError ? 'recovered' : 'activated' - } - - await lease.assertOwned() - appendMemoryActivationEvent(storage, activationJournalPath, { - ...activationEventIdentity, - status: 'activated', - outcome, - recordedAt: (options.now ?? (() => new Date()))().toISOString(), - }) - writeMemoryImprovementResult(storage, options.experimentId, result) - } - return result -} - -function withCostContext( - proposer: SurfaceProposer, - costLedger: CostLedgerHandle, - lease: OwnedRunLease, - label: string, -): SurfaceProposer { - return { - kind: proposer.kind, - async propose(context) { - await lease.assertOwned() - const proposal = await proposer.propose({ - ...context, - costLedger, - costPhase: `memory.proposal.${context.track?.id ?? label}`, - }) - await lease.assertOwned() - return proposal - }, - ...(proposer.decide ? { decide: (input) => proposer.decide!(input) } : {}), - } -} - -function withGovernorCostContext( - governor: AgentMemoryGovernor, - costLedger: CostLedgerHandle, - lease: OwnedRunLease, -): Governor { - return { - async decide(context) { - await lease.assertOwned() - const decision = await governor.decide({ - ...context, - costLedger, - costPhase: 'memory.governor', - }) - await lease.assertOwned() - return decision - }, - } -} - -async function buildCandidate( - options: RunAgentMemoryImprovementOptions, - config: TConfig, - hash: string, - role: string, -): Promise { - const built = await options.createCandidate({ - config, - candidateId: `${role}-${hash}`, - surfaceHash: hash, - }) - return { - ...built, - id: `${role}-${hash}`, - ref: built.ref, - } -} - -function experimentOptions( - options: RunAgentMemoryImprovementOptions, - costLedger: ReturnType, - storage: CampaignStorage, - lease: OwnedRunLease, -): Pick< - RunAgentMemoryExperimentOptions, - | 'storage' - | 'seed' - | 'reps' - | 'resumable' - | 'maxConcurrency' - | 'dispatchTimeoutMs' - | 'cleanupTimeoutMs' - | 'maxRecoveryAttempts' - | 'maxRecoveryRetriesPerAttempt' - | 'executeStep' - | 'executeStepRef' - | 'onBranchSnapshot' - | 'cleanupBranches' - | 'costLedger' - | 'acquireRunLease' - | 'now' -> { - return { - storage, - seed: options.seed, - reps: options.reps, - resumable: options.resumable, - maxConcurrency: options.sequenceConcurrency, - dispatchTimeoutMs: options.dispatchTimeoutMs, - cleanupTimeoutMs: options.cleanupTimeoutMs, - maxRecoveryAttempts: options.maxRecoveryAttempts, - maxRecoveryRetriesPerAttempt: options.maxRecoveryRetriesPerAttempt, - executeStep: options.executeStep, - executeStepRef: options.executeStepRef, - onBranchSnapshot: options.onBranchSnapshot, - cleanupBranches: options.cleanupBranches ?? true, - costLedger, - acquireRunLease: async () => ({ - assertOwned: () => lease.assertOwned(), - release() {}, - }), - now: options.now, - } -} - -type OwnedRunLease = OwnedAgentMemoryRunLease - -function assertMemoryImprovementIdentity( - options: RunAgentMemoryImprovementOptions, - storage: CampaignStorage, - runDir: string, - serialize: (config: TConfig) => string, -): void { - const path = join(runDir, 'memory-improvement-manifest.json') - const identity = { - schema: 6, - implementationRef: MEMORY_IMPROVEMENT_IMPLEMENTATION_REF, - experimentId: options.experimentId, - improvementRef: options.improvementRef, - activationRef: options.activation?.ref ?? null, - proposerKind: options.proposer.kind, - proposerKinds: Object.fromEntries( - Object.entries(options.proposers ?? {}) - .sort(([a], [b]) => a.localeCompare(b)) - .map(([name, proposer]) => [name, proposer.kind]), - ), - populationSize: options.populationSize ?? 4, - budget: { maxSteps: options.budget.maxSteps }, - executeStepRef: options.executeStepRef ?? null, - cleanupBranches: options.cleanupBranches ?? true, - promotionPolicy: normalizedPromotionPolicy(options), - seed: options.seed ?? null, - reps: options.reps ?? 1, - seeds: options.seeds.map((entry) => ({ - config: serialize(entry.config), - track: entry.track, - vision: entry.vision ?? null, - proposer: entry.proposer, - })), - trainSequences: options.trainSequences, - holdoutSequences: options.holdoutSequences, - } - const identityHash = surfaceHash(canonicalJson(identity)) - const stored = storage.read(path) - if (stored === undefined) { - if (storage.exists(path)) throw new Error(`cannot read memory improvement manifest '${path}'`) - if ( - storage.exists(join(runDir, 'lineage.jsonl')) || - storage.exists(join(runDir, 'memory-improvement-result.json')) - ) { - throw new Error(`memory improvement run '${runDir}' has state without an identity manifest`) - } - storage.write(path, `${JSON.stringify({ identityHash, identity }, null, 2)}\n`) - return - } - let manifest: unknown - try { - manifest = JSON.parse(stored) - } catch (error) { - throw new Error(`invalid memory improvement manifest '${path}'`, { cause: error }) - } - if ( - !manifest || - typeof manifest !== 'object' || - (manifest as Record).identityHash !== identityHash || - canonicalJson((manifest as Record).identity) !== canonicalJson(identity) - ) { - throw new Error( - `memory improvement run '${runDir}' does not match its persisted inputs or implementationRef`, - ) - } -} - -function appendMemoryActivationEvent( - storage: CampaignStorage, - path: string, - event: AgentMemoryActivationEvent, -): void { - appendDurableJournalEvent({ - storage, - path, - event, - label: 'memory activation journal', - }) -} - -function readMemoryActivationJournal( - storage: CampaignStorage, - path: string, - expected: Omit, -): AgentMemoryActivationJournalState { - const stored = storage.read(path) - if (stored === undefined) { - if (storage.exists(path)) throw new Error(`cannot read memory activation journal '${path}'`) - return { prepared: false } - } - let prepared = false - let activated: AgentMemoryActivationEvent | undefined - for (const [index, line] of stored.split('\n').entries()) { - if (!line) continue - let raw: unknown - try { - raw = JSON.parse(line) - } catch (error) { - throw new Error(`invalid memory activation journal '${path}' line ${index + 1}`, { - cause: error, - }) - } - const event = parseMemoryActivationEvent(raw, path, index + 1, expected) - if (event.status === 'prepared') { - if (prepared || activated) { - throw new Error(`memory activation journal '${path}' repeats its prepared event`) - } - prepared = true - continue - } - if (!prepared || activated) { - throw new Error(`memory activation journal '${path}' has an out-of-order activated event`) - } - activated = event - } - return { prepared, ...(activated ? { activated } : {}) } -} - -function parseMemoryActivationEvent( - value: unknown, - path: string, - line: number, - expected: Omit, -): AgentMemoryActivationEvent { - if (!value || typeof value !== 'object' || Array.isArray(value)) { - throw new Error(`invalid memory activation journal '${path}' line ${line}`) - } - const event = value as Record - for (const [key, expectedValue] of Object.entries(expected)) { - if (event[key] !== expectedValue) { - throw new Error( - `memory activation journal '${path}' line ${line} does not match the measured winner`, - ) - } - } - if (event.status !== 'prepared' && event.status !== 'activated') { - throw new Error(`invalid memory activation journal '${path}' line ${line} status`) - } - if (typeof event.recordedAt !== 'string' || !Number.isFinite(Date.parse(event.recordedAt))) { - throw new Error(`invalid memory activation journal '${path}' line ${line} recordedAt`) - } - if ( - event.status === 'activated' && - event.outcome !== 'applied' && - event.outcome !== 'recovered' && - event.outcome !== 'already-current' - ) { - throw new Error(`invalid memory activation journal '${path}' line ${line} outcome`) - } - if (event.status === 'prepared' && event.outcome !== undefined) { - throw new Error(`invalid memory activation journal '${path}' line ${line} prepared outcome`) - } - return value as AgentMemoryActivationEvent -} - -function writeMemoryImprovementResult( - storage: CampaignStorage, - experimentId: string, - result: RunAgentMemoryImprovementResult, -): void { - storage.write( - result.resultJsonPath, - `${JSON.stringify( - { - experimentId, - baselineSurfaceHash: result.baselineSurfaceHash, - winnerSurfaceHash: result.winnerSurfaceHash, - baselineSurface: result.baselineSurface, - winnerSurface: result.winnerSurface, - decision: result.decision, - activation: result.activation, - totalCostUsd: result.totalCostUsd, - lineage: result.lineage.toGraph(), - }, - null, - 2, - )}\n`, - ) -} - -function decidePromotion(input: { - options: RunAgentMemoryImprovementOptions - result: RunAgentMemoryExperimentResult - baselineId: string - winnerId: string -}): AgentMemoryPromotionDecision { - const { options, result, baselineId, winnerId } = input - const baselineRow = result.rows.find((row) => row.candidateId === baselineId) - const winnerRow = result.rows.find((row) => row.candidateId === winnerId) - if (!baselineRow || !winnerRow) throw new Error('holdout result is missing a comparison arm') - const paired = pairedArtifacts(result, baselineId, winnerId, (artifact) => artifact.score) - const significance = heldoutSignificance(paired, options.significance) - const tolerance = options.criticalDimensionTolerance ?? 0.05 - const criticalDimensions = (options.criticalDimensions ?? DEFAULT_CRITICAL_DIMENSIONS).map( - (dimension) => { - const expectedN = - applicableSequenceCount(options.holdoutSequences, dimension) * (options.reps ?? 1) - const dimensionPairs = pairedArtifacts(result, baselineId, winnerId, (artifact) => - (artifact.dimensionSampleCounts?.[dimension] ?? 0) > 0 - ? artifact.dimensions[dimension] - : undefined, - ) - const comparison = heldoutSignificance(dimensionPairs, { - ...options.significance, - deltaThreshold: 0, - }) - return { - dimension, - n: comparison.n, - expectedN, - measured: expectedN > 0 && comparison.n === expectedN, - meanDelta: comparison.bootstrap.mean, - low: comparison.bootstrap.low, - high: comparison.bootstrap.high, - tolerance, - regressed: - expectedN > 0 && comparison.n === expectedN && comparison.bootstrap.low < -tolerance, - } - }, - ) - const reasons: string[] = [] - if (baselineRow.cellsFailed > 0 || winnerRow.cellsFailed > 0) { - reasons.push('at least one holdout cell failed') - } - if (!significance.significant) { - reasons.push( - significance.fewRuns - ? `only ${significance.n} paired holdout cells; more are required` - : 'holdout lift is not confidently above the promotion threshold', - ) - } - if (winnerRow.scoreMean < (options.minHoldoutScore ?? 0)) { - reasons.push(`winner holdout score ${winnerRow.scoreMean} is below the required minimum`) - } - for (const dimension of criticalDimensions) { - if (!dimension.measured) { - reasons.push( - dimension.expectedN === 0 - ? `critical dimension ${dimension.dimension} has no applicable holdout histories` - : `critical dimension ${dimension.dimension} was measured on ${dimension.n}/${dimension.expectedN} applicable paired holdout cells`, - ) - } else if (dimension.regressed) { - reasons.push(`${dimension.dimension} may regress beyond ${tolerance}`) - } - } - return { - status: reasons.length === 0 ? 'promote' : 'hold', - reasons, - baselineScore: baselineRow.scoreMean, - winnerScore: winnerRow.scoreMean, - lift: winnerRow.scoreMean - baselineRow.scoreMean, - significance, - criticalDimensions, - } -} - -function applicableSequenceCount( - sequences: readonly AgentMemorySequence[], - dimension: string, -): number { - return sequences.filter((sequence) => - sequence.steps.some((step) => - (step.probes ?? []).some((probe) => probeAppliesToDimension(probe, dimension)), - ), - ).length -} - -function probeAppliesToDimension(probe: AgentMemorySequenceProbe, dimension: string): boolean { - switch (dimension) { - case 'memory_fact_recall': - case 'memory_required_fact_count': - case 'memory_matched_fact_count': - return Boolean( - (probe.requiredFacts && probe.requiredFacts.length > 0) || probe.referenceAnswer, - ) - case 'memory_event_recall': - return Boolean(probe.expectedEventIds && probe.expectedEventIds.length > 0) - case 'memory_actor_recall': - return Boolean(probe.expectedActorIds && probe.expectedActorIds.length > 0) - case 'memory_stale_safe': - case 'memory_stale_rate': - case 'memory_forbidden_fact_count': - case 'memory_matched_forbidden_fact_count': - return Boolean(probe.forbiddenFacts && probe.forbiddenFacts.length > 0) - default: - return true - } -} - -function normalizedPromotionPolicy( - options: RunAgentMemoryImprovementOptions, -): Record { - return { - significance: { - deltaThreshold: options.significance?.deltaThreshold ?? 0, - minProductiveRuns: options.significance?.minProductiveRuns ?? 3, - confidence: options.significance?.confidence ?? 0.95, - resamples: options.significance?.resamples ?? 2000, - seed: options.significance?.seed ?? 1337, - statistic: options.significance?.statistic ?? 'mean', - }, - criticalDimensions: [...(options.criticalDimensions ?? DEFAULT_CRITICAL_DIMENSIONS)], - criticalDimensionTolerance: options.criticalDimensionTolerance ?? 0.05, - minHoldoutScore: options.minHoldoutScore ?? 0, - } -} - -function pairedArtifacts( - result: RunAgentMemoryExperimentResult, - baselineId: string, - winnerId: string, - select: (artifact: AgentMemorySequenceArtifact) => number | undefined, -): PairedHoldout { - const baseline = artifactValues(result, baselineId, select) - const winner = artifactValues(result, winnerId, select) - const keys = [...baseline.keys()].filter((key) => winner.has(key)).sort() - return { - before: keys.map((key) => baseline.get(key)!), - after: keys.map((key) => winner.get(key)!), - cellIds: keys, - } -} - -function artifactValues( - result: RunAgentMemoryExperimentResult, - candidateId: string, - select: (artifact: AgentMemorySequenceArtifact) => number | undefined, -): Map { - const values = new Map() - for (const cell of result.campaign.cells) { - if (cell.error || cell.artifact.candidateId !== candidateId) continue - const value = select(cell.artifact) - if (value === undefined || !Number.isFinite(value)) continue - values.set(`${cell.artifact.sequenceId}:${cell.rep}`, value) - } - return values -} - -function sequenceScores( - result: RunAgentMemoryExperimentResult, - sequences: readonly AgentMemorySequence[], - candidateId: string, -): number[] { - const bySequence = new Map() - for (const cell of result.campaign.cells) { - if (cell.error || cell.artifact.candidateId !== candidateId) continue - const bucket = bySequence.get(cell.artifact.sequenceId) ?? [] - bucket.push(cell.artifact.score) - bySequence.set(cell.artifact.sequenceId, bucket) - } - return sequences.map((sequence) => mean(bySequence.get(sequence.id) ?? [])) -} - -function requireStringSurface(surface: MutableSurface): string { - if (typeof surface !== 'string' || surface.trim().length === 0) { - throw new Error('memory config proposer must return a JSON string surface') - } - return surface -} - -function serializeMemoryConfig( - serialize: (config: TConfig) => string, - config: TConfig, -): string { - let surface: unknown - try { - surface = serialize(config) - } catch (error) { - throw new Error('memory config serializer failed', { cause: error }) - } - if (typeof surface !== 'string' || surface.trim().length === 0) { - throw new Error('memory config serializer must return a non-empty string') - } - return surface -} - -function parseMemoryConfig(parse: (surface: string) => TConfig, surface: string): TConfig { - try { - const config = parse(surface) - if (config === undefined) throw new Error('parser returned undefined') - return config - } catch (error) { - throw new Error('memory config surface could not be parsed', { cause: error }) - } -} - -function assertMemoryConfigRoundTrip( - config: TConfig, - serialize: (config: TConfig) => string, - parse: (surface: string) => TConfig, -): void { - const first = serialize(config) - const second = serialize(parse(first)) - if (first !== second) { - throw new Error('memory config serializer and parser must round-trip seed configs exactly') - } -} - -function memorySequenceFingerprint(sequence: AgentMemorySequence): string { - const { id: _id, split: _split, tags: _tags, ...content } = sequence - return surfaceHash(canonicalJson(content)) -} - -function assertMemoryImprovementOptions( - options: RunAgentMemoryImprovementOptions, -): void { - for (const [name, value] of [ - ['experimentId', options.experimentId], - ['runDir', options.runDir], - ] as const) { - if (typeof value !== 'string' || !value.trim()) { - throw new Error(`memory improvement ${name} must be a non-empty string`) - } - } - if (typeof options.proposer?.kind !== 'string' || !options.proposer.kind.trim()) { - throw new Error('memory improvement proposer.kind must be a non-empty string') - } - if (typeof options.proposer?.propose !== 'function') { - throw new Error('memory improvement proposer.propose must be a function') - } - if (options.governor !== undefined && typeof options.governor.decide !== 'function') { - throw new Error('memory improvement governor.decide must be a function') - } - if (options.activation !== undefined) { - if (typeof options.activation.ref !== 'string' || !options.activation.ref.trim()) { - throw new Error('memory improvement activation.ref must be a non-empty string') - } - if (typeof options.activation.readCurrent !== 'function') { - throw new Error('memory improvement activation.readCurrent must be a function') - } - if (typeof options.activation.compareAndSet !== 'function') { - throw new Error('memory improvement activation.compareAndSet must be a function') - } - } - for (const [name, proposer] of Object.entries(options.proposers ?? {})) { - if (!name.trim()) throw new Error('memory improvement proposer labels must be non-empty') - if ( - !proposer || - typeof proposer.kind !== 'string' || - !proposer.kind.trim() || - typeof proposer.propose !== 'function' - ) { - throw new Error(`memory improvement proposer '${name}' is invalid`) - } - } - if (options.serializeConfig !== undefined && typeof options.serializeConfig !== 'function') { - throw new Error('memory improvement serializeConfig must be a function') - } - if (options.parseConfig !== undefined && typeof options.parseConfig !== 'function') { - throw new Error('memory improvement parseConfig must be a function') - } - if (!Number.isSafeInteger(options.budget.maxSteps) || options.budget.maxSteps < 0) { - throw new Error('memory improvement budget.maxSteps must be a non-negative safe integer') - } - for (const [name, value] of [ - ['populationSize', options.populationSize], - ['candidateConcurrency', options.candidateConcurrency], - ['sequenceConcurrency', options.sequenceConcurrency], - ['reps', options.reps], - ['maxRecoveryAttempts', options.maxRecoveryAttempts], - ['maxRecoveryRetriesPerAttempt', options.maxRecoveryRetriesPerAttempt], - ] as const) { - if (value !== undefined && (!Number.isSafeInteger(value) || value <= 0)) { - throw new Error(`memory improvement ${name} must be a positive safe integer`) - } - } - if ( - options.maxTotalCostUsd !== undefined && - (!Number.isFinite(options.maxTotalCostUsd) || options.maxTotalCostUsd < 0) - ) { - throw new Error('memory improvement maxTotalCostUsd must be a non-negative finite number') - } - if ( - options.activationTimeoutMs !== undefined && - (!Number.isSafeInteger(options.activationTimeoutMs) || options.activationTimeoutMs <= 0) - ) { - throw new Error('memory improvement activationTimeoutMs must be a positive safe integer') - } - const tolerance = options.criticalDimensionTolerance - if (tolerance !== undefined && (!Number.isFinite(tolerance) || tolerance < 0 || tolerance > 1)) { - throw new Error('memory improvement criticalDimensionTolerance must be between 0 and 1') - } - const minimum = options.minHoldoutScore - if (minimum !== undefined && (!Number.isFinite(minimum) || minimum < 0 || minimum > 1)) { - throw new Error('memory improvement minHoldoutScore must be between 0 and 1') - } - for (const seed of options.seeds) { - if ( - typeof seed.track !== 'string' || - !seed.track.trim() || - typeof seed.proposer !== 'string' || - !seed.proposer.trim() - ) { - throw new Error('memory improvement seeds require non-empty track and proposer values') - } - if (seed.vision !== undefined && (typeof seed.vision !== 'string' || !seed.vision.trim())) { - throw new Error('memory improvement seed vision must be a non-empty string when provided') - } - } - const dimensions = options.criticalDimensions ?? DEFAULT_CRITICAL_DIMENSIONS - if (dimensions.some((dimension) => typeof dimension !== 'string' || !dimension.trim())) { - throw new Error('memory improvement criticalDimensions must contain non-empty names') - } - if (new Set(dimensions).size !== dimensions.length) { - throw new Error('memory improvement criticalDimensions must be unique') - } - assertDistinctSequenceContent(options.trainSequences, 'train') - assertDistinctSequenceContent(options.holdoutSequences, 'holdout') -} - -function assertDistinctSequenceContent( - sequences: readonly AgentMemorySequence[], - split: string, -): void { - const idsByFingerprint = new Map() - for (const sequence of sequences) { - const fingerprint = memorySequenceFingerprint(sequence) - const prior = idsByFingerprint.get(fingerprint) - if (prior) { - throw new Error( - `memory improvement ${split} histories duplicate content: ${prior}/${sequence.id}`, - ) - } - idsByFingerprint.set(fingerprint, sequence.id) - } -} - -function mean(values: readonly number[]): number { - return values.length === 0 ? 0 : values.reduce((sum, value) => sum + value, 0) / values.length -} +export { runAgentMemoryImprovement } from './improvement/run' +export type { + AgentMemoryActivation, + AgentMemoryActivationDriver, + AgentMemoryDimensionComparison, + AgentMemoryGovernor, + AgentMemoryImprovementRunLease, + AgentMemoryImprovementSeed, + AgentMemoryPromotionDecision, + RunAgentMemoryImprovementOptions, + RunAgentMemoryImprovementResult, +} from './improvement/types' diff --git a/src/memory/improvement/activation.ts b/src/memory/improvement/activation.ts new file mode 100644 index 0000000..61124ae --- /dev/null +++ b/src/memory/improvement/activation.ts @@ -0,0 +1,91 @@ +import type { CampaignStorage } from '@tangle-network/agent-eval/campaign' +import { appendDurableJournalEvent } from '../attempt-log' +import type { AgentMemoryActivationEvent, AgentMemoryActivationJournalState } from './types' + +export function appendMemoryActivationEvent( + storage: CampaignStorage, + path: string, + event: AgentMemoryActivationEvent, +): void { + appendDurableJournalEvent({ + storage, + path, + event, + label: 'memory activation journal', + }) +} + +export function readMemoryActivationJournal( + storage: CampaignStorage, + path: string, + expected: Omit, +): AgentMemoryActivationJournalState { + const stored = storage.read(path) + if (stored === undefined) { + if (storage.exists(path)) throw new Error(`cannot read memory activation journal '${path}'`) + return { prepared: false } + } + let prepared = false + let activated: AgentMemoryActivationEvent | undefined + for (const [index, line] of stored.split('\n').entries()) { + if (!line) continue + let raw: unknown + try { + raw = JSON.parse(line) + } catch (error) { + throw new Error(`invalid memory activation journal '${path}' line ${index + 1}`, { + cause: error, + }) + } + const event = parseMemoryActivationEvent(raw, path, index + 1, expected) + if (event.status === 'prepared') { + if (prepared || activated) { + throw new Error(`memory activation journal '${path}' repeats its prepared event`) + } + prepared = true + continue + } + if (!prepared || activated) { + throw new Error(`memory activation journal '${path}' has an out-of-order activated event`) + } + activated = event + } + return { prepared, ...(activated ? { activated } : {}) } +} + +function parseMemoryActivationEvent( + value: unknown, + path: string, + line: number, + expected: Omit, +): AgentMemoryActivationEvent { + if (!value || typeof value !== 'object' || Array.isArray(value)) { + throw new Error(`invalid memory activation journal '${path}' line ${line}`) + } + const event = value as Record + for (const [key, expectedValue] of Object.entries(expected)) { + if (event[key] !== expectedValue) { + throw new Error( + `memory activation journal '${path}' line ${line} does not match the measured winner`, + ) + } + } + if (event.status !== 'prepared' && event.status !== 'activated') { + throw new Error(`invalid memory activation journal '${path}' line ${line} status`) + } + if (typeof event.recordedAt !== 'string' || !Number.isFinite(Date.parse(event.recordedAt))) { + throw new Error(`invalid memory activation journal '${path}' line ${line} recordedAt`) + } + if ( + event.status === 'activated' && + event.outcome !== 'applied' && + event.outcome !== 'recovered' && + event.outcome !== 'already-current' + ) { + throw new Error(`invalid memory activation journal '${path}' line ${line} outcome`) + } + if (event.status === 'prepared' && event.outcome !== undefined) { + throw new Error(`invalid memory activation journal '${path}' line ${line} prepared outcome`) + } + return value as AgentMemoryActivationEvent +} diff --git a/src/memory/improvement/candidate.ts b/src/memory/improvement/candidate.ts new file mode 100644 index 0000000..8db16b8 --- /dev/null +++ b/src/memory/improvement/candidate.ts @@ -0,0 +1,115 @@ +import type { + CampaignStorage, + CostLedgerHandle, + createRunCostLedger, + Governor, + SurfaceProposer, +} from '@tangle-network/agent-eval/campaign' +import type { AgentMemoryExperimentCandidate, RunAgentMemoryExperimentOptions } from '../experiment' +import type { AgentMemoryGovernor, OwnedRunLease, RunAgentMemoryImprovementOptions } from './types' + +export function withCostContext( + proposer: SurfaceProposer, + costLedger: CostLedgerHandle, + lease: OwnedRunLease, + label: string, +): SurfaceProposer { + return { + kind: proposer.kind, + async propose(context) { + await lease.assertOwned() + const proposal = await proposer.propose({ + ...context, + costLedger, + costPhase: `memory.proposal.${context.track?.id ?? label}`, + }) + await lease.assertOwned() + return proposal + }, + ...(proposer.decide ? { decide: (input) => proposer.decide!(input) } : {}), + } +} + +export function withGovernorCostContext( + governor: AgentMemoryGovernor, + costLedger: CostLedgerHandle, + lease: OwnedRunLease, +): Governor { + return { + async decide(context) { + await lease.assertOwned() + const decision = await governor.decide({ + ...context, + costLedger, + costPhase: 'memory.governor', + }) + await lease.assertOwned() + return decision + }, + } +} + +export async function buildCandidate( + options: RunAgentMemoryImprovementOptions, + config: TConfig, + hash: string, + role: string, +): Promise { + const built = await options.createCandidate({ + config, + candidateId: `${role}-${hash}`, + surfaceHash: hash, + }) + return { + ...built, + id: `${role}-${hash}`, + ref: built.ref, + } +} + +export function experimentOptions( + options: RunAgentMemoryImprovementOptions, + costLedger: ReturnType, + storage: CampaignStorage, + lease: OwnedRunLease, +): Pick< + RunAgentMemoryExperimentOptions, + | 'storage' + | 'seed' + | 'reps' + | 'resumable' + | 'maxConcurrency' + | 'dispatchTimeoutMs' + | 'cleanupTimeoutMs' + | 'maxRecoveryAttempts' + | 'maxRecoveryRetriesPerAttempt' + | 'executeStep' + | 'executeStepRef' + | 'onBranchSnapshot' + | 'cleanupBranches' + | 'costLedger' + | 'acquireRunLease' + | 'now' +> { + return { + storage, + seed: options.seed, + reps: options.reps, + resumable: options.resumable, + maxConcurrency: options.sequenceConcurrency, + dispatchTimeoutMs: options.dispatchTimeoutMs, + cleanupTimeoutMs: options.cleanupTimeoutMs, + maxRecoveryAttempts: options.maxRecoveryAttempts, + maxRecoveryRetriesPerAttempt: options.maxRecoveryRetriesPerAttempt, + executeStep: options.executeStep, + executeStepRef: options.executeStepRef, + onBranchSnapshot: options.onBranchSnapshot, + cleanupBranches: options.cleanupBranches ?? true, + costLedger, + acquireRunLease: async () => ({ + assertOwned: () => lease.assertOwned(), + release() {}, + }), + now: options.now, + } +} diff --git a/src/memory/improvement/identity.ts b/src/memory/improvement/identity.ts new file mode 100644 index 0000000..1167a2b --- /dev/null +++ b/src/memory/improvement/identity.ts @@ -0,0 +1,132 @@ +import { join } from 'node:path' +import { canonicalJson } from '@tangle-network/agent-eval' +import { + type CampaignStorage, + type MutableSurface, + surfaceHash, +} from '@tangle-network/agent-eval/campaign' +import type { AgentMemorySequence } from '../experiment' +import { normalizedPromotionPolicy } from './promotion' +import { + MEMORY_IMPROVEMENT_IMPLEMENTATION_REF, + type RunAgentMemoryImprovementOptions, +} from './types' + +export function assertMemoryImprovementIdentity( + options: RunAgentMemoryImprovementOptions, + storage: CampaignStorage, + runDir: string, + serialize: (config: TConfig) => string, +): void { + const path = join(runDir, 'memory-improvement-manifest.json') + const identity = { + schema: 6, + implementationRef: MEMORY_IMPROVEMENT_IMPLEMENTATION_REF, + experimentId: options.experimentId, + improvementRef: options.improvementRef, + activationRef: options.activation?.ref ?? null, + proposerKind: options.proposer.kind, + proposerKinds: Object.fromEntries( + Object.entries(options.proposers ?? {}) + .sort(([a], [b]) => a.localeCompare(b)) + .map(([name, proposer]) => [name, proposer.kind]), + ), + populationSize: options.populationSize ?? 4, + budget: { maxSteps: options.budget.maxSteps }, + executeStepRef: options.executeStepRef ?? null, + cleanupBranches: options.cleanupBranches ?? true, + promotionPolicy: normalizedPromotionPolicy(options), + seed: options.seed ?? null, + reps: options.reps ?? 1, + seeds: options.seeds.map((entry) => ({ + config: serialize(entry.config), + track: entry.track, + vision: entry.vision ?? null, + proposer: entry.proposer, + })), + trainSequences: options.trainSequences, + holdoutSequences: options.holdoutSequences, + } + const identityHash = surfaceHash(canonicalJson(identity)) + const stored = storage.read(path) + if (stored === undefined) { + if (storage.exists(path)) throw new Error(`cannot read memory improvement manifest '${path}'`) + if ( + storage.exists(join(runDir, 'lineage.jsonl')) || + storage.exists(join(runDir, 'memory-improvement-result.json')) + ) { + throw new Error(`memory improvement run '${runDir}' has state without an identity manifest`) + } + storage.write(path, `${JSON.stringify({ identityHash, identity }, null, 2)}\n`) + return + } + let manifest: unknown + try { + manifest = JSON.parse(stored) + } catch (error) { + throw new Error(`invalid memory improvement manifest '${path}'`, { cause: error }) + } + if ( + !manifest || + typeof manifest !== 'object' || + (manifest as Record).identityHash !== identityHash || + canonicalJson((manifest as Record).identity) !== canonicalJson(identity) + ) { + throw new Error( + `memory improvement run '${runDir}' does not match its persisted inputs or implementationRef`, + ) + } +} + +export function requireStringSurface(surface: MutableSurface): string { + if (typeof surface !== 'string' || surface.trim().length === 0) { + throw new Error('memory config proposer must return a JSON string surface') + } + return surface +} + +export function serializeMemoryConfig( + serialize: (config: TConfig) => string, + config: TConfig, +): string { + let surface: unknown + try { + surface = serialize(config) + } catch (error) { + throw new Error('memory config serializer failed', { cause: error }) + } + if (typeof surface !== 'string' || surface.trim().length === 0) { + throw new Error('memory config serializer must return a non-empty string') + } + return surface +} + +export function parseMemoryConfig( + parse: (surface: string) => TConfig, + surface: string, +): TConfig { + try { + const config = parse(surface) + if (config === undefined) throw new Error('parser returned undefined') + return config + } catch (error) { + throw new Error('memory config surface could not be parsed', { cause: error }) + } +} + +export function assertMemoryConfigRoundTrip( + config: TConfig, + serialize: (config: TConfig) => string, + parse: (surface: string) => TConfig, +): void { + const first = serialize(config) + const second = serialize(parse(first)) + if (first !== second) { + throw new Error('memory config serializer and parser must round-trip seed configs exactly') + } +} + +export function memorySequenceFingerprint(sequence: AgentMemorySequence): string { + const { id: _id, split: _split, tags: _tags, ...content } = sequence + return surfaceHash(canonicalJson(content)) +} diff --git a/src/memory/improvement/output.ts b/src/memory/improvement/output.ts new file mode 100644 index 0000000..396261c --- /dev/null +++ b/src/memory/improvement/output.ts @@ -0,0 +1,27 @@ +import type { CampaignStorage } from '@tangle-network/agent-eval/campaign' +import type { RunAgentMemoryImprovementResult } from './types' + +export function writeMemoryImprovementResult( + storage: CampaignStorage, + experimentId: string, + result: RunAgentMemoryImprovementResult, +): void { + storage.write( + result.resultJsonPath, + `${JSON.stringify( + { + experimentId, + baselineSurfaceHash: result.baselineSurfaceHash, + winnerSurfaceHash: result.winnerSurfaceHash, + baselineSurface: result.baselineSurface, + winnerSurface: result.winnerSurface, + decision: result.decision, + activation: result.activation, + totalCostUsd: result.totalCostUsd, + lineage: result.lineage.toGraph(), + }, + null, + 2, + )}\n`, + ) +} diff --git a/src/memory/improvement/promotion.ts b/src/memory/improvement/promotion.ts new file mode 100644 index 0000000..b8f4925 --- /dev/null +++ b/src/memory/improvement/promotion.ts @@ -0,0 +1,189 @@ +import { heldoutSignificance, type PairedHoldout } from '@tangle-network/agent-eval/campaign' +import type { + AgentMemorySequence, + AgentMemorySequenceArtifact, + AgentMemorySequenceProbe, + RunAgentMemoryExperimentResult, +} from '../experiment' +import { + type AgentMemoryPromotionDecision, + DEFAULT_CRITICAL_DIMENSIONS, + type RunAgentMemoryImprovementOptions, +} from './types' + +export function decidePromotion(input: { + options: RunAgentMemoryImprovementOptions + result: RunAgentMemoryExperimentResult + baselineId: string + winnerId: string +}): AgentMemoryPromotionDecision { + const { options, result, baselineId, winnerId } = input + const baselineRow = result.rows.find((row) => row.candidateId === baselineId) + const winnerRow = result.rows.find((row) => row.candidateId === winnerId) + if (!baselineRow || !winnerRow) throw new Error('holdout result is missing a comparison arm') + const paired = pairedArtifacts(result, baselineId, winnerId, (artifact) => artifact.score) + const significance = heldoutSignificance(paired, options.significance) + const tolerance = options.criticalDimensionTolerance ?? 0.05 + const criticalDimensions = (options.criticalDimensions ?? DEFAULT_CRITICAL_DIMENSIONS).map( + (dimension) => { + const expectedN = + applicableSequenceCount(options.holdoutSequences, dimension) * (options.reps ?? 1) + const dimensionPairs = pairedArtifacts(result, baselineId, winnerId, (artifact) => + (artifact.dimensionSampleCounts?.[dimension] ?? 0) > 0 + ? artifact.dimensions[dimension] + : undefined, + ) + const comparison = heldoutSignificance(dimensionPairs, { + ...options.significance, + deltaThreshold: 0, + }) + return { + dimension, + n: comparison.n, + expectedN, + measured: expectedN > 0 && comparison.n === expectedN, + meanDelta: comparison.bootstrap.mean, + low: comparison.bootstrap.low, + high: comparison.bootstrap.high, + tolerance, + regressed: + expectedN > 0 && comparison.n === expectedN && comparison.bootstrap.low < -tolerance, + } + }, + ) + const reasons: string[] = [] + if (baselineRow.cellsFailed > 0 || winnerRow.cellsFailed > 0) { + reasons.push('at least one holdout cell failed') + } + if (!significance.significant) { + reasons.push( + significance.fewRuns + ? `only ${significance.n} paired holdout cells; more are required` + : 'holdout lift is not confidently above the promotion threshold', + ) + } + if (winnerRow.scoreMean < (options.minHoldoutScore ?? 0)) { + reasons.push(`winner holdout score ${winnerRow.scoreMean} is below the required minimum`) + } + for (const dimension of criticalDimensions) { + if (!dimension.measured) { + reasons.push( + dimension.expectedN === 0 + ? `critical dimension ${dimension.dimension} has no applicable holdout histories` + : `critical dimension ${dimension.dimension} was measured on ${dimension.n}/${dimension.expectedN} applicable paired holdout cells`, + ) + } else if (dimension.regressed) { + reasons.push(`${dimension.dimension} may regress beyond ${tolerance}`) + } + } + return { + status: reasons.length === 0 ? 'promote' : 'hold', + reasons, + baselineScore: baselineRow.scoreMean, + winnerScore: winnerRow.scoreMean, + lift: winnerRow.scoreMean - baselineRow.scoreMean, + significance, + criticalDimensions, + } +} + +function applicableSequenceCount( + sequences: readonly AgentMemorySequence[], + dimension: string, +): number { + return sequences.filter((sequence) => + sequence.steps.some((step) => + (step.probes ?? []).some((probe) => probeAppliesToDimension(probe, dimension)), + ), + ).length +} + +function probeAppliesToDimension(probe: AgentMemorySequenceProbe, dimension: string): boolean { + switch (dimension) { + case 'memory_fact_recall': + case 'memory_required_fact_count': + case 'memory_matched_fact_count': + return Boolean( + (probe.requiredFacts && probe.requiredFacts.length > 0) || probe.referenceAnswer, + ) + case 'memory_event_recall': + return Boolean(probe.expectedEventIds && probe.expectedEventIds.length > 0) + case 'memory_actor_recall': + return Boolean(probe.expectedActorIds && probe.expectedActorIds.length > 0) + case 'memory_stale_safe': + case 'memory_stale_rate': + case 'memory_forbidden_fact_count': + case 'memory_matched_forbidden_fact_count': + return Boolean(probe.forbiddenFacts && probe.forbiddenFacts.length > 0) + default: + return true + } +} + +export function normalizedPromotionPolicy( + options: RunAgentMemoryImprovementOptions, +): Record { + return { + significance: { + deltaThreshold: options.significance?.deltaThreshold ?? 0, + minProductiveRuns: options.significance?.minProductiveRuns ?? 3, + confidence: options.significance?.confidence ?? 0.95, + resamples: options.significance?.resamples ?? 2000, + seed: options.significance?.seed ?? 1337, + statistic: options.significance?.statistic ?? 'mean', + }, + criticalDimensions: [...(options.criticalDimensions ?? DEFAULT_CRITICAL_DIMENSIONS)], + criticalDimensionTolerance: options.criticalDimensionTolerance ?? 0.05, + minHoldoutScore: options.minHoldoutScore ?? 0, + } +} + +function pairedArtifacts( + result: RunAgentMemoryExperimentResult, + baselineId: string, + winnerId: string, + select: (artifact: AgentMemorySequenceArtifact) => number | undefined, +): PairedHoldout { + const baseline = artifactValues(result, baselineId, select) + const winner = artifactValues(result, winnerId, select) + const keys = [...baseline.keys()].filter((key) => winner.has(key)).sort() + return { + before: keys.map((key) => baseline.get(key)!), + after: keys.map((key) => winner.get(key)!), + cellIds: keys, + } +} + +function artifactValues( + result: RunAgentMemoryExperimentResult, + candidateId: string, + select: (artifact: AgentMemorySequenceArtifact) => number | undefined, +): Map { + const values = new Map() + for (const cell of result.campaign.cells) { + if (cell.error || cell.artifact.candidateId !== candidateId) continue + const value = select(cell.artifact) + if (value === undefined || !Number.isFinite(value)) continue + values.set(`${cell.artifact.sequenceId}:${cell.rep}`, value) + } + return values +} + +export function sequenceScores( + result: RunAgentMemoryExperimentResult, + sequences: readonly AgentMemorySequence[], + candidateId: string, +): number[] { + const bySequence = new Map() + for (const cell of result.campaign.cells) { + if (cell.error || cell.artifact.candidateId !== candidateId) continue + const bucket = bySequence.get(cell.artifact.sequenceId) ?? [] + bucket.push(cell.artifact.score) + bySequence.set(cell.artifact.sequenceId, bucket) + } + return sequences.map((sequence) => mean(bySequence.get(sequence.id) ?? [])) +} + +function mean(values: readonly number[]): number { + return values.length === 0 ? 0 : values.reduce((sum, value) => sum + value, 0) / values.length +} diff --git a/src/memory/improvement/run.ts b/src/memory/improvement/run.ts new file mode 100644 index 0000000..9286370 --- /dev/null +++ b/src/memory/improvement/run.ts @@ -0,0 +1,439 @@ +import { join } from 'node:path' +import { canonicalJson } from '@tangle-network/agent-eval' +import { + type CampaignStorage, + campaignLineageStore, + createRunCostLedger, + fsCampaignStorage, + type MutableSurface, + memLineageStore, + resolveRunDir, + runLineageLoop, + surfaceHash, +} from '@tangle-network/agent-eval/campaign' +import { assertNoInterruptedPaidCalls, reconcileInterruptedRunPaidCalls } from '../attempt-log' +import { type RunAgentMemoryExperimentResult, runAgentMemoryExperiment } from '../experiment' +import { runBoundedMemoryLifecycle } from '../lifecycle' +import { acquireAgentMemoryRunLease } from '../run-control' +import { appendMemoryActivationEvent, readMemoryActivationJournal } from './activation' +import { + buildCandidate, + experimentOptions, + withCostContext, + withGovernorCostContext, +} from './candidate' +import { + assertMemoryConfigRoundTrip, + assertMemoryImprovementIdentity, + memorySequenceFingerprint, + parseMemoryConfig, + requireStringSurface, + serializeMemoryConfig, +} from './identity' +import { writeMemoryImprovementResult } from './output' +import { decidePromotion, normalizedPromotionPolicy, sequenceScores } from './promotion' +import type { + AgentMemoryActivation, + AgentMemoryActivationEvent, + AgentMemoryPromotionDecision, + MemoryConfigScenario, + OwnedRunLease, + RunAgentMemoryImprovementOptions, + RunAgentMemoryImprovementResult, +} from './types' +import { assertMemoryImprovementOptions } from './validation' + +/** Searches branchable memory configurations and activates only a fresh holdout win. */ +export async function runAgentMemoryImprovement( + options: RunAgentMemoryImprovementOptions, +): Promise> { + if ('onPromote' in options) { + throw new Error( + 'memory improvement onPromote was removed; use activation.readCurrent and activation.compareAndSet', + ) + } + if (options.seeds.length === 0) throw new Error('memory improvement requires seed configs') + if (options.trainSequences.length === 0) { + throw new Error('memory improvement requires training sequences') + } + if (options.holdoutSequences.length === 0) { + throw new Error('memory improvement requires holdout sequences') + } + const trainIds = new Set(options.trainSequences.map((sequence) => sequence.id)) + const overlap = options.holdoutSequences + .map((sequence) => sequence.id) + .filter((id) => trainIds.has(id)) + if (overlap.length > 0) { + throw new Error(`memory improvement train/holdout overlap: ${overlap.join(', ')}`) + } + const trainFingerprints = new Map( + options.trainSequences.map((sequence) => [memorySequenceFingerprint(sequence), sequence.id]), + ) + const duplicateHistories = options.holdoutSequences.flatMap((sequence) => { + const trainId = trainFingerprints.get(memorySequenceFingerprint(sequence)) + return trainId ? [`${trainId}/${sequence.id}`] : [] + }) + if (duplicateHistories.length > 0) { + throw new Error( + `memory improvement train/holdout histories duplicate content: ${duplicateHistories.join(', ')}`, + ) + } + if (typeof options.improvementRef !== 'string' || !options.improvementRef.trim()) { + throw new Error('memory improvement improvementRef must be a non-empty string') + } + assertMemoryImprovementOptions(options) + const storage = options.storage ?? fsCampaignStorage() + const runDir = resolveRunDir(options.runDir, options.repo) + storage.ensureDir(runDir) + const lease = await acquireAgentMemoryRunLease({ + experimentId: options.experimentId, + runDir, + storage, + customStorage: options.storage !== undefined, + lockFileName: 'memory-improvement.lock', + label: 'memory improvement', + controllerMode: options.controllerMode, + acquireRunLease: options.acquireRunLease, + }) + let result: RunAgentMemoryImprovementResult | undefined + let primaryError: unknown + try { + result = await runAgentMemoryImprovementOwned(options, storage, runDir, lease) + } catch (error) { + primaryError = error + } + let releaseError: unknown + try { + await lease.release() + } catch (error) { + releaseError = error + } + if (primaryError && releaseError) { + throw new AggregateError( + [primaryError, releaseError], + 'memory improvement failed and its controller lease could not be released', + ) + } + if (primaryError) throw primaryError + if (releaseError) throw releaseError + if (!result) throw new Error('memory improvement produced no result') + return result +} + +async function runAgentMemoryImprovementOwned( + options: RunAgentMemoryImprovementOptions, + storage: CampaignStorage, + runDir: string, + lease: OwnedRunLease, +): Promise> { + await lease.assertOwned() + const serializeRaw = options.serializeConfig ?? ((config: TConfig) => canonicalJson(config)) + const parseRaw = options.parseConfig ?? ((surface: string) => JSON.parse(surface) as TConfig) + const serialize = (config: TConfig): string => serializeMemoryConfig(serializeRaw, config) + const parse = (surface: string): TConfig => parseMemoryConfig(parseRaw, surface) + for (const seed of options.seeds) assertMemoryConfigRoundTrip(seed.config, serialize, parse) + if (options.activation && !storage.append) { + throw new Error('memory activation requires CampaignStorage.append') + } + if (options.resumable !== false && !options.lineageStore && !storage.append) { + throw new Error('resumable memory improvement requires CampaignStorage.append') + } + assertMemoryImprovementIdentity(options, storage, runDir, serialize) + const costLedger = createRunCostLedger({ + storage, + runDir, + costCeilingUsd: options.maxTotalCostUsd ?? 0, + }) + reconcileInterruptedRunPaidCalls(costLedger, 'memory improvement run') + assertNoInterruptedPaidCalls(costLedger, 'memory improvement recovery') + const trainScenarios: MemoryConfigScenario[] = options.trainSequences.map((sequence) => ({ + id: sequence.id, + kind: 'agent-memory-config-search', + sequenceId: sequence.id, + })) + const evaluations = new Map>() + + const evaluateSurface = async ( + surface: MutableSurface, + ): Promise<{ score: number; scoreVector: number[] }> => { + await lease.assertOwned() + const text = requireStringSurface(surface) + const config = parse(text) + const canonicalSurface = serialize(config) + const hash = surfaceHash(canonicalSurface) + let pending = evaluations.get(hash) + if (!pending) { + pending = (async () => { + const candidate = await buildCandidate(options, config, hash, `search-${hash}`) + await lease.assertOwned() + const experiment = await runAgentMemoryExperiment({ + ...experimentOptions(options, costLedger, storage, lease), + experimentId: `${options.experimentId}:search:${hash}`, + sequences: options.trainSequences, + candidates: [candidate], + runDir: join(runDir, 'search', hash), + costPhase: `memory.search.${hash}`, + }) + await lease.assertOwned() + return experiment + })() + evaluations.set(hash, pending) + void pending.catch(() => evaluations.delete(hash)) + } + const result = await pending + await lease.assertOwned() + const row = result.rows[0] + if (!row || row.cellsFailed > 0) { + throw new Error(`${hash}: memory candidate did not complete every training cell`) + } + return { + score: row.scoreMean, + scoreVector: sequenceScores(result, options.trainSequences, row.candidateId), + } + } + + const lineageStore = + options.lineageStore ?? + (options.resumable === false + ? memLineageStore() + : campaignLineageStore(storage, join(runDir, 'lineage.jsonl'))) + const lineageOptions = { + seeds: options.seeds.map((seed) => ({ + surface: serialize(seed.config), + track: seed.track, + proposer: seed.proposer, + ...(seed.vision !== undefined ? { vision: seed.vision } : {}), + })), + scenarios: trainScenarios, + proposer: withCostContext(options.proposer, costLedger, lease, 'default'), + proposers: options.proposers + ? Object.fromEntries( + Object.entries(options.proposers).map(([name, proposer]) => [ + name, + withCostContext(proposer, costLedger, lease, name), + ]), + ) + : undefined, + scoreSurface: evaluateSurface, + governor: options.governor + ? withGovernorCostContext(options.governor, costLedger, lease) + : undefined, + budget: { + ...options.budget, + maxNodes: options.seeds.length + options.budget.maxSteps, + }, + store: lineageStore, + populationSize: options.populationSize, + candidateConcurrency: options.candidateConcurrency, + } + const search = await runLineageLoop(lineageOptions) + await lease.assertOwned() + const best = search.best + if (!best) throw new Error('memory improvement produced no measured config') + + const baselineSurface = serialize(options.seeds[0]!.config) + const baselineHash = surfaceHash(baselineSurface) + const winnerConfig = parse(requireStringSurface(best.surface)) + const winnerSurface = serialize(winnerConfig) + const winnerHash = surfaceHash(winnerSurface) + const baselineConfig = parse(baselineSurface) + + let holdout: RunAgentMemoryExperimentResult | undefined + let decision: AgentMemoryPromotionDecision + if (winnerHash === baselineHash) { + const baselineMeasurement = await evaluateSurface(baselineSurface) + decision = { + status: 'no-change', + reasons: ['search did not find a config better than the baseline'], + baselineScore: baselineMeasurement.score, + winnerScore: baselineMeasurement.score, + lift: 0, + criticalDimensions: [], + } + } else { + await lease.assertOwned() + const [baselineCandidate, winnerCandidate] = await Promise.all([ + buildCandidate(options, baselineConfig, baselineHash, 'baseline'), + buildCandidate(options, winnerConfig, winnerHash, 'winner'), + ]) + await lease.assertOwned() + holdout = await runAgentMemoryExperiment({ + ...experimentOptions(options, costLedger, storage, lease), + experimentId: `${options.experimentId}:holdout`, + sequences: options.holdoutSequences, + candidates: [baselineCandidate, winnerCandidate], + runDir: join(runDir, 'holdout'), + costPhase: 'memory.holdout', + }) + decision = decidePromotion({ + options, + result: holdout, + baselineId: baselineCandidate.id, + winnerId: winnerCandidate.id, + }) + } + + const resultJsonPath = join(runDir, 'memory-improvement-result.json') + const activationRef = options.activation?.ref ?? 'not-configured' + const activationId = `memory-activation-${surfaceHash( + canonicalJson({ + experimentId: options.experimentId, + improvementRef: options.improvementRef, + activationRef, + baselineSurfaceHash: baselineHash, + winnerSurfaceHash: winnerHash, + holdoutManifestHash: holdout?.campaign.manifestHash ?? null, + promotionPolicy: normalizedPromotionPolicy(options), + }), + )}` + const activationJournalDir = join(runDir, 'activations') + const activationJournalPath = join(activationJournalDir, `${activationId}.jsonl`) + const activationEligible = decision.status === 'promote' && holdout !== undefined + const activationEventIdentity = holdout + ? { + schema: 1 as const, + activationId, + experimentId: options.experimentId, + activationRef, + baselineSurfaceHash: baselineHash, + winnerSurfaceHash: winnerHash, + holdoutManifestHash: holdout.campaign.manifestHash, + } + : undefined + const activationJournal = + activationEligible && activationEventIdentity + ? readMemoryActivationJournal(storage, activationJournalPath, activationEventIdentity) + : { prepared: false } + const activation: AgentMemoryActivation = { + id: activationId, + status: !activationEligible + ? 'not-eligible' + : activationJournal.activated + ? 'already-activated' + : options.activation + ? 'pending' + : 'not-configured', + journalPath: activationJournalPath, + } + const result = { + lineage: search.lineage, + baselineConfig, + winnerConfig, + baselineSurface, + winnerSurface, + baselineSurfaceHash: baselineHash, + winnerSurfaceHash: winnerHash, + decision, + activation, + ...(holdout ? { holdout } : {}), + totalCostUsd: costLedger.summary().totalCostUsd, + resultJsonPath, + } satisfies RunAgentMemoryImprovementResult + await lease.assertOwned() + writeMemoryImprovementResult(storage, options.experimentId, result) + if ( + activationEligible && + !activationJournal.activated && + activationEventIdentity && + holdout && + options.activation + ) { + const activationDriver = options.activation + const activationTimeoutMs = options.activationTimeoutMs ?? 60_000 + const hadPreparedEvent = activationJournal.prepared + if (!hadPreparedEvent) { + await lease.assertOwned() + storage.ensureDir(activationJournalDir) + appendMemoryActivationEvent(storage, activationJournalPath, { + ...activationEventIdentity, + status: 'prepared', + recordedAt: (options.now ?? (() => new Date()))().toISOString(), + }) + } + + await lease.assertOwned() + const currentConfig = await runBoundedMemoryLifecycle({ + operation: `${activationDriver.ref}: read current memory configuration`, + timeoutMs: activationTimeoutMs, + run: () => activationDriver.readCurrent(), + }) + await lease.assertOwned() + const currentHash = surfaceHash(serialize(currentConfig)) + if (currentHash !== baselineHash && currentHash !== winnerHash) { + throw new Error( + `memory activation target '${activationDriver.ref}' changed concurrently; expected '${baselineHash}' or '${winnerHash}', found '${currentHash}'`, + ) + } + + let outcome: NonNullable + if (currentHash === winnerHash) { + outcome = hadPreparedEvent ? 'recovered' : 'already-current' + activation.status = 'recovered' + } else { + let compareError: unknown + try { + await runBoundedMemoryLifecycle({ + operation: `${activationDriver.ref}: activate memory configuration`, + timeoutMs: activationTimeoutMs, + run: () => + activationDriver.compareAndSet({ + activationId, + expectedConfig: baselineConfig, + expectedSurfaceHash: baselineHash, + config: winnerConfig, + surfaceHash: winnerHash, + decision, + lineage: search.lineage, + holdout, + }), + }) + } catch (error) { + compareError = error + } + await lease.assertOwned() + + let observedConfig: TConfig + try { + observedConfig = await runBoundedMemoryLifecycle({ + operation: `${activationDriver.ref}: confirm memory configuration`, + timeoutMs: activationTimeoutMs, + run: () => activationDriver.readCurrent(), + }) + } catch (error) { + if (compareError) { + throw new AggregateError( + [compareError, error], + `memory activation '${activationId}' failed and its live state could not be confirmed`, + ) + } + throw error + } + await lease.assertOwned() + const observedHash = surfaceHash(serialize(observedConfig)) + if (observedHash !== winnerHash) { + const mismatch = new Error( + `memory activation '${activationId}' did not install the measured winner; found '${observedHash}'`, + ) + if (compareError) { + throw new AggregateError( + [compareError, mismatch], + `memory activation '${activationId}' failed without applying the measured winner`, + ) + } + throw mismatch + } + outcome = compareError ? 'recovered' : 'applied' + activation.status = compareError ? 'recovered' : 'activated' + } + + await lease.assertOwned() + appendMemoryActivationEvent(storage, activationJournalPath, { + ...activationEventIdentity, + status: 'activated', + outcome, + recordedAt: (options.now ?? (() => new Date()))().toISOString(), + }) + writeMemoryImprovementResult(storage, options.experimentId, result) + } + return result +} diff --git a/src/memory/improvement/types.ts b/src/memory/improvement/types.ts new file mode 100644 index 0000000..0aa38ed --- /dev/null +++ b/src/memory/improvement/types.ts @@ -0,0 +1,196 @@ +import type { + CampaignStorage, + CostLedgerHandle, + GovernorContext, + GovernorOp, + HeldoutSignificance, + HeldoutSignificanceOptions, + Lineage, + LineageStore, + Scenario, + SurfaceProposer, +} from '@tangle-network/agent-eval/campaign' +import type { + AgentMemoryExperimentCandidate, + AgentMemorySequence, + RunAgentMemoryExperimentOptions, + RunAgentMemoryExperimentResult, +} from '../experiment' +import type { + AgentMemoryAcquireRunLease, + AgentMemoryControllerMode, + AgentMemoryRunLease, + OwnedAgentMemoryRunLease, +} from '../run-control' + +export interface AgentMemoryImprovementSeed { + config: TConfig + track: string + vision?: string + proposer: string +} + +export interface AgentMemoryDimensionComparison { + dimension: string + n: number + expectedN: number + measured: boolean + meanDelta: number + low: number + high: number + tolerance: number + regressed: boolean +} + +export interface AgentMemoryPromotionDecision { + status: 'promote' | 'hold' | 'no-change' + reasons: readonly string[] + baselineScore: number + winnerScore: number + lift: number + significance?: HeldoutSignificance + criticalDimensions: readonly AgentMemoryDimensionComparison[] +} + +export interface AgentMemoryActivation { + id: string + status: + | 'not-eligible' + | 'not-configured' + | 'pending' + | 'activated' + | 'recovered' + | 'already-activated' + journalPath: string +} + +export interface AgentMemoryActivationDriver { + /** Change whenever activation behavior or the external target changes. */ + ref: string + /** Return the exact currently active configuration. */ + readCurrent(): Promise + /** Atomically replace expectedConfig with config, or fail on a concurrent change. */ + compareAndSet(input: { + activationId: string + expectedConfig: TConfig + expectedSurfaceHash: string + config: TConfig + surfaceHash: string + decision: AgentMemoryPromotionDecision + lineage: Lineage + holdout: RunAgentMemoryExperimentResult + }): Promise +} + +export interface AgentMemoryActivationEvent { + schema: 1 + status: 'prepared' | 'activated' + activationId: string + experimentId: string + activationRef: string + baselineSurfaceHash: string + winnerSurfaceHash: string + holdoutManifestHash: string + recordedAt: string + outcome?: 'applied' | 'recovered' | 'already-current' +} + +export interface AgentMemoryActivationJournalState { + prepared: boolean + activated?: AgentMemoryActivationEvent +} + +export type AgentMemoryImprovementRunLease = AgentMemoryRunLease + +export interface AgentMemoryGovernor { + decide( + context: GovernorContext & { + costLedger: CostLedgerHandle + costPhase: string + }, + ): GovernorOp | Promise +} + +export interface RunAgentMemoryImprovementOptions { + experimentId: string + trainSequences: readonly AgentMemorySequence[] + holdoutSequences: readonly AgentMemorySequence[] + /** First entry is the current baseline; remaining entries seed independent search tracks. */ + seeds: readonly AgentMemoryImprovementSeed[] + createCandidate(input: { + config: TConfig + candidateId: string + surfaceHash: string + }): + | Omit + | Promise> + proposer: SurfaceProposer + /** Optional proposer implementations keyed by seed and branch proposer labels. */ + proposers?: Readonly> + /** Stable version or commit for the candidate factory, proposer, and governor. */ + improvementRef: string + governor?: AgentMemoryGovernor + budget: { maxSteps: number } + populationSize?: number + candidateConcurrency?: number + sequenceConcurrency?: number + runDir: string + repo?: string + storage?: CampaignStorage + lineageStore?: LineageStore + /** Required with custom storage when all controllers are confined to one process. */ + controllerMode?: AgentMemoryControllerMode + /** Required for distributed controllers using custom storage. Worker concurrency is independent. */ + acquireRunLease?: AgentMemoryAcquireRunLease + seed?: number + reps?: number + resumable?: boolean + dispatchTimeoutMs?: number + cleanupTimeoutMs?: number + maxRecoveryAttempts?: number + maxRecoveryRetriesPerAttempt?: number + maxTotalCostUsd?: number + executeStep?: RunAgentMemoryExperimentOptions['executeStep'] + executeStepRef?: string + onBranchSnapshot?: RunAgentMemoryExperimentOptions['onBranchSnapshot'] + cleanupBranches?: boolean + serializeConfig?: (config: TConfig) => string + parseConfig?: (surface: string) => TConfig + significance?: HeldoutSignificanceOptions + criticalDimensions?: readonly string[] + criticalDimensionTolerance?: number + minHoldoutScore?: number + activation?: AgentMemoryActivationDriver + activationTimeoutMs?: number + now?: () => Date +} + +export interface RunAgentMemoryImprovementResult { + lineage: Lineage + baselineConfig: TConfig + winnerConfig: TConfig + baselineSurface: string + winnerSurface: string + baselineSurfaceHash: string + winnerSurfaceHash: string + decision: AgentMemoryPromotionDecision + activation: AgentMemoryActivation + holdout?: RunAgentMemoryExperimentResult + totalCostUsd: number + resultJsonPath: string +} + +export interface MemoryConfigScenario extends Scenario { + kind: 'agent-memory-config-search' + sequenceId: string +} + +export const DEFAULT_CRITICAL_DIMENSIONS = [ + 'memory_stale_safe', + 'memory_actor_recall', + 'memory_event_recall', +] as const + +export const MEMORY_IMPROVEMENT_IMPLEMENTATION_REF = 'agent-knowledge:memory-improvement:v2' + +export type OwnedRunLease = OwnedAgentMemoryRunLease diff --git a/src/memory/improvement/validation.ts b/src/memory/improvement/validation.ts new file mode 100644 index 0000000..67bba42 --- /dev/null +++ b/src/memory/improvement/validation.ts @@ -0,0 +1,127 @@ +import type { AgentMemorySequence } from '../experiment' +import { memorySequenceFingerprint } from './identity' +import { DEFAULT_CRITICAL_DIMENSIONS, type RunAgentMemoryImprovementOptions } from './types' + +export function assertMemoryImprovementOptions( + options: RunAgentMemoryImprovementOptions, +): void { + for (const [name, value] of [ + ['experimentId', options.experimentId], + ['runDir', options.runDir], + ] as const) { + if (typeof value !== 'string' || !value.trim()) { + throw new Error(`memory improvement ${name} must be a non-empty string`) + } + } + if (typeof options.proposer?.kind !== 'string' || !options.proposer.kind.trim()) { + throw new Error('memory improvement proposer.kind must be a non-empty string') + } + if (typeof options.proposer?.propose !== 'function') { + throw new Error('memory improvement proposer.propose must be a function') + } + if (options.governor !== undefined && typeof options.governor.decide !== 'function') { + throw new Error('memory improvement governor.decide must be a function') + } + if (options.activation !== undefined) { + if (typeof options.activation.ref !== 'string' || !options.activation.ref.trim()) { + throw new Error('memory improvement activation.ref must be a non-empty string') + } + if (typeof options.activation.readCurrent !== 'function') { + throw new Error('memory improvement activation.readCurrent must be a function') + } + if (typeof options.activation.compareAndSet !== 'function') { + throw new Error('memory improvement activation.compareAndSet must be a function') + } + } + for (const [name, proposer] of Object.entries(options.proposers ?? {})) { + if (!name.trim()) throw new Error('memory improvement proposer labels must be non-empty') + if ( + !proposer || + typeof proposer.kind !== 'string' || + !proposer.kind.trim() || + typeof proposer.propose !== 'function' + ) { + throw new Error(`memory improvement proposer '${name}' is invalid`) + } + } + if (options.serializeConfig !== undefined && typeof options.serializeConfig !== 'function') { + throw new Error('memory improvement serializeConfig must be a function') + } + if (options.parseConfig !== undefined && typeof options.parseConfig !== 'function') { + throw new Error('memory improvement parseConfig must be a function') + } + if (!Number.isSafeInteger(options.budget.maxSteps) || options.budget.maxSteps < 0) { + throw new Error('memory improvement budget.maxSteps must be a non-negative safe integer') + } + for (const [name, value] of [ + ['populationSize', options.populationSize], + ['candidateConcurrency', options.candidateConcurrency], + ['sequenceConcurrency', options.sequenceConcurrency], + ['reps', options.reps], + ['maxRecoveryAttempts', options.maxRecoveryAttempts], + ['maxRecoveryRetriesPerAttempt', options.maxRecoveryRetriesPerAttempt], + ] as const) { + if (value !== undefined && (!Number.isSafeInteger(value) || value <= 0)) { + throw new Error(`memory improvement ${name} must be a positive safe integer`) + } + } + if ( + options.maxTotalCostUsd !== undefined && + (!Number.isFinite(options.maxTotalCostUsd) || options.maxTotalCostUsd < 0) + ) { + throw new Error('memory improvement maxTotalCostUsd must be a non-negative finite number') + } + if ( + options.activationTimeoutMs !== undefined && + (!Number.isSafeInteger(options.activationTimeoutMs) || options.activationTimeoutMs <= 0) + ) { + throw new Error('memory improvement activationTimeoutMs must be a positive safe integer') + } + const tolerance = options.criticalDimensionTolerance + if (tolerance !== undefined && (!Number.isFinite(tolerance) || tolerance < 0 || tolerance > 1)) { + throw new Error('memory improvement criticalDimensionTolerance must be between 0 and 1') + } + const minimum = options.minHoldoutScore + if (minimum !== undefined && (!Number.isFinite(minimum) || minimum < 0 || minimum > 1)) { + throw new Error('memory improvement minHoldoutScore must be between 0 and 1') + } + for (const seed of options.seeds) { + if ( + typeof seed.track !== 'string' || + !seed.track.trim() || + typeof seed.proposer !== 'string' || + !seed.proposer.trim() + ) { + throw new Error('memory improvement seeds require non-empty track and proposer values') + } + if (seed.vision !== undefined && (typeof seed.vision !== 'string' || !seed.vision.trim())) { + throw new Error('memory improvement seed vision must be a non-empty string when provided') + } + } + const dimensions = options.criticalDimensions ?? DEFAULT_CRITICAL_DIMENSIONS + if (dimensions.some((dimension) => typeof dimension !== 'string' || !dimension.trim())) { + throw new Error('memory improvement criticalDimensions must contain non-empty names') + } + if (new Set(dimensions).size !== dimensions.length) { + throw new Error('memory improvement criticalDimensions must be unique') + } + assertDistinctSequenceContent(options.trainSequences, 'train') + assertDistinctSequenceContent(options.holdoutSequences, 'holdout') +} + +function assertDistinctSequenceContent( + sequences: readonly AgentMemorySequence[], + split: string, +): void { + const idsByFingerprint = new Map() + for (const sequence of sequences) { + const fingerprint = memorySequenceFingerprint(sequence) + const prior = idsByFingerprint.get(fingerprint) + if (prior) { + throw new Error( + `memory improvement ${split} histories duplicate content: ${prior}/${sequence.id}`, + ) + } + idsByFingerprint.set(fingerprint, sequence.id) + } +} diff --git a/src/memory/types.ts b/src/memory/types.ts index 0a73408..c6d4b9b 100644 --- a/src/memory/types.ts +++ b/src/memory/types.ts @@ -1,5 +1,4 @@ import type { SourceRecord } from '../types' -import type { RetrievalHoldoutConfig } from './holdout' export type AgentMemoryKind = | 'message' @@ -110,3 +109,132 @@ export interface AgentMemoryAdapter { flush?(): Promise close?(): Promise } + +// Randomized retrieval holdout (epsilon-dropout) for per-item treatment-effect logging. +// Default-off: nothing in this module runs unless a consumer passes a RetrievalHoldoutConfig. +// The library never does I/O here; persistence is the consumer's job via onEvent. +// Design + estimator + sample-size analysis: research repo, +// projects/probabilistic-agent-optimization/notes/2026-07-03-DRAFT-o3-holdout-design.md (O3 / EXP-007). + +export interface RetrievalHoldoutConfig { + /** Per-session probability that one eligible watchlist item is suppressed. 0 logs the full schema without ever dropping. */ + epsilon: number + /** Item ids eligible for suppression. Empty or absent means no item can ever be dropped. */ + watchlist?: string[] + /** Ties every event to the exact epsilon/watchlist in force, for audit and replay. */ + configVersion?: string + /** Copied onto every event so multi-adapter logs stay attributable. */ + adapterId?: string + /** Corpus/store version stamp; an edited item under the same id is a different treatment. */ + corpusVersion?: string + /** + * Emit plaintext sessionId and scope on events. Default false: events carry only + * sessionIdHash/scopeHash, so PII-bearing identifiers (tenantId/userId/tags) never reach a + * consumer-controlled sink unless the consumer explicitly owns that decision. Note that + * replaying assignment draws from logs alone needs the plaintext sessionId, so + * privacy-default logs require the consumer's own sessionId mapping for replay audits. + */ + includePlaintextIdentifiers?: boolean + /** + * Cap on tracked sessions per experiment config in the sticky wrapper's registry. + * Exists so tests can exercise eviction; production should keep the default (10,000). + */ + maxTrackedSessions?: number + /** + * Uniform-[0,1) generator keyed by a string. Defaults to a sha256-derived deterministic + * generator so every assignment is replayable from the logged keys alone (design rule D5). + */ + rng?: (key: string) => number + /** Receives one event per retrieval call, INCLUDING no-drop calls: control-arm membership is half the data. */ + onEvent: (event: RetrievalHoldoutEvent) => void +} + +export interface RetrievalHoldoutEligibleItem { + id: string + /** 1-based position in the post-filter hit list. */ + rank: number + score?: number + kind: string + /** sha256(hit.text) prefix; effects are estimated per (id, contentHash) pair. */ + contentHash: string +} + +export interface RetrievalHoldoutEvent { + v: 1 + eventId: string + ts: string + adapterId?: string + /** Plaintext session id — emitted ONLY when config.includePlaintextIdentifiers is true. */ + sessionId?: string + /** Consumer-supplied experiment/outcome join id (scope.tags.taskId); deliberately plaintext. */ + taskId?: string + /** 1-based call counter within the session; 0 when the call is outside session randomization. */ + callIndex: number + /** + * sha256(sessionId) prefix — the default privacy-preserving session join key AND the seed-key + * reference for the assignment draws (previously named rngKey; identical derivation, deduped). + */ + sessionIdHash?: string + queryHash?: string + /** Verbatim scope — emitted ONLY when config.includePlaintextIdentifiers is true (PII risk). */ + scope?: AgentMemoryScope + /** sha256 prefix of the canonical-JSON scope (keys sorted, undefined stripped). */ + scopeHash?: string + config: { epsilon: number; watchlist: string[]; configVersion?: string } + /** + * Value-hash of the experiment-defining knobs, sha256({epsilon, sorted watchlist}) prefix. + * The estimator groups events by it; the sticky-session registry is keyed by it. + */ + configHash: string + /** + * False when no sessionId is available or the adapter answered without retrieval + * (see bypassReason), so the fraction-under-experiment denominator stays honest. + */ + holdoutEligible: boolean + /** Present only on adapter paths that bypassed retrieval, where no suppression could apply. */ + bypassReason?: RetrievalHoldoutBypassReason + /** The full post-filter eligibility set E, logged on every call (control arm + interference probes). */ + eligible: RetrievalHoldoutEligibleItem[] + /** Ids in watchlist ∩ E, in eligibility order. */ + watchlistEligible: string[] + sessionHoldout: boolean + /** The session's sticky drop target once drawn; distinguishes "target absent from E" from "not yet drawn". */ + sessionTargetId: string | null + /** The item suppressed in THIS call, or null. */ + droppedId: string | null + /** 1/|watchlist ∩ E| recorded at draw time; the exact inverse-propensity weight input. */ + pickPropensity: number | null + /** epsilon * pickPropensity, recorded at draw time so analysis never re-derives assignment probabilities. */ + dropPropensity: number | null + deliveredIds: string[] + corpusVersion?: string +} + +export interface RetrievalHoldoutSessionState { + sessionId: string + /** Calls observed so far in this session. */ + callCount: number + sessionHoldout: boolean + /** Sticky drop target; drawn once at the first call whose eligibility set intersects the watchlist. */ + targetId: string | null + pickPropensity: number | null +} + +export interface RetrievalHoldoutCallContext { + sessionId?: string + taskId?: string + /** Raw query; only its sha256 prefix is logged. */ + query?: string + scope?: AgentMemoryScope + /** State returned by the previous call of this session; threading it is what makes suppression sticky. */ + session?: RetrievalHoldoutSessionState +} + +export interface RetrievalHoldoutResult { + delivered: AgentMemoryHit[] + event: RetrievalHoldoutEvent + session?: RetrievalHoldoutSessionState +} + +/** Adapter context paths that answer without retrieval, so no holdout draw can happen. */ +export type RetrievalHoldoutBypassReason = 'short-term-context' | 'raw-string-context' diff --git a/tests/benchmarks.test.ts b/tests/benchmarks.test.ts deleted file mode 100644 index e0a7944..0000000 --- a/tests/benchmarks.test.ts +++ /dev/null @@ -1,1206 +0,0 @@ -import { createRunCostLedger, inMemoryCampaignStorage } from '@tangle-network/agent-eval/campaign' -import { describe, expect, it } from 'vitest' -import { - buildFirstPartyMemoryLifecycleBenchmarkCases, - buildIndustryMemoryBenchmarkSmokeCases, - buildIndustryRagBenchmarkSmokeCases, - buildRetrievalBenchmarkCasesFromQrels, - createInMemoryBenchmarkAdapter, - createNoopMemoryBenchmarkAdapter, - INDUSTRY_MEMORY_BENCHMARKS, - INDUSTRY_RAG_BENCHMARKS, - isKnowledgeMemoryBenchmarkCase, - type KnowledgeAnswerBenchmarkCase, - type KnowledgeMemoryBenchmarkCase, - knowledgeBenchmarkJudge, - parseKnowledgeBenchmarkJsonl, - parseKnowledgeBenchmarkQrels, - type RunMemoryAdapterBenchmarkOptions, - respondToIndustryMemoryBenchmarkSmokeCase, - respondToIndustryRagBenchmarkSmokeCase, - runKnowledgeBenchmarkSuite, - runMemoryAdapterBenchmark as runMemoryAdapterBenchmarkRaw, - scoreKnowledgeBenchmarkArtifact, - scoreMemoryBenchmarkArtifact, -} from '../src/benchmarks/index' -import type { AgentMemoryAdapter, AgentMemoryHit, AgentMemoryScope } from '../src/memory/types' - -function runMemoryAdapterBenchmark(options: RunMemoryAdapterBenchmarkOptions) { - return runMemoryAdapterBenchmarkRaw({ - ...options, - ...(options.storage && !options.controllerMode && !options.acquireRunLease - ? { controllerMode: 'process-local' as const } - : {}), - }) -} - -describe('knowledge benchmark adapters', () => { - it('versions benchmark scoring for resumable cache safety', () => { - expect(knowledgeBenchmarkJudge().judgeVersion).toBe('agent-knowledge:knowledge-benchmark:v2') - }) - - it('requires a responder version for resumable benchmark rows', async () => { - await expect( - runKnowledgeBenchmarkSuite({ - cases: buildIndustryRagBenchmarkSmokeCases().slice(0, 1), - runDir: '/runs/missing-responder-ref', - storage: inMemoryCampaignStorage(), - respond: respondToIndustryRagBenchmarkSmokeCase, - }), - ).rejects.toThrow('respondRef is required when resumable is enabled') - }) - - it('parses qrels/jsonl and builds retrieval cases for public benchmark formats', () => { - expect(parseKnowledgeBenchmarkJsonl('{"id":"q1"}\n{"id":"q2"}\n')).toEqual([ - { id: 'q1' }, - { id: 'q2' }, - ]) - const qrels = parseKnowledgeBenchmarkQrels('q1 0 page-1 1\nq1 0 page-2 0\nq2 page-3 2') - expect(qrels).toEqual([ - { queryId: 'q1', documentId: 'page-1', score: 1 }, - { queryId: 'q1', documentId: 'page-2', score: 0 }, - { queryId: 'q2', documentId: 'page-3', score: 2 }, - ]) - - const cases = buildRetrievalBenchmarkCasesFromQrels({ - benchmarkId: 'beir/smoke', - family: 'beir', - queries: [ - { id: 'q1', text: 'refund policy', split: 'search' }, - { id: 'q2', text: 'shipping speed', split: 'holdout', tags: ['commerce'] }, - ], - qrels, - targetKind: 'page', - }) - - expect(cases).toHaveLength(2) - expect(cases[0]).toMatchObject({ - id: 'beir/smoke:q1', - taskKind: 'retrieval', - split: 'search', - expected: [{ kind: 'page', pageId: 'page-1' }], - }) - expect(cases[1]?.tags).toEqual(['commerce', 'holdout']) - }) - - it('runs retrieval benchmark cases through the campaign-backed suite', async () => { - const storage = inMemoryCampaignStorage() - const cases = buildRetrievalBenchmarkCasesFromQrels({ - benchmarkId: 'beir/suite-smoke', - family: 'beir', - queries: [ - { id: 'q1', text: 'refund policy', split: 'search' }, - { id: 'q2', text: 'shipping speed', split: 'holdout' }, - ], - qrels: [ - { queryId: 'q1', documentId: 'page-1', score: 1 }, - { queryId: 'q2', documentId: 'page-2', score: 1 }, - ], - targetKind: 'page', - k: 2, - }) - const result = await runKnowledgeBenchmarkSuite({ - cases, - runDir: '/runs/knowledge-benchmark-smoke', - storage, - respondRef: 'retriever-fixture:v1', - costCeiling: 1, - respond: async ({ case: testCase, context }) => { - const paid = await context.cost.runPaidCall({ - actor: 'benchmark-retriever', - model: 'retriever-fixture', - maximumCharge: { externallyEnforcedMaximumUsd: 0.01 }, - execute: async () => ({ - costUsd: 0.01, - hits: [ - testCase.id.endsWith('q1') - ? { pageId: 'page-1', path: 'knowledge/page-1.md', rank: 1 } - : { pageId: 'miss', path: 'knowledge/miss.md', rank: 1 }, - ], - }), - receipt: () => ({ - model: 'retriever-fixture', - inputTokens: 0, - outputTokens: 0, - usageUnknown: true, - actualCostUsd: 0.01, - }), - }) - if (!paid.succeeded) throw paid.error - return paid.value - }, - }) - - expect(result.report.totalCases).toBe(2) - expect(result.report.cellsFailed).toBe(0) - expect(result.report.score.mean).toBe(0.5) - expect(result.report.byFamily.beir?.n).toBe(2) - expect(result.report.bySplit.search?.meanScore).toBe(1) - expect(result.report.bySplit.holdout?.meanScore).toBe(0) - expect(result.report.totalCostUsd).toBe(0.02) - expect(storage.read(result.reportJsonPath)).toContain('"totalCases": 2') - expect(storage.read(result.reportMarkdownPath)).toContain('# Knowledge Benchmark Report') - }) - - it('scores RAG answer, hallucination, and KB-improvement cases with claim/source checks', () => { - const testCase: KnowledgeAnswerBenchmarkCase = { - id: 'crag/smoke:q1', - family: 'crag', - taskKind: 'rag-answer', - prompt: 'What is the refund policy?', - requiredClaims: [{ id: 'refund-window', anyOf: ['30 day refund', '30-day refund'] }], - forbiddenClaims: [{ id: 'unsupported-lifetime', anyOf: ['lifetime refund'] }], - expectedSourceIds: ['src-policy', 'src-terms'], - } - - const partial = scoreKnowledgeBenchmarkArtifact(testCase, { - answer: 'The product has a 30-day refund period.', - citedSourceIds: ['src-policy'], - }) - expect(partial.dimensions.claim_recall).toBe(1) - expect(partial.dimensions.citation_recall).toBe(0.5) - expect(partial.dimensions.hallucination_safe).toBe(1) - expect(partial.score).toBeCloseTo(5 / 6) - expect(partial.passed).toBe(false) - - const hallucinated = scoreKnowledgeBenchmarkArtifact( - { ...testCase, taskKind: 'hallucination' }, - { answer: 'The product has a lifetime refund.' }, - ) - expect(hallucinated.dimensions.hallucination_safe).toBe(0) - expect(hallucinated.raw.matchedForbiddenClaimIds).toEqual(['unsupported-lifetime']) - }) - - it('declares every requested industry benchmark family', () => { - const ids = new Set(INDUSTRY_RAG_BENCHMARKS.map((benchmark) => benchmark.id)) - for (const id of [ - 'beir', - 'mteb-retrieval', - 'msmarco', - 'trec-dl', - 'miracl', - 'lotte', - 'bright', - 'crag', - 'hotpotqa', - 'kilt', - 'ragtruth', - 'faithbench', - 'first-party/kb-improvement', - ]) { - expect(ids.has(id)).toBe(true) - } - }) - - it('runs one persisted benchmark cell for every declared industry family', async () => { - const storage = inMemoryCampaignStorage() - const cases = buildIndustryRagBenchmarkSmokeCases() - const result = await runKnowledgeBenchmarkSuite({ - cases, - runDir: '/runs/knowledge-benchmark-family-smoke', - storage, - respondRef: 'industry-rag-smoke:v1', - respond: respondToIndustryRagBenchmarkSmokeCase, - }) - - expect(cases).toHaveLength(INDUSTRY_RAG_BENCHMARKS.length) - expect(result.report.totalCases).toBe(INDUSTRY_RAG_BENCHMARKS.length) - expect(result.report.totalCells).toBe(INDUSTRY_RAG_BENCHMARKS.length) - expect(result.report.cellsFailed).toBe(0) - expect(result.report.score.mean).toBe(1) - expect(result.report.byTaskKind.retrieval?.n).toBe(7) - expect(result.report.byTaskKind['rag-answer']?.n).toBe(3) - expect(result.report.byTaskKind.hallucination?.n).toBe(2) - expect(result.report.byTaskKind['kb-improvement']?.n).toBe(1) - expect(result.report.totalCostUsd).toBe(0) - for (const benchmark of INDUSTRY_RAG_BENCHMARKS) { - expect(result.report.byFamily[benchmark.family]?.n).toBeGreaterThanOrEqual(1) - } - expect(storage.read(result.reportJsonPath)).toContain('"totalCases": 13') - expect(storage.read(result.reportMarkdownPath)).toContain('## Task Kinds') - }) - - it('calibrates memory scoring against stale and current facts', () => { - const testCase: KnowledgeMemoryBenchmarkCase = { - id: 'memora/smoke:q1', - family: 'memora', - taskKind: 'memory-update', - prompt: 'What does the user currently prefer for daily briefings?', - events: [ - { - id: 'e-old', - actorId: 'user', - timestamp: '2026-01-01T00:00:00.000Z', - text: 'The user used to prefer SMS briefings.', - }, - { - id: 'e-new', - actorId: 'user', - timestamp: '2026-02-01T00:00:00.000Z', - text: 'The user now prefers email briefings.', - }, - ], - requiredFacts: [{ id: 'current-channel', anyOf: ['email briefings'] }], - forbiddenFacts: [{ id: 'stale-channel', anyOf: ['SMS briefings'], obsolete: true }], - expectedEventIds: ['e-new'], - expectedActorIds: ['user'], - } - - const strong = scoreMemoryBenchmarkArtifact(testCase, { - answer: 'The user currently prefers email briefings.', - citedEventIds: ['e-new'], - actorIds: ['user'], - }) - const weak = scoreMemoryBenchmarkArtifact(testCase, { - answer: 'The user prefers SMS briefings.', - citedEventIds: ['e-old'], - actorIds: ['user'], - }) - - expect(strong.score).toBeGreaterThanOrEqual(0.7) - expect(strong.passed).toBe(true) - expect(weak.score).toBeLessThanOrEqual(0.3) - expect(weak.passed).toBe(false) - expect(strong.score - weak.score).toBeGreaterThanOrEqual(0.4) - expect(weak.dimensions.memory_stale_safe).toBe(0) - expect(strong.applicableDimensions).toEqual( - expect.arrayContaining([ - 'memory_fact_recall', - 'memory_event_recall', - 'memory_actor_recall', - 'memory_stale_safe', - ]), - ) - - const recallOnly = scoreMemoryBenchmarkArtifact( - { ...testCase, forbiddenFacts: undefined, expectedActorIds: undefined }, - { answer: 'The user currently prefers email briefings.', citedEventIds: ['e-new'] }, - ) - expect(recallOnly.dimensions).not.toHaveProperty('memory_stale_safe') - expect(recallOnly.dimensions).not.toHaveProperty('memory_actor_recall') - expect(recallOnly.applicableDimensions).not.toContain('memory_stale_safe') - }) - - it('runs one persisted benchmark cell for every declared memory benchmark family', async () => { - const storage = inMemoryCampaignStorage() - const cases = buildIndustryMemoryBenchmarkSmokeCases() - const result = await runKnowledgeBenchmarkSuite({ - cases, - runDir: '/runs/memory-benchmark-family-smoke', - storage, - respondRef: 'industry-memory-smoke:v1', - respond: ({ case: testCase }) => { - expect(isKnowledgeMemoryBenchmarkCase(testCase)).toBe(true) - if (!isKnowledgeMemoryBenchmarkCase(testCase)) { - throw new Error(`expected memory case, got ${testCase.taskKind}`) - } - return respondToIndustryMemoryBenchmarkSmokeCase({ - case: testCase, - }) - }, - }) - - expect(cases).toHaveLength(INDUSTRY_MEMORY_BENCHMARKS.length) - expect(result.report.totalCases).toBe(INDUSTRY_MEMORY_BENCHMARKS.length) - expect(result.report.totalCells).toBe(INDUSTRY_MEMORY_BENCHMARKS.length) - expect(result.report.cellsFailed).toBe(0) - expect(result.report.score.mean).toBe(1) - expect(result.report.byTaskKind['memory-ingest']?.n).toBe(1) - expect(result.report.byTaskKind['memory-recall']?.n).toBe(1) - expect(result.report.byTaskKind['memory-temporal']?.n).toBe(1) - expect(result.report.byTaskKind['memory-update']?.n).toBe(1) - expect(result.report.byTaskKind['memory-forgetting']?.n).toBe(1) - expect(result.report.byTaskKind['memory-reasoning']?.n).toBe(1) - expect(result.report.byTaskKind['memory-summarization']?.n).toBe(1) - expect(result.report.byTaskKind['memory-recommendation']?.n).toBe(1) - expect(result.report.byTaskKind['memory-multiparty']?.n).toBe(1) - expect(result.report.totalCostUsd).toBe(0) - for (const benchmark of INDUSTRY_MEMORY_BENCHMARKS) { - expect(result.report.byFamily[benchmark.family]?.n).toBeGreaterThanOrEqual(1) - } - expect(storage.read(result.reportJsonPath)).toContain('"totalCases": 9') - expect(storage.read(result.reportMarkdownPath)).toContain('memory_stale_safe') - }) - - it('ranks actual memory adapters on the first-party lifecycle benchmark', async () => { - const storage = inMemoryCampaignStorage() - const cases = buildFirstPartyMemoryLifecycleBenchmarkCases() - const result = await runMemoryAdapterBenchmark({ - cases, - runDir: '/runs/memory-adapter-ranking', - storage, - candidates: [ - { - id: 'no-memory', - ref: 'no-memory:v1', - createAdapter: () => createNoopMemoryBenchmarkAdapter(), - }, - { - id: 'in-memory', - ref: 'in-memory:v1', - createAdapter: () => createInMemoryBenchmarkAdapter(), - searchLimit: 1, - }, - ], - }) - - expect(cases).toHaveLength(12) - expect(result.rows).toHaveLength(2) - expect(result.rows[0]?.candidateId).toBe('in-memory') - expect(result.rows[0]?.scoreMean).toBeGreaterThan(0.9) - expect(result.rows[0]?.totalCells).toBe(12) - expect(result.rows[0]?.cellsFailed).toBe(0) - expect(result.rows[1]?.candidateId).toBe('no-memory') - expect(result.rows[1]?.scoreMean).toBeLessThan(0.3) - expect(storage.read(result.rankingJsonPath)).toContain('"candidateId": "in-memory"') - expect(storage.read(result.rankingMarkdownPath)).toContain('| 1 | in-memory |') - expect(storage.read(result.rows[0]!.reportJsonPath)).toContain('"memory_stale_safe"') - }) - - it('requires an explicit controller policy for custom benchmark storage', async () => { - await expect( - runMemoryAdapterBenchmarkRaw({ - cases: buildFirstPartyMemoryLifecycleBenchmarkCases().slice(0, 1), - runDir: '/runs/custom-benchmark-storage', - storage: inMemoryCampaignStorage(), - candidates: [ - { - id: 'in-memory', - ref: 'in-memory:v1', - createAdapter: () => createInMemoryBenchmarkAdapter(), - }, - ], - }), - ).rejects.toThrow("requires acquireRunLease or controllerMode='process-local'") - }) - - it('recovers an unfinished provider scope before retrying a direct benchmark cell', async () => { - const storage = inMemoryCampaignStorage() - const providerRows = new Map() - const operations: string[] = [] - const purposes: string[] = [] - let firstExecution = true - const candidate = { - id: 'recoverable', - ref: 'recoverable:v1', - adapterId: 'recoverable-provider', - recoveryCostUsdPerAttempt: 0.1, - createAdapter({ purpose }: { purpose: 'execute' | 'recovery' }) { - purposes.push(purpose) - const label = purpose === 'recovery' ? 'recovery' : firstExecution ? 'first' : 'retry' - if (purpose === 'execute') firstExecution = false - const adapter: AgentMemoryAdapter = { - id: 'recoverable-provider', - branchIsolation: { mode: 'scoped' }, - async search(_query, options) { - return [...(providerRows.get(options?.scope?.namespace ?? '') ?? [])] - }, - async getContext(query, options) { - const hits = await adapter.search(query, options) - return { query, text: hits.map((hit) => hit.text).join('\n'), hits, sourceRecords: [] } - }, - async write(input) { - operations.push(`write:${label}`) - const namespace = input.scope?.namespace ?? '' - const hit: AgentMemoryHit = { - id: input.id ?? `${label}:memory`, - uri: `memory://recoverable/${label}`, - kind: input.kind, - text: input.text, - metadata: input.metadata, - } - providerRows.set(namespace, [...(providerRows.get(namespace) ?? []), hit]) - return { accepted: true, id: hit.id, uri: hit.uri, kind: hit.kind } - }, - async clear(scope) { - operations.push(`clear:${label}`) - if (label === 'first') throw new Error('provider cleanup unavailable') - providerRows.delete(scope?.namespace ?? '') - }, - async close() {}, - } - return adapter - }, - } - const run = () => - runMemoryAdapterBenchmark({ - cases: buildFirstPartyMemoryLifecycleBenchmarkCases().slice(0, 1), - runDir: '/runs/direct-benchmark-recovery', - storage, - candidates: [candidate], - costCeiling: 1, - }) - - await expect(run()).rejects.toThrow('memory benchmark attempt cleanup failed') - expect(providerRows.size).toBe(1) - - const result = await run() - expect(result.rows[0]).toMatchObject({ candidateId: 'recoverable', cellsFailed: 0 }) - expect(result.rows[0]?.totalCostUsd).toBe(0.1) - expect(result.totalCostUsd).toBe(0.1) - expect(purposes).toEqual(['execute', 'recovery', 'execute']) - expect(operations.indexOf('clear:recovery')).toBeLessThan(operations.indexOf('write:retry')) - expect(providerRows.size).toBe(0) - const events = storage - .read(result.attemptLogPath)! - .trim() - .split('\n') - .map((line) => JSON.parse(line) as { status: string; recovery: boolean }) - expect(events.map(({ status, recovery }) => ({ status, recovery }))).toEqual([ - { status: 'started', recovery: false }, - { status: 'cleaned', recovery: true }, - { status: 'started', recovery: false }, - { status: 'cleaned', recovery: false }, - ]) - }) - - it('includes paid cleanup for a retired candidate in the benchmark total', async () => { - const storage = inMemoryCampaignStorage() - const runDir = '/runs/retired-candidate-recovery-cost' - storage.write( - `${runDir}/memory-adapter-attempts.jsonl`, - `${JSON.stringify({ - schema: 3, - status: 'started', - attemptId: 'retired-attempt', - candidateId: 'retired', - candidateRef: 'retired:v1', - adapterId: 'retired-provider', - caseId: 'old-case', - cellId: 'old-cell', - scope: { namespace: 'retired-scope' }, - adapterCreationCostUsd: 0.05, - costUsdPerCase: 0, - recoveryCostUsdPerAttempt: 0.1, - recordedAt: '2026-01-01T00:00:00.000Z', - recovery: false, - })}\n`, - ) - let retiredClears = 0 - const result = await runMemoryAdapterBenchmark({ - cases: buildFirstPartyMemoryLifecycleBenchmarkCases().slice(0, 1), - runDir, - storage, - costCeiling: 1, - candidates: [ - { - id: 'active', - ref: 'active:v1', - adapterId: 'active-provider', - createAdapter: () => createInMemoryBenchmarkAdapter({ id: 'active-provider' }), - }, - ], - recoveryCandidates: [ - { - id: 'retired', - ref: 'retired:v1', - adapterId: 'retired-provider', - adapterCreationCostUsd: 0.05, - recoveryCostUsdPerAttempt: 0.1, - createAdapter({ markExternalCall }) { - markExternalCall() - const adapter = createInMemoryBenchmarkAdapter({ id: 'retired-provider' }) - adapter.clear = async () => { - retiredClears += 1 - } - return adapter - }, - }, - ], - }) - - expect(retiredClears).toBe(1) - expect(result.rows[0]).toMatchObject({ candidateId: 'active', totalCostUsd: 0 }) - expect(result).toMatchObject({ totalCostUsd: 0.15, unrankedRecoveryCostUsd: 0.15 }) - expect(storage.read(result.rankingJsonPath)).toContain('"unrankedRecoveryCostUsd": 0.15') - }) - - it('refuses benchmark recovery when candidate cost settings changed', async () => { - const storage = inMemoryCampaignStorage() - const runDir = '/runs/benchmark-changed-recovery-costs' - storage.write( - `${runDir}/memory-adapter-attempts.jsonl`, - `${JSON.stringify({ - schema: 3, - status: 'started', - attemptId: 'unfinished-attempt', - candidateId: 'memory', - candidateRef: 'memory:v1', - adapterId: 'memory-provider', - caseId: 'old-case', - cellId: 'old-cell', - scope: { namespace: 'unfinished-scope' }, - adapterCreationCostUsd: 0, - costUsdPerCase: 0, - recoveryCostUsdPerAttempt: 0, - recordedAt: '2026-01-01T00:00:00.000Z', - recovery: false, - })}\n`, - ) - let adapterCreates = 0 - - await expect( - runMemoryAdapterBenchmark({ - cases: buildFirstPartyMemoryLifecycleBenchmarkCases().slice(0, 1), - runDir, - storage, - costCeiling: 1, - candidates: [ - { - id: 'memory', - ref: 'memory:v1', - adapterId: 'memory-provider', - costUsdPerCase: 0.1, - createAdapter() { - adapterCreates += 1 - return createInMemoryBenchmarkAdapter({ id: 'memory-provider' }) - }, - }, - ], - }), - ).rejects.toThrow('candidate cost settings changed') - - expect(adapterCreates).toBe(0) - expect( - storage.read(`${runDir}/memory-adapter-attempts.jsonl`)?.trim().split('\n'), - ).toHaveLength(1) - }) - - it('bounds repeated direct benchmark recovery across process restarts', async () => { - const storage = inMemoryCampaignStorage() - const runDir = '/runs/benchmark-recovery-retry-limit' - storage.write( - `${runDir}/memory-adapter-attempts.jsonl`, - `${JSON.stringify({ - schema: 3, - status: 'started', - attemptId: 'unfinished-attempt', - candidateId: 'memory', - candidateRef: 'memory:v1', - adapterId: 'memory-provider', - caseId: 'old-case', - cellId: 'old-cell', - scope: { namespace: 'unfinished-scope' }, - adapterCreationCostUsd: 0, - costUsdPerCase: 0, - recoveryCostUsdPerAttempt: 0, - recordedAt: '2026-01-01T00:00:00.000Z', - recovery: false, - })}\n`, - ) - let recoveryCreates = 0 - const run = () => - runMemoryAdapterBenchmark({ - cases: buildFirstPartyMemoryLifecycleBenchmarkCases().slice(0, 1), - runDir, - storage, - maxRecoveryRetriesPerAttempt: 2, - candidates: [ - { - id: 'memory', - ref: 'memory:v1', - adapterId: 'memory-provider', - createAdapter({ purpose }) { - if (purpose === 'recovery') recoveryCreates += 1 - throw new Error('provider recovery unavailable') - }, - }, - ], - }) - - await expect(run()).rejects.toThrow('provider recovery unavailable') - await expect(run()).rejects.toThrow('provider recovery unavailable') - await expect(run()).rejects.toThrow('exhausted 2 recovery attempts') - expect(recoveryCreates).toBe(2) - expect( - storage.read(`${runDir}/memory-adapter-recovery-attempts.jsonl`)?.trim().split('\n'), - ).toHaveLength(2) - }) - - it('closes a recovery adapter that arrives after its factory timeout', async () => { - const storage = inMemoryCampaignStorage() - const runDir = '/runs/late-benchmark-recovery-adapter' - storage.write( - `${runDir}/memory-adapter-attempts.jsonl`, - `${JSON.stringify({ - schema: 3, - status: 'started', - attemptId: 'unfinished-attempt', - candidateId: 'memory', - candidateRef: 'memory:v1', - adapterId: 'memory-provider', - caseId: 'old-case', - cellId: 'old-cell', - scope: { namespace: 'unfinished-scope' }, - adapterCreationCostUsd: 0, - costUsdPerCase: 0, - recoveryCostUsdPerAttempt: 0, - recordedAt: '2026-01-01T00:00:00.000Z', - recovery: false, - })}\n`, - ) - let resolveCreation!: (adapter: AgentMemoryAdapter) => void - const creation = new Promise((resolve) => { - resolveCreation = resolve - }) - let reportClosed!: () => void - const closed = new Promise((resolve) => { - reportClosed = resolve - }) - let closeCalls = 0 - const lateAdapter = createInMemoryBenchmarkAdapter({ id: 'memory-provider' }) - lateAdapter.close = async () => { - closeCalls += 1 - reportClosed() - } - - await expect( - runMemoryAdapterBenchmark({ - cases: buildFirstPartyMemoryLifecycleBenchmarkCases().slice(0, 1), - runDir, - storage, - cleanupTimeoutMs: 10, - candidates: [ - { - id: 'memory', - ref: 'memory:v1', - adapterId: 'memory-provider', - createAdapter: ({ purpose }) => - purpose === 'recovery' ? creation : createInMemoryBenchmarkAdapter(), - }, - ], - }), - ).rejects.toThrow('benchmark recovery adapter creation did not finish within 10ms') - - resolveCreation(lateAdapter) - await closed - expect(closeCalls).toBe(1) - }) - - it('reconciles a crash after direct recovery but before its cost receipt', async () => { - const storage = inMemoryCampaignStorage() - const append = storage.append!.bind(storage) - let failRecoveryReceipt = false - storage.append = (path, value, expectedBytes) => { - if ( - failRecoveryReceipt && - path.endsWith('/cost-ledger.jsonl') && - value.includes('"status":"settled"') && - value.includes('memory-adapter-recovery') - ) { - failRecoveryReceipt = false - throw new Error('simulated process exit before benchmark recovery receipt') - } - return append(path, value, expectedBytes) - } - - let firstExecution = true - let recoveryClears = 0 - const candidate = { - id: 'receipt-crash', - ref: 'receipt-crash:v1', - adapterId: 'receipt-crash-provider', - costUsdPerCase: 0.1, - recoveryCostUsdPerAttempt: 0.1, - createAdapter({ purpose }: { purpose: 'execute' | 'recovery' }) { - const adapter = createInMemoryBenchmarkAdapter({ id: 'receipt-crash-provider' }) - const clear = adapter.clear! - const failThisExecution = purpose === 'execute' && firstExecution - if (purpose === 'execute') firstExecution = false - adapter.clear = async (scope) => { - if (purpose === 'recovery') recoveryClears += 1 - if (failThisExecution) throw new Error('leave direct benchmark state active') - await clear(scope) - } - return adapter - }, - } - const run = () => - runMemoryAdapterBenchmark({ - cases: buildFirstPartyMemoryLifecycleBenchmarkCases().slice(0, 1), - runDir: '/runs/direct-recovery-receipt-crash', - storage, - costCeiling: 1, - candidates: [candidate], - }) - - await expect(run()).rejects.toThrow('memory benchmark attempt cleanup failed') - failRecoveryReceipt = true - await expect(run()).rejects.toThrow('failed to persist') - expect( - storage - .read('/runs/direct-recovery-receipt-crash/memory-adapter-attempts.jsonl') - ?.trim() - .split('\n'), - ).toHaveLength(2) - - const result = await run() - const costLedger = createRunCostLedger({ - storage, - runDir: '/runs/direct-recovery-receipt-crash', - costCeilingUsd: 1, - }) - - expect(result.rows[0]).toMatchObject({ - candidateId: 'receipt-crash', - cellsFailed: 0, - }) - expect(result.rows[0]?.totalCostUsd).toBeCloseTo(0.3) - expect(result.totalCostUsd).toBeCloseTo(0.3) - expect(recoveryClears).toBe(1) - expect(costLedger.summary()).toMatchObject({ unresolvedCalls: 0, accountingComplete: true }) - expect(costLedger.summary().totalCostUsd).toBeCloseTo(0.3) - }) - - it('bounds direct benchmark cleanup and preserves its recovery record', async () => { - const storage = inMemoryCampaignStorage() - const adapter = createInMemoryBenchmarkAdapter({ id: 'hung-cleanup' }) - let closeCalls = 0 - adapter.clear = () => new Promise(() => {}) - adapter.close = async () => { - closeCalls += 1 - } - const startedAt = Date.now() - - await expect( - runMemoryAdapterBenchmark({ - cases: buildFirstPartyMemoryLifecycleBenchmarkCases().slice(0, 1), - runDir: '/runs/direct-benchmark-cleanup-timeout', - storage, - cleanupTimeoutMs: 10, - candidates: [ - { - id: 'hung-cleanup', - ref: 'hung-cleanup:v1', - createAdapter: () => adapter, - }, - ], - }), - ).rejects.toThrow('memory adapter benchmark cleanup failed') - - expect(Date.now() - startedAt).toBeLessThan(500) - expect( - storage - .read('/runs/direct-benchmark-cleanup-timeout/memory-adapter-attempts.jsonl') - ?.trim() - .split('\n'), - ).toHaveLength(1) - expect(closeCalls).toBe(0) - }) - - it('counts billable adapter creation in the shared benchmark budget', async () => { - const storage = inMemoryCampaignStorage() - let creates = 0 - const result = await runMemoryAdapterBenchmark({ - cases: buildFirstPartyMemoryLifecycleBenchmarkCases().slice(0, 1), - runDir: '/runs/paid-adapter-creation', - storage, - costCeiling: 0.5, - candidates: [ - { - id: 'paid-creation', - ref: 'paid-creation:v1', - adapterId: 'paid-creation-provider', - adapterCreationCostUsd: 0.2, - createAdapter({ markExternalCall }) { - markExternalCall() - creates += 1 - return createInMemoryBenchmarkAdapter({ id: 'paid-creation-provider' }) - }, - }, - ], - }) - - const ledger = createRunCostLedger({ - storage, - runDir: '/runs/paid-adapter-creation', - costCeilingUsd: 0.5, - }) - expect(creates).toBe(1) - expect(result.rows[0]).toMatchObject({ candidateId: 'paid-creation', totalCostUsd: 0.2 }) - expect(result.totalCostUsd).toBe(0.2) - expect(ledger.list()).toMatchObject([ - { - actor: 'agent-knowledge:memory-adapter:paid-creation', - costUsd: 0.2, - tags: { - candidateId: 'paid-creation', - memoryAdapterCreation: 'execute', - runDir: '/runs/paid-adapter-creation/paid-creation', - }, - }, - ]) - }) - - it('does not recreate or recharge a billable adapter when every cell resumes', async () => { - const storage = inMemoryCampaignStorage() - let creates = 0 - const run = () => - runMemoryAdapterBenchmark({ - cases: buildFirstPartyMemoryLifecycleBenchmarkCases().slice(0, 1), - runDir: '/runs/resumed-paid-adapter-creation', - storage, - costCeiling: 0.5, - candidates: [ - { - id: 'paid-resume', - ref: 'paid-resume:v1', - adapterId: 'paid-resume-provider', - adapterCreationCostUsd: 0.2, - createAdapter({ markExternalCall }) { - markExternalCall() - creates += 1 - return createInMemoryBenchmarkAdapter({ id: 'paid-resume-provider' }) - }, - }, - ], - }) - - const initial = await run() - const resumed = await run() - - expect(creates).toBe(1) - expect(initial).toMatchObject({ totalCostUsd: 0.2 }) - expect(resumed).toMatchObject({ totalCostUsd: 0.2 }) - expect(resumed.rows[0]).toMatchObject({ adapterId: 'paid-resume-provider' }) - expect(resumed.rows[0]?.report.cellsCached).toBe(1) - }) - - it('closes a mismatched lazy adapter before any benchmark case writes', async () => { - const storage = inMemoryCampaignStorage() - let closes = 0 - let writes = 0 - const adapter = createInMemoryBenchmarkAdapter({ id: 'actual-provider' }) - const write = adapter.write.bind(adapter) - adapter.write = async (input) => { - writes += 1 - return write(input) - } - adapter.close = async () => { - closes += 1 - } - - await expect( - runMemoryAdapterBenchmark({ - cases: buildFirstPartyMemoryLifecycleBenchmarkCases().slice(0, 1), - runDir: '/runs/mismatched-lazy-adapter', - storage, - candidates: [ - { - id: 'candidate', - ref: 'candidate:v1', - adapterId: 'expected-provider', - createAdapter: () => adapter, - }, - ], - }), - ).rejects.toThrow("returned id 'actual-provider', expected 'expected-provider'") - expect({ closes, writes }).toEqual({ closes: 1, writes: 0 }) - expect( - storage.read('/runs/mismatched-lazy-adapter/memory-adapter-attempts.jsonl'), - ).toBeUndefined() - }) - - it('aborts execute adapter creation at the configured timeout', async () => { - const storage = inMemoryCampaignStorage() - let aborted = false - const startedAt = Date.now() - - await expect( - runMemoryAdapterBenchmark({ - cases: buildFirstPartyMemoryLifecycleBenchmarkCases().slice(0, 1), - runDir: '/runs/timed-out-adapter-creation', - storage, - cleanupTimeoutMs: 10, - candidates: [ - { - id: 'hung-factory', - ref: 'hung-factory:v1', - createAdapter: ({ signal }) => - new Promise((_resolve, reject) => { - signal.addEventListener( - 'abort', - () => { - aborted = true - reject(signal.reason) - }, - { once: true }, - ) - }), - }, - ], - }), - ).rejects.toThrow('benchmark execute adapter creation did not finish within 10ms') - expect(aborted).toBe(true) - expect(Date.now() - startedAt).toBeLessThan(500) - expect( - storage.read('/runs/timed-out-adapter-creation/memory-adapter-attempts.jsonl'), - ).toBeUndefined() - }) - - it('does not charge a local adapter factory failure', async () => { - const storage = inMemoryCampaignStorage() - await expect( - runMemoryAdapterBenchmark({ - cases: buildFirstPartyMemoryLifecycleBenchmarkCases().slice(0, 1), - runDir: '/runs/local-adapter-creation-failure', - storage, - costCeiling: 0.5, - candidates: [ - { - id: 'local-failure', - ref: 'local-failure:v1', - adapterCreationCostUsd: 0.2, - createAdapter() { - throw new Error('invalid local configuration') - }, - }, - ], - }), - ).rejects.toThrow('invalid local configuration') - - const ledger = createRunCostLedger({ - storage, - runDir: '/runs/local-adapter-creation-failure', - costCeilingUsd: 0.5, - }) - expect(ledger.summary()).toMatchObject({ totalCostUsd: 0, unresolvedCalls: 0 }) - }) - - it('enforces one cost ceiling across every compared adapter', async () => { - const result = await runMemoryAdapterBenchmark({ - cases: buildFirstPartyMemoryLifecycleBenchmarkCases().slice(0, 1), - runDir: '/runs/shared-adapter-cost-ceiling', - storage: inMemoryCampaignStorage(), - costCeiling: 0.75, - candidates: ['first', 'second'].map((id) => ({ - id, - ref: `${id}:v1`, - costUsdPerCase: 0.5, - createAdapter: () => createInMemoryBenchmarkAdapter({ id }), - })), - }) - - expect(result.totalCostUsd).toBe(0.5) - expect(result.rows).toHaveLength(2) - expect(result.rows.find((row) => row.candidateId === 'first')).toMatchObject({ - cellsFailed: 0, - totalCostUsd: 0.5, - }) - expect(result.rows.find((row) => row.candidateId === 'second')).toMatchObject({ - cellsFailed: 1, - totalCostUsd: 0, - }) - }) - - it('runs sequential paid adapter cases after exact external-cost receipts', async () => { - const result = await runMemoryAdapterBenchmark({ - cases: buildFirstPartyMemoryLifecycleBenchmarkCases().slice(0, 2), - runDir: '/runs/sequential-adapter-cost-accounting', - storage: inMemoryCampaignStorage(), - costCeiling: 0.2, - maxConcurrency: 1, - candidates: [ - { - id: 'sequential-paid', - ref: 'sequential-paid:v1', - costUsdPerCase: 0.1, - createAdapter: () => createInMemoryBenchmarkAdapter({ id: 'sequential-paid' }), - }, - ], - }) - - expect(result.rows[0]).toMatchObject({ cellsFailed: 0, totalCostUsd: 0.2 }) - expect(result.totalCostUsd).toBe(0.2) - }) - - it('refuses paid adapter calls when no dollar limit is configured', async () => { - let providerCalls = 0 - const adapter = createInMemoryBenchmarkAdapter({ id: 'paid-by-default' }) - const write = adapter.write.bind(adapter) - adapter.write = async (input) => { - providerCalls += 1 - return write(input) - } - - const result = await runMemoryAdapterBenchmark({ - cases: buildFirstPartyMemoryLifecycleBenchmarkCases().slice(0, 1), - runDir: '/runs/default-zero-cost-ceiling', - storage: inMemoryCampaignStorage(), - candidates: [ - { - id: 'paid-by-default', - ref: 'paid-by-default:v1', - costUsdPerCase: 0.01, - createAdapter: () => adapter, - }, - ], - }) - - expect(result.rows[0]).toMatchObject({ cellsFailed: 1, totalCostUsd: 0 }) - expect(providerCalls).toBe(0) - }) - - it('does not reuse resumable rows when adapter benchmark options change', async () => { - const storage = inMemoryCampaignStorage() - const run = (searchLimit: number) => - runMemoryAdapterBenchmark({ - cases: buildFirstPartyMemoryLifecycleBenchmarkCases().slice(0, 1), - runDir: '/runs/memory-adapter-cache-identity', - storage, - candidates: [ - { - id: 'in-memory', - ref: 'in-memory:v1', - createAdapter: () => createInMemoryBenchmarkAdapter(), - searchLimit, - }, - ], - }) - - const initial = await run(1) - const changed = await run(2) - const repeated = await run(2) - - expect(initial.rows[0]?.report.cellsCached).toBe(0) - expect(changed.rows[0]?.report.cellsCached).toBe(0) - expect(repeated.rows[0]?.report.cellsCached).toBe(1) - }) - - it('isolates and clears every concurrent adapter repetition', async () => { - const adapter = createInMemoryBenchmarkAdapter() - const clearedScopes: AgentMemoryScope[] = [] - const clear = adapter.clear! - adapter.clear = async (scope) => { - clearedScopes.push(scope ?? {}) - await clear(scope) - } - - const result = await runMemoryAdapterBenchmark({ - cases: buildFirstPartyMemoryLifecycleBenchmarkCases().slice(0, 1), - runDir: '/runs/isolated-memory-repetitions', - storage: inMemoryCampaignStorage(), - reps: 3, - maxConcurrency: 3, - candidates: [ - { - id: 'in-memory', - ref: 'in-memory:v1', - createAdapter: () => adapter, - }, - ], - }) - - expect(result.rows[0]?.totalCells).toBe(3) - expect(result.rows[0]?.cellsFailed).toBe(0) - expect(clearedScopes).toHaveLength(3) - expect(new Set(clearedScopes.map((scope) => scope.namespace)).size).toBe(3) - expect(new Set(clearedScopes.map((scope) => scope.tags?.benchmarkAttemptId)).size).toBe(3) - }) - - it('settles timed-out provider work before clearing and closing the adapter', async () => { - let closed = false - let writes = 0 - let writesAfterClose = 0 - let clears = 0 - let clearsAfterClose = 0 - let contextReads = 0 - const adapter: AgentMemoryAdapter = { - id: 'delayed-provider', - branchIsolation: { mode: 'scoped' }, - async search() { - return [] - }, - async getContext(query) { - contextReads += 1 - return { query, text: '', hits: [], sourceRecords: [] } - }, - async write(input) { - await new Promise((resolve) => setTimeout(resolve, 25)) - writes += 1 - if (closed) writesAfterClose += 1 - return { - accepted: true, - id: input.id ?? String(writes), - uri: `memory://delayed-provider/${writes}`, - kind: input.kind, - } - }, - async clear() { - await new Promise((resolve) => setTimeout(resolve, 1)) - clears += 1 - if (closed) clearsAfterClose += 1 - }, - async close() { - closed = true - }, - } - - const result = await runMemoryAdapterBenchmark({ - cases: buildFirstPartyMemoryLifecycleBenchmarkCases().slice(0, 1), - runDir: '/runs/timed-out-memory-provider', - storage: inMemoryCampaignStorage(), - dispatchTimeoutMs: 5, - candidates: [ - { - id: 'delayed-provider', - ref: 'delayed-provider:v1', - createAdapter: () => adapter, - }, - { - id: 'no-memory', - ref: 'no-memory:v1', - createAdapter: () => createNoopMemoryBenchmarkAdapter(), - }, - ], - }) - - expect(result.rows[0]).toMatchObject({ candidateId: 'no-memory', cellsFailed: 0 }) - expect(result.rows[1]).toMatchObject({ candidateId: 'delayed-provider', cellsFailed: 1 }) - expect({ closed, writes, writesAfterClose, clears, clearsAfterClose, contextReads }).toEqual({ - closed: true, - writes: 1, - writesAfterClose: 0, - clears: 1, - clearsAfterClose: 0, - contextReads: 0, - }) - await new Promise((resolve) => setTimeout(resolve, 50)) - expect({ writes, writesAfterClose, clears, clearsAfterClose }).toEqual({ - writes: 1, - writesAfterClose: 0, - clears: 1, - clearsAfterClose: 0, - }) - }) - - it('rejects every unsafe candidate id before creating any adapter', async () => { - let creates = 0 - await expect( - runMemoryAdapterBenchmark({ - cases: buildFirstPartyMemoryLifecycleBenchmarkCases().slice(0, 1), - runDir: '/runs/invalid-memory-candidate', - storage: inMemoryCampaignStorage(), - candidates: [ - { - id: 'valid', - ref: 'valid:v1', - createAdapter() { - creates += 1 - return createInMemoryBenchmarkAdapter() - }, - }, - { - id: '../invalid', - ref: 'invalid:v1', - createAdapter() { - creates += 1 - return createInMemoryBenchmarkAdapter() - }, - }, - ], - }), - ).rejects.toThrow('must be a safe directory segment') - expect(creates).toBe(0) - }) -}) diff --git a/tests/benchmarks/catalog.test.ts b/tests/benchmarks/catalog.test.ts new file mode 100644 index 0000000..2cbf279 --- /dev/null +++ b/tests/benchmarks/catalog.test.ts @@ -0,0 +1,309 @@ +import { inMemoryCampaignStorage } from '@tangle-network/agent-eval/campaign' +import { describe, expect, it } from 'vitest' +import { + buildIndustryMemoryBenchmarkSmokeCases, + buildIndustryRagBenchmarkSmokeCases, + buildRetrievalBenchmarkCasesFromQrels, + INDUSTRY_MEMORY_BENCHMARKS, + INDUSTRY_RAG_BENCHMARKS, + isKnowledgeMemoryBenchmarkCase, + type KnowledgeAnswerBenchmarkCase, + type KnowledgeMemoryBenchmarkCase, + knowledgeBenchmarkJudge, + parseKnowledgeBenchmarkJsonl, + parseKnowledgeBenchmarkQrels, + respondToIndustryMemoryBenchmarkSmokeCase, + respondToIndustryRagBenchmarkSmokeCase, + runKnowledgeBenchmarkSuite, + scoreKnowledgeBenchmarkArtifact, + scoreMemoryBenchmarkArtifact, +} from '../../src/benchmarks/index' + +describe('knowledge benchmark catalog and scoring', () => { + it('versions benchmark scoring for resumable cache safety', () => { + expect(knowledgeBenchmarkJudge().judgeVersion).toBe('agent-knowledge:knowledge-benchmark:v2') + }) + + it('requires a responder version for resumable benchmark rows', async () => { + await expect( + runKnowledgeBenchmarkSuite({ + cases: buildIndustryRagBenchmarkSmokeCases().slice(0, 1), + runDir: '/runs/missing-responder-ref', + storage: inMemoryCampaignStorage(), + respond: respondToIndustryRagBenchmarkSmokeCase, + }), + ).rejects.toThrow('respondRef is required when resumable is enabled') + }) + + it('parses qrels/jsonl and builds retrieval cases for public benchmark formats', () => { + expect(parseKnowledgeBenchmarkJsonl('{"id":"q1"}\n{"id":"q2"}\n')).toEqual([ + { id: 'q1' }, + { id: 'q2' }, + ]) + const qrels = parseKnowledgeBenchmarkQrels('q1 0 page-1 1\nq1 0 page-2 0\nq2 page-3 2') + expect(qrels).toEqual([ + { queryId: 'q1', documentId: 'page-1', score: 1 }, + { queryId: 'q1', documentId: 'page-2', score: 0 }, + { queryId: 'q2', documentId: 'page-3', score: 2 }, + ]) + + const cases = buildRetrievalBenchmarkCasesFromQrels({ + benchmarkId: 'beir/smoke', + family: 'beir', + queries: [ + { id: 'q1', text: 'refund policy', split: 'search' }, + { id: 'q2', text: 'shipping speed', split: 'holdout', tags: ['commerce'] }, + ], + qrels, + targetKind: 'page', + }) + + expect(cases).toHaveLength(2) + expect(cases[0]).toMatchObject({ + id: 'beir/smoke:q1', + taskKind: 'retrieval', + split: 'search', + expected: [{ kind: 'page', pageId: 'page-1' }], + }) + expect(cases[1]?.tags).toEqual(['commerce', 'holdout']) + }) + + it('runs retrieval benchmark cases through the campaign-backed suite', async () => { + const storage = inMemoryCampaignStorage() + const cases = buildRetrievalBenchmarkCasesFromQrels({ + benchmarkId: 'beir/suite-smoke', + family: 'beir', + queries: [ + { id: 'q1', text: 'refund policy', split: 'search' }, + { id: 'q2', text: 'shipping speed', split: 'holdout' }, + ], + qrels: [ + { queryId: 'q1', documentId: 'page-1', score: 1 }, + { queryId: 'q2', documentId: 'page-2', score: 1 }, + ], + targetKind: 'page', + k: 2, + }) + const result = await runKnowledgeBenchmarkSuite({ + cases, + runDir: '/runs/knowledge-benchmark-smoke', + storage, + respondRef: 'retriever-fixture:v1', + costCeiling: 1, + respond: async ({ case: testCase, context }) => { + const paid = await context.cost.runPaidCall({ + actor: 'benchmark-retriever', + model: 'retriever-fixture', + maximumCharge: { externallyEnforcedMaximumUsd: 0.01 }, + execute: async () => ({ + costUsd: 0.01, + hits: [ + testCase.id.endsWith('q1') + ? { pageId: 'page-1', path: 'knowledge/page-1.md', rank: 1 } + : { pageId: 'miss', path: 'knowledge/miss.md', rank: 1 }, + ], + }), + receipt: () => ({ + model: 'retriever-fixture', + inputTokens: 0, + outputTokens: 0, + usageUnknown: true, + actualCostUsd: 0.01, + }), + }) + if (!paid.succeeded) throw paid.error + return paid.value + }, + }) + + expect(result.report.totalCases).toBe(2) + expect(result.report.cellsFailed).toBe(0) + expect(result.report.score.mean).toBe(0.5) + expect(result.report.byFamily.beir?.n).toBe(2) + expect(result.report.bySplit.search?.meanScore).toBe(1) + expect(result.report.bySplit.holdout?.meanScore).toBe(0) + expect(result.report.totalCostUsd).toBe(0.02) + expect(storage.read(result.reportJsonPath)).toContain('"totalCases": 2') + expect(storage.read(result.reportMarkdownPath)).toContain('# Knowledge Benchmark Report') + }) + + it('scores RAG answer, hallucination, and KB-improvement cases with claim/source checks', () => { + const testCase: KnowledgeAnswerBenchmarkCase = { + id: 'crag/smoke:q1', + family: 'crag', + taskKind: 'rag-answer', + prompt: 'What is the refund policy?', + requiredClaims: [{ id: 'refund-window', anyOf: ['30 day refund', '30-day refund'] }], + forbiddenClaims: [{ id: 'unsupported-lifetime', anyOf: ['lifetime refund'] }], + expectedSourceIds: ['src-policy', 'src-terms'], + } + + const partial = scoreKnowledgeBenchmarkArtifact(testCase, { + answer: 'The product has a 30-day refund period.', + citedSourceIds: ['src-policy'], + }) + expect(partial.dimensions.claim_recall).toBe(1) + expect(partial.dimensions.citation_recall).toBe(0.5) + expect(partial.dimensions.hallucination_safe).toBe(1) + expect(partial.score).toBeCloseTo(5 / 6) + expect(partial.passed).toBe(false) + + const hallucinated = scoreKnowledgeBenchmarkArtifact( + { ...testCase, taskKind: 'hallucination' }, + { answer: 'The product has a lifetime refund.' }, + ) + expect(hallucinated.dimensions.hallucination_safe).toBe(0) + expect(hallucinated.raw.matchedForbiddenClaimIds).toEqual(['unsupported-lifetime']) + }) + + it('declares every requested industry benchmark family', () => { + const ids = new Set(INDUSTRY_RAG_BENCHMARKS.map((benchmark) => benchmark.id)) + for (const id of [ + 'beir', + 'mteb-retrieval', + 'msmarco', + 'trec-dl', + 'miracl', + 'lotte', + 'bright', + 'crag', + 'hotpotqa', + 'kilt', + 'ragtruth', + 'faithbench', + 'first-party/kb-improvement', + ]) { + expect(ids.has(id)).toBe(true) + } + }) + + it('runs one persisted benchmark cell for every declared industry family', async () => { + const storage = inMemoryCampaignStorage() + const cases = buildIndustryRagBenchmarkSmokeCases() + const result = await runKnowledgeBenchmarkSuite({ + cases, + runDir: '/runs/knowledge-benchmark-family-smoke', + storage, + respondRef: 'industry-rag-smoke:v1', + respond: respondToIndustryRagBenchmarkSmokeCase, + }) + + expect(cases).toHaveLength(INDUSTRY_RAG_BENCHMARKS.length) + expect(result.report.totalCases).toBe(INDUSTRY_RAG_BENCHMARKS.length) + expect(result.report.totalCells).toBe(INDUSTRY_RAG_BENCHMARKS.length) + expect(result.report.cellsFailed).toBe(0) + expect(result.report.score.mean).toBe(1) + expect(result.report.byTaskKind.retrieval?.n).toBe(7) + expect(result.report.byTaskKind['rag-answer']?.n).toBe(3) + expect(result.report.byTaskKind.hallucination?.n).toBe(2) + expect(result.report.byTaskKind['kb-improvement']?.n).toBe(1) + expect(result.report.totalCostUsd).toBe(0) + for (const benchmark of INDUSTRY_RAG_BENCHMARKS) { + expect(result.report.byFamily[benchmark.family]?.n).toBeGreaterThanOrEqual(1) + } + expect(storage.read(result.reportJsonPath)).toContain('"totalCases": 13') + expect(storage.read(result.reportMarkdownPath)).toContain('## Task Kinds') + }) + + it('calibrates memory scoring against stale and current facts', () => { + const testCase: KnowledgeMemoryBenchmarkCase = { + id: 'memora/smoke:q1', + family: 'memora', + taskKind: 'memory-update', + prompt: 'What does the user currently prefer for daily briefings?', + events: [ + { + id: 'e-old', + actorId: 'user', + timestamp: '2026-01-01T00:00:00.000Z', + text: 'The user used to prefer SMS briefings.', + }, + { + id: 'e-new', + actorId: 'user', + timestamp: '2026-02-01T00:00:00.000Z', + text: 'The user now prefers email briefings.', + }, + ], + requiredFacts: [{ id: 'current-channel', anyOf: ['email briefings'] }], + forbiddenFacts: [{ id: 'stale-channel', anyOf: ['SMS briefings'], obsolete: true }], + expectedEventIds: ['e-new'], + expectedActorIds: ['user'], + } + + const strong = scoreMemoryBenchmarkArtifact(testCase, { + answer: 'The user currently prefers email briefings.', + citedEventIds: ['e-new'], + actorIds: ['user'], + }) + const weak = scoreMemoryBenchmarkArtifact(testCase, { + answer: 'The user prefers SMS briefings.', + citedEventIds: ['e-old'], + actorIds: ['user'], + }) + + expect(strong.score).toBeGreaterThanOrEqual(0.7) + expect(strong.passed).toBe(true) + expect(weak.score).toBeLessThanOrEqual(0.3) + expect(weak.passed).toBe(false) + expect(strong.score - weak.score).toBeGreaterThanOrEqual(0.4) + expect(weak.dimensions.memory_stale_safe).toBe(0) + expect(strong.applicableDimensions).toEqual( + expect.arrayContaining([ + 'memory_fact_recall', + 'memory_event_recall', + 'memory_actor_recall', + 'memory_stale_safe', + ]), + ) + + const recallOnly = scoreMemoryBenchmarkArtifact( + { ...testCase, forbiddenFacts: undefined, expectedActorIds: undefined }, + { answer: 'The user currently prefers email briefings.', citedEventIds: ['e-new'] }, + ) + expect(recallOnly.dimensions).not.toHaveProperty('memory_stale_safe') + expect(recallOnly.dimensions).not.toHaveProperty('memory_actor_recall') + expect(recallOnly.applicableDimensions).not.toContain('memory_stale_safe') + }) + + it('runs one persisted benchmark cell for every declared memory benchmark family', async () => { + const storage = inMemoryCampaignStorage() + const cases = buildIndustryMemoryBenchmarkSmokeCases() + const result = await runKnowledgeBenchmarkSuite({ + cases, + runDir: '/runs/memory-benchmark-family-smoke', + storage, + respondRef: 'industry-memory-smoke:v1', + respond: ({ case: testCase }) => { + expect(isKnowledgeMemoryBenchmarkCase(testCase)).toBe(true) + if (!isKnowledgeMemoryBenchmarkCase(testCase)) { + throw new Error(`expected memory case, got ${testCase.taskKind}`) + } + return respondToIndustryMemoryBenchmarkSmokeCase({ + case: testCase, + }) + }, + }) + + expect(cases).toHaveLength(INDUSTRY_MEMORY_BENCHMARKS.length) + expect(result.report.totalCases).toBe(INDUSTRY_MEMORY_BENCHMARKS.length) + expect(result.report.totalCells).toBe(INDUSTRY_MEMORY_BENCHMARKS.length) + expect(result.report.cellsFailed).toBe(0) + expect(result.report.score.mean).toBe(1) + expect(result.report.byTaskKind['memory-ingest']?.n).toBe(1) + expect(result.report.byTaskKind['memory-recall']?.n).toBe(1) + expect(result.report.byTaskKind['memory-temporal']?.n).toBe(1) + expect(result.report.byTaskKind['memory-update']?.n).toBe(1) + expect(result.report.byTaskKind['memory-forgetting']?.n).toBe(1) + expect(result.report.byTaskKind['memory-reasoning']?.n).toBe(1) + expect(result.report.byTaskKind['memory-summarization']?.n).toBe(1) + expect(result.report.byTaskKind['memory-recommendation']?.n).toBe(1) + expect(result.report.byTaskKind['memory-multiparty']?.n).toBe(1) + expect(result.report.totalCostUsd).toBe(0) + for (const benchmark of INDUSTRY_MEMORY_BENCHMARKS) { + expect(result.report.byFamily[benchmark.family]?.n).toBeGreaterThanOrEqual(1) + } + expect(storage.read(result.reportJsonPath)).toContain('"totalCases": 9') + expect(storage.read(result.reportMarkdownPath)).toContain('memory_stale_safe') + }) +}) diff --git a/tests/benchmarks/cost.test.ts b/tests/benchmarks/cost.test.ts new file mode 100644 index 0000000..296ab6c --- /dev/null +++ b/tests/benchmarks/cost.test.ts @@ -0,0 +1,288 @@ +import { createRunCostLedger, inMemoryCampaignStorage } from '@tangle-network/agent-eval/campaign' +import { describe, expect, it } from 'vitest' +import { + buildFirstPartyMemoryLifecycleBenchmarkCases, + createInMemoryBenchmarkAdapter, +} from '../../src/benchmarks/index' +import { runMemoryAdapterBenchmark } from '../support/benchmarks' + +describe('memory adapter benchmark cost and resume', () => { + it('counts billable adapter creation in the shared benchmark budget', async () => { + const storage = inMemoryCampaignStorage() + let creates = 0 + const result = await runMemoryAdapterBenchmark({ + cases: buildFirstPartyMemoryLifecycleBenchmarkCases().slice(0, 1), + runDir: '/runs/paid-adapter-creation', + storage, + costCeiling: 0.5, + candidates: [ + { + id: 'paid-creation', + ref: 'paid-creation:v1', + adapterId: 'paid-creation-provider', + adapterCreationCostUsd: 0.2, + createAdapter({ markExternalCall }) { + markExternalCall() + creates += 1 + return createInMemoryBenchmarkAdapter({ id: 'paid-creation-provider' }) + }, + }, + ], + }) + + const ledger = createRunCostLedger({ + storage, + runDir: '/runs/paid-adapter-creation', + costCeilingUsd: 0.5, + }) + expect(creates).toBe(1) + expect(result.rows[0]).toMatchObject({ candidateId: 'paid-creation', totalCostUsd: 0.2 }) + expect(result.totalCostUsd).toBe(0.2) + expect(ledger.list()).toMatchObject([ + { + actor: 'agent-knowledge:memory-adapter:paid-creation', + costUsd: 0.2, + tags: { + candidateId: 'paid-creation', + memoryAdapterCreation: 'execute', + runDir: '/runs/paid-adapter-creation/paid-creation', + }, + }, + ]) + }) + + it('does not recreate or recharge a billable adapter when every cell resumes', async () => { + const storage = inMemoryCampaignStorage() + let creates = 0 + const run = () => + runMemoryAdapterBenchmark({ + cases: buildFirstPartyMemoryLifecycleBenchmarkCases().slice(0, 1), + runDir: '/runs/resumed-paid-adapter-creation', + storage, + costCeiling: 0.5, + candidates: [ + { + id: 'paid-resume', + ref: 'paid-resume:v1', + adapterId: 'paid-resume-provider', + adapterCreationCostUsd: 0.2, + createAdapter({ markExternalCall }) { + markExternalCall() + creates += 1 + return createInMemoryBenchmarkAdapter({ id: 'paid-resume-provider' }) + }, + }, + ], + }) + + const initial = await run() + const resumed = await run() + + expect(creates).toBe(1) + expect(initial).toMatchObject({ totalCostUsd: 0.2 }) + expect(resumed).toMatchObject({ totalCostUsd: 0.2 }) + expect(resumed.rows[0]).toMatchObject({ adapterId: 'paid-resume-provider' }) + expect(resumed.rows[0]?.report.cellsCached).toBe(1) + }) + + it('closes a mismatched lazy adapter before any benchmark case writes', async () => { + const storage = inMemoryCampaignStorage() + let closes = 0 + let writes = 0 + const adapter = createInMemoryBenchmarkAdapter({ id: 'actual-provider' }) + const write = adapter.write.bind(adapter) + adapter.write = async (input) => { + writes += 1 + return write(input) + } + adapter.close = async () => { + closes += 1 + } + + await expect( + runMemoryAdapterBenchmark({ + cases: buildFirstPartyMemoryLifecycleBenchmarkCases().slice(0, 1), + runDir: '/runs/mismatched-lazy-adapter', + storage, + candidates: [ + { + id: 'candidate', + ref: 'candidate:v1', + adapterId: 'expected-provider', + createAdapter: () => adapter, + }, + ], + }), + ).rejects.toThrow("returned id 'actual-provider', expected 'expected-provider'") + expect({ closes, writes }).toEqual({ closes: 1, writes: 0 }) + expect( + storage.read('/runs/mismatched-lazy-adapter/memory-adapter-attempts.jsonl'), + ).toBeUndefined() + }) + + it('aborts execute adapter creation at the configured timeout', async () => { + const storage = inMemoryCampaignStorage() + let aborted = false + const startedAt = Date.now() + + await expect( + runMemoryAdapterBenchmark({ + cases: buildFirstPartyMemoryLifecycleBenchmarkCases().slice(0, 1), + runDir: '/runs/timed-out-adapter-creation', + storage, + cleanupTimeoutMs: 10, + candidates: [ + { + id: 'hung-factory', + ref: 'hung-factory:v1', + createAdapter: ({ signal }) => + new Promise((_resolve, reject) => { + signal.addEventListener( + 'abort', + () => { + aborted = true + reject(signal.reason) + }, + { once: true }, + ) + }), + }, + ], + }), + ).rejects.toThrow('benchmark execute adapter creation did not finish within 10ms') + expect(aborted).toBe(true) + expect(Date.now() - startedAt).toBeLessThan(500) + expect( + storage.read('/runs/timed-out-adapter-creation/memory-adapter-attempts.jsonl'), + ).toBeUndefined() + }) + + it('does not charge a local adapter factory failure', async () => { + const storage = inMemoryCampaignStorage() + await expect( + runMemoryAdapterBenchmark({ + cases: buildFirstPartyMemoryLifecycleBenchmarkCases().slice(0, 1), + runDir: '/runs/local-adapter-creation-failure', + storage, + costCeiling: 0.5, + candidates: [ + { + id: 'local-failure', + ref: 'local-failure:v1', + adapterCreationCostUsd: 0.2, + createAdapter() { + throw new Error('invalid local configuration') + }, + }, + ], + }), + ).rejects.toThrow('invalid local configuration') + + const ledger = createRunCostLedger({ + storage, + runDir: '/runs/local-adapter-creation-failure', + costCeilingUsd: 0.5, + }) + expect(ledger.summary()).toMatchObject({ totalCostUsd: 0, unresolvedCalls: 0 }) + }) + + it('enforces one cost ceiling across every compared adapter', async () => { + const result = await runMemoryAdapterBenchmark({ + cases: buildFirstPartyMemoryLifecycleBenchmarkCases().slice(0, 1), + runDir: '/runs/shared-adapter-cost-ceiling', + storage: inMemoryCampaignStorage(), + costCeiling: 0.75, + candidates: ['first', 'second'].map((id) => ({ + id, + ref: `${id}:v1`, + costUsdPerCase: 0.5, + createAdapter: () => createInMemoryBenchmarkAdapter({ id }), + })), + }) + + expect(result.totalCostUsd).toBe(0.5) + expect(result.rows).toHaveLength(2) + expect(result.rows.find((row) => row.candidateId === 'first')).toMatchObject({ + cellsFailed: 0, + totalCostUsd: 0.5, + }) + expect(result.rows.find((row) => row.candidateId === 'second')).toMatchObject({ + cellsFailed: 1, + totalCostUsd: 0, + }) + }) + + it('runs sequential paid adapter cases after exact external-cost receipts', async () => { + const result = await runMemoryAdapterBenchmark({ + cases: buildFirstPartyMemoryLifecycleBenchmarkCases().slice(0, 2), + runDir: '/runs/sequential-adapter-cost-accounting', + storage: inMemoryCampaignStorage(), + costCeiling: 0.2, + maxConcurrency: 1, + candidates: [ + { + id: 'sequential-paid', + ref: 'sequential-paid:v1', + costUsdPerCase: 0.1, + createAdapter: () => createInMemoryBenchmarkAdapter({ id: 'sequential-paid' }), + }, + ], + }) + + expect(result.rows[0]).toMatchObject({ cellsFailed: 0, totalCostUsd: 0.2 }) + expect(result.totalCostUsd).toBe(0.2) + }) + + it('refuses paid adapter calls when no dollar limit is configured', async () => { + let providerCalls = 0 + const adapter = createInMemoryBenchmarkAdapter({ id: 'paid-by-default' }) + const write = adapter.write.bind(adapter) + adapter.write = async (input) => { + providerCalls += 1 + return write(input) + } + + const result = await runMemoryAdapterBenchmark({ + cases: buildFirstPartyMemoryLifecycleBenchmarkCases().slice(0, 1), + runDir: '/runs/default-zero-cost-ceiling', + storage: inMemoryCampaignStorage(), + candidates: [ + { + id: 'paid-by-default', + ref: 'paid-by-default:v1', + costUsdPerCase: 0.01, + createAdapter: () => adapter, + }, + ], + }) + + expect(result.rows[0]).toMatchObject({ cellsFailed: 1, totalCostUsd: 0 }) + expect(providerCalls).toBe(0) + }) + + it('does not reuse resumable rows when adapter benchmark options change', async () => { + const storage = inMemoryCampaignStorage() + const run = (searchLimit: number) => + runMemoryAdapterBenchmark({ + cases: buildFirstPartyMemoryLifecycleBenchmarkCases().slice(0, 1), + runDir: '/runs/memory-adapter-cache-identity', + storage, + candidates: [ + { + id: 'in-memory', + ref: 'in-memory:v1', + createAdapter: () => createInMemoryBenchmarkAdapter(), + searchLimit, + }, + ], + }) + + const initial = await run(1) + const changed = await run(2) + const repeated = await run(2) + + expect(initial.rows[0]?.report.cellsCached).toBe(0) + expect(changed.rows[0]?.report.cellsCached).toBe(0) + expect(repeated.rows[0]?.report.cellsCached).toBe(1) + }) +}) diff --git a/tests/benchmarks/execution.test.ts b/tests/benchmarks/execution.test.ts new file mode 100644 index 0000000..c1977a4 --- /dev/null +++ b/tests/benchmarks/execution.test.ts @@ -0,0 +1,201 @@ +import { inMemoryCampaignStorage } from '@tangle-network/agent-eval/campaign' +import { describe, expect, it } from 'vitest' +import { + buildFirstPartyMemoryLifecycleBenchmarkCases, + createInMemoryBenchmarkAdapter, + createNoopMemoryBenchmarkAdapter, + runMemoryAdapterBenchmark as runMemoryAdapterBenchmarkRaw, +} from '../../src/benchmarks/index' +import type { AgentMemoryAdapter, AgentMemoryScope } from '../../src/memory/types' +import { runMemoryAdapterBenchmark } from '../support/benchmarks' + +describe('memory adapter benchmark execution', () => { + it('ranks actual memory adapters on the first-party lifecycle benchmark', async () => { + const storage = inMemoryCampaignStorage() + const cases = buildFirstPartyMemoryLifecycleBenchmarkCases() + const result = await runMemoryAdapterBenchmark({ + cases, + runDir: '/runs/memory-adapter-ranking', + storage, + candidates: [ + { + id: 'no-memory', + ref: 'no-memory:v1', + createAdapter: () => createNoopMemoryBenchmarkAdapter(), + }, + { + id: 'in-memory', + ref: 'in-memory:v1', + createAdapter: () => createInMemoryBenchmarkAdapter(), + searchLimit: 1, + }, + ], + }) + + expect(cases).toHaveLength(12) + expect(result.rows).toHaveLength(2) + expect(result.rows[0]?.candidateId).toBe('in-memory') + expect(result.rows[0]?.scoreMean).toBeGreaterThan(0.9) + expect(result.rows[0]?.totalCells).toBe(12) + expect(result.rows[0]?.cellsFailed).toBe(0) + expect(result.rows[1]?.candidateId).toBe('no-memory') + expect(result.rows[1]?.scoreMean).toBeLessThan(0.3) + expect(storage.read(result.rankingJsonPath)).toContain('"candidateId": "in-memory"') + expect(storage.read(result.rankingMarkdownPath)).toContain('| 1 | in-memory |') + expect(storage.read(result.rows[0]!.reportJsonPath)).toContain('"memory_stale_safe"') + }) + + it('requires an explicit controller policy for custom benchmark storage', async () => { + await expect( + runMemoryAdapterBenchmarkRaw({ + cases: buildFirstPartyMemoryLifecycleBenchmarkCases().slice(0, 1), + runDir: '/runs/custom-benchmark-storage', + storage: inMemoryCampaignStorage(), + candidates: [ + { + id: 'in-memory', + ref: 'in-memory:v1', + createAdapter: () => createInMemoryBenchmarkAdapter(), + }, + ], + }), + ).rejects.toThrow("requires acquireRunLease or controllerMode='process-local'") + }) + + it('isolates and clears every concurrent adapter repetition', async () => { + const adapter = createInMemoryBenchmarkAdapter() + const clearedScopes: AgentMemoryScope[] = [] + const clear = adapter.clear! + adapter.clear = async (scope) => { + clearedScopes.push(scope ?? {}) + await clear(scope) + } + + const result = await runMemoryAdapterBenchmark({ + cases: buildFirstPartyMemoryLifecycleBenchmarkCases().slice(0, 1), + runDir: '/runs/isolated-memory-repetitions', + storage: inMemoryCampaignStorage(), + reps: 3, + maxConcurrency: 3, + candidates: [ + { + id: 'in-memory', + ref: 'in-memory:v1', + createAdapter: () => adapter, + }, + ], + }) + + expect(result.rows[0]?.totalCells).toBe(3) + expect(result.rows[0]?.cellsFailed).toBe(0) + expect(clearedScopes).toHaveLength(3) + expect(new Set(clearedScopes.map((scope) => scope.namespace)).size).toBe(3) + expect(new Set(clearedScopes.map((scope) => scope.tags?.benchmarkAttemptId)).size).toBe(3) + }) + + it('settles timed-out provider work before clearing and closing the adapter', async () => { + let closed = false + let writes = 0 + let writesAfterClose = 0 + let clears = 0 + let clearsAfterClose = 0 + let contextReads = 0 + const adapter: AgentMemoryAdapter = { + id: 'delayed-provider', + branchIsolation: { mode: 'scoped' }, + async search() { + return [] + }, + async getContext(query) { + contextReads += 1 + return { query, text: '', hits: [], sourceRecords: [] } + }, + async write(input) { + await new Promise((resolve) => setTimeout(resolve, 25)) + writes += 1 + if (closed) writesAfterClose += 1 + return { + accepted: true, + id: input.id ?? String(writes), + uri: `memory://delayed-provider/${writes}`, + kind: input.kind, + } + }, + async clear() { + await new Promise((resolve) => setTimeout(resolve, 1)) + clears += 1 + if (closed) clearsAfterClose += 1 + }, + async close() { + closed = true + }, + } + + const result = await runMemoryAdapterBenchmark({ + cases: buildFirstPartyMemoryLifecycleBenchmarkCases().slice(0, 1), + runDir: '/runs/timed-out-memory-provider', + storage: inMemoryCampaignStorage(), + dispatchTimeoutMs: 5, + candidates: [ + { + id: 'delayed-provider', + ref: 'delayed-provider:v1', + createAdapter: () => adapter, + }, + { + id: 'no-memory', + ref: 'no-memory:v1', + createAdapter: () => createNoopMemoryBenchmarkAdapter(), + }, + ], + }) + + expect(result.rows[0]).toMatchObject({ candidateId: 'no-memory', cellsFailed: 0 }) + expect(result.rows[1]).toMatchObject({ candidateId: 'delayed-provider', cellsFailed: 1 }) + expect({ closed, writes, writesAfterClose, clears, clearsAfterClose, contextReads }).toEqual({ + closed: true, + writes: 1, + writesAfterClose: 0, + clears: 1, + clearsAfterClose: 0, + contextReads: 0, + }) + await new Promise((resolve) => setTimeout(resolve, 50)) + expect({ writes, writesAfterClose, clears, clearsAfterClose }).toEqual({ + writes: 1, + writesAfterClose: 0, + clears: 1, + clearsAfterClose: 0, + }) + }) + + it('rejects every unsafe candidate id before creating any adapter', async () => { + let creates = 0 + await expect( + runMemoryAdapterBenchmark({ + cases: buildFirstPartyMemoryLifecycleBenchmarkCases().slice(0, 1), + runDir: '/runs/invalid-memory-candidate', + storage: inMemoryCampaignStorage(), + candidates: [ + { + id: 'valid', + ref: 'valid:v1', + createAdapter() { + creates += 1 + return createInMemoryBenchmarkAdapter() + }, + }, + { + id: '../invalid', + ref: 'invalid:v1', + createAdapter() { + creates += 1 + return createInMemoryBenchmarkAdapter() + }, + }, + ], + }), + ).rejects.toThrow('must be a safe directory segment') + expect(creates).toBe(0) + }) +}) diff --git a/tests/benchmarks/recovery.test.ts b/tests/benchmarks/recovery.test.ts new file mode 100644 index 0000000..7b01420 --- /dev/null +++ b/tests/benchmarks/recovery.test.ts @@ -0,0 +1,424 @@ +import { createRunCostLedger, inMemoryCampaignStorage } from '@tangle-network/agent-eval/campaign' +import { describe, expect, it } from 'vitest' +import { + buildFirstPartyMemoryLifecycleBenchmarkCases, + createInMemoryBenchmarkAdapter, +} from '../../src/benchmarks/index' +import type { AgentMemoryAdapter, AgentMemoryHit } from '../../src/memory/types' +import { runMemoryAdapterBenchmark } from '../support/benchmarks' + +describe('memory adapter benchmark recovery', () => { + it('recovers an unfinished provider scope before retrying a direct benchmark cell', async () => { + const storage = inMemoryCampaignStorage() + const providerRows = new Map() + const operations: string[] = [] + const purposes: string[] = [] + let firstExecution = true + const candidate = { + id: 'recoverable', + ref: 'recoverable:v1', + adapterId: 'recoverable-provider', + recoveryCostUsdPerAttempt: 0.1, + createAdapter({ purpose }: { purpose: 'execute' | 'recovery' }) { + purposes.push(purpose) + const label = purpose === 'recovery' ? 'recovery' : firstExecution ? 'first' : 'retry' + if (purpose === 'execute') firstExecution = false + const adapter: AgentMemoryAdapter = { + id: 'recoverable-provider', + branchIsolation: { mode: 'scoped' }, + async search(_query, options) { + return [...(providerRows.get(options?.scope?.namespace ?? '') ?? [])] + }, + async getContext(query, options) { + const hits = await adapter.search(query, options) + return { query, text: hits.map((hit) => hit.text).join('\n'), hits, sourceRecords: [] } + }, + async write(input) { + operations.push(`write:${label}`) + const namespace = input.scope?.namespace ?? '' + const hit: AgentMemoryHit = { + id: input.id ?? `${label}:memory`, + uri: `memory://recoverable/${label}`, + kind: input.kind, + text: input.text, + metadata: input.metadata, + } + providerRows.set(namespace, [...(providerRows.get(namespace) ?? []), hit]) + return { accepted: true, id: hit.id, uri: hit.uri, kind: hit.kind } + }, + async clear(scope) { + operations.push(`clear:${label}`) + if (label === 'first') throw new Error('provider cleanup unavailable') + providerRows.delete(scope?.namespace ?? '') + }, + async close() {}, + } + return adapter + }, + } + const run = () => + runMemoryAdapterBenchmark({ + cases: buildFirstPartyMemoryLifecycleBenchmarkCases().slice(0, 1), + runDir: '/runs/direct-benchmark-recovery', + storage, + candidates: [candidate], + costCeiling: 1, + }) + + await expect(run()).rejects.toThrow('memory benchmark attempt cleanup failed') + expect(providerRows.size).toBe(1) + + const result = await run() + expect(result.rows[0]).toMatchObject({ candidateId: 'recoverable', cellsFailed: 0 }) + expect(result.rows[0]?.totalCostUsd).toBe(0.1) + expect(result.totalCostUsd).toBe(0.1) + expect(purposes).toEqual(['execute', 'recovery', 'execute']) + expect(operations.indexOf('clear:recovery')).toBeLessThan(operations.indexOf('write:retry')) + expect(providerRows.size).toBe(0) + const events = storage + .read(result.attemptLogPath)! + .trim() + .split('\n') + .map((line) => JSON.parse(line) as { status: string; recovery: boolean }) + expect(events.map(({ status, recovery }) => ({ status, recovery }))).toEqual([ + { status: 'started', recovery: false }, + { status: 'cleaned', recovery: true }, + { status: 'started', recovery: false }, + { status: 'cleaned', recovery: false }, + ]) + }) + + it('includes paid cleanup for a retired candidate in the benchmark total', async () => { + const storage = inMemoryCampaignStorage() + const runDir = '/runs/retired-candidate-recovery-cost' + storage.write( + `${runDir}/memory-adapter-attempts.jsonl`, + `${JSON.stringify({ + schema: 3, + status: 'started', + attemptId: 'retired-attempt', + candidateId: 'retired', + candidateRef: 'retired:v1', + adapterId: 'retired-provider', + caseId: 'old-case', + cellId: 'old-cell', + scope: { namespace: 'retired-scope' }, + adapterCreationCostUsd: 0.05, + costUsdPerCase: 0, + recoveryCostUsdPerAttempt: 0.1, + recordedAt: '2026-01-01T00:00:00.000Z', + recovery: false, + })}\n`, + ) + let retiredClears = 0 + const result = await runMemoryAdapterBenchmark({ + cases: buildFirstPartyMemoryLifecycleBenchmarkCases().slice(0, 1), + runDir, + storage, + costCeiling: 1, + candidates: [ + { + id: 'active', + ref: 'active:v1', + adapterId: 'active-provider', + createAdapter: () => createInMemoryBenchmarkAdapter({ id: 'active-provider' }), + }, + ], + recoveryCandidates: [ + { + id: 'retired', + ref: 'retired:v1', + adapterId: 'retired-provider', + adapterCreationCostUsd: 0.05, + recoveryCostUsdPerAttempt: 0.1, + createAdapter({ markExternalCall }) { + markExternalCall() + const adapter = createInMemoryBenchmarkAdapter({ id: 'retired-provider' }) + adapter.clear = async () => { + retiredClears += 1 + } + return adapter + }, + }, + ], + }) + + expect(retiredClears).toBe(1) + expect(result.rows[0]).toMatchObject({ candidateId: 'active', totalCostUsd: 0 }) + expect(result).toMatchObject({ totalCostUsd: 0.15, unrankedRecoveryCostUsd: 0.15 }) + expect(storage.read(result.rankingJsonPath)).toContain('"unrankedRecoveryCostUsd": 0.15') + }) + + it('refuses benchmark recovery when candidate cost settings changed', async () => { + const storage = inMemoryCampaignStorage() + const runDir = '/runs/benchmark-changed-recovery-costs' + storage.write( + `${runDir}/memory-adapter-attempts.jsonl`, + `${JSON.stringify({ + schema: 3, + status: 'started', + attemptId: 'unfinished-attempt', + candidateId: 'memory', + candidateRef: 'memory:v1', + adapterId: 'memory-provider', + caseId: 'old-case', + cellId: 'old-cell', + scope: { namespace: 'unfinished-scope' }, + adapterCreationCostUsd: 0, + costUsdPerCase: 0, + recoveryCostUsdPerAttempt: 0, + recordedAt: '2026-01-01T00:00:00.000Z', + recovery: false, + })}\n`, + ) + let adapterCreates = 0 + + await expect( + runMemoryAdapterBenchmark({ + cases: buildFirstPartyMemoryLifecycleBenchmarkCases().slice(0, 1), + runDir, + storage, + costCeiling: 1, + candidates: [ + { + id: 'memory', + ref: 'memory:v1', + adapterId: 'memory-provider', + costUsdPerCase: 0.1, + createAdapter() { + adapterCreates += 1 + return createInMemoryBenchmarkAdapter({ id: 'memory-provider' }) + }, + }, + ], + }), + ).rejects.toThrow('candidate cost settings changed') + + expect(adapterCreates).toBe(0) + expect( + storage.read(`${runDir}/memory-adapter-attempts.jsonl`)?.trim().split('\n'), + ).toHaveLength(1) + }) + + it('bounds repeated direct benchmark recovery across process restarts', async () => { + const storage = inMemoryCampaignStorage() + const runDir = '/runs/benchmark-recovery-retry-limit' + storage.write( + `${runDir}/memory-adapter-attempts.jsonl`, + `${JSON.stringify({ + schema: 3, + status: 'started', + attemptId: 'unfinished-attempt', + candidateId: 'memory', + candidateRef: 'memory:v1', + adapterId: 'memory-provider', + caseId: 'old-case', + cellId: 'old-cell', + scope: { namespace: 'unfinished-scope' }, + adapterCreationCostUsd: 0, + costUsdPerCase: 0, + recoveryCostUsdPerAttempt: 0, + recordedAt: '2026-01-01T00:00:00.000Z', + recovery: false, + })}\n`, + ) + let recoveryCreates = 0 + const run = () => + runMemoryAdapterBenchmark({ + cases: buildFirstPartyMemoryLifecycleBenchmarkCases().slice(0, 1), + runDir, + storage, + maxRecoveryRetriesPerAttempt: 2, + candidates: [ + { + id: 'memory', + ref: 'memory:v1', + adapterId: 'memory-provider', + createAdapter({ purpose }) { + if (purpose === 'recovery') recoveryCreates += 1 + throw new Error('provider recovery unavailable') + }, + }, + ], + }) + + await expect(run()).rejects.toThrow('provider recovery unavailable') + await expect(run()).rejects.toThrow('provider recovery unavailable') + await expect(run()).rejects.toThrow('exhausted 2 recovery attempts') + expect(recoveryCreates).toBe(2) + expect( + storage.read(`${runDir}/memory-adapter-recovery-attempts.jsonl`)?.trim().split('\n'), + ).toHaveLength(2) + }) + + it('closes a recovery adapter that arrives after its factory timeout', async () => { + const storage = inMemoryCampaignStorage() + const runDir = '/runs/late-benchmark-recovery-adapter' + storage.write( + `${runDir}/memory-adapter-attempts.jsonl`, + `${JSON.stringify({ + schema: 3, + status: 'started', + attemptId: 'unfinished-attempt', + candidateId: 'memory', + candidateRef: 'memory:v1', + adapterId: 'memory-provider', + caseId: 'old-case', + cellId: 'old-cell', + scope: { namespace: 'unfinished-scope' }, + adapterCreationCostUsd: 0, + costUsdPerCase: 0, + recoveryCostUsdPerAttempt: 0, + recordedAt: '2026-01-01T00:00:00.000Z', + recovery: false, + })}\n`, + ) + let resolveCreation!: (adapter: AgentMemoryAdapter) => void + const creation = new Promise((resolve) => { + resolveCreation = resolve + }) + let reportClosed!: () => void + const closed = new Promise((resolve) => { + reportClosed = resolve + }) + let closeCalls = 0 + const lateAdapter = createInMemoryBenchmarkAdapter({ id: 'memory-provider' }) + lateAdapter.close = async () => { + closeCalls += 1 + reportClosed() + } + + await expect( + runMemoryAdapterBenchmark({ + cases: buildFirstPartyMemoryLifecycleBenchmarkCases().slice(0, 1), + runDir, + storage, + cleanupTimeoutMs: 10, + candidates: [ + { + id: 'memory', + ref: 'memory:v1', + adapterId: 'memory-provider', + createAdapter: ({ purpose }) => + purpose === 'recovery' ? creation : createInMemoryBenchmarkAdapter(), + }, + ], + }), + ).rejects.toThrow('benchmark recovery adapter creation did not finish within 10ms') + + resolveCreation(lateAdapter) + await closed + expect(closeCalls).toBe(1) + }) + + it('reconciles a crash after direct recovery but before its cost receipt', async () => { + const storage = inMemoryCampaignStorage() + const append = storage.append!.bind(storage) + let failRecoveryReceipt = false + storage.append = (path, value, expectedBytes) => { + if ( + failRecoveryReceipt && + path.endsWith('/cost-ledger.jsonl') && + value.includes('"status":"settled"') && + value.includes('memory-adapter-recovery') + ) { + failRecoveryReceipt = false + throw new Error('simulated process exit before benchmark recovery receipt') + } + return append(path, value, expectedBytes) + } + + let firstExecution = true + let recoveryClears = 0 + const candidate = { + id: 'receipt-crash', + ref: 'receipt-crash:v1', + adapterId: 'receipt-crash-provider', + costUsdPerCase: 0.1, + recoveryCostUsdPerAttempt: 0.1, + createAdapter({ purpose }: { purpose: 'execute' | 'recovery' }) { + const adapter = createInMemoryBenchmarkAdapter({ id: 'receipt-crash-provider' }) + const clear = adapter.clear! + const failThisExecution = purpose === 'execute' && firstExecution + if (purpose === 'execute') firstExecution = false + adapter.clear = async (scope) => { + if (purpose === 'recovery') recoveryClears += 1 + if (failThisExecution) throw new Error('leave direct benchmark state active') + await clear(scope) + } + return adapter + }, + } + const run = () => + runMemoryAdapterBenchmark({ + cases: buildFirstPartyMemoryLifecycleBenchmarkCases().slice(0, 1), + runDir: '/runs/direct-recovery-receipt-crash', + storage, + costCeiling: 1, + candidates: [candidate], + }) + + await expect(run()).rejects.toThrow('memory benchmark attempt cleanup failed') + failRecoveryReceipt = true + await expect(run()).rejects.toThrow('failed to persist') + expect( + storage + .read('/runs/direct-recovery-receipt-crash/memory-adapter-attempts.jsonl') + ?.trim() + .split('\n'), + ).toHaveLength(2) + + const result = await run() + const costLedger = createRunCostLedger({ + storage, + runDir: '/runs/direct-recovery-receipt-crash', + costCeilingUsd: 1, + }) + + expect(result.rows[0]).toMatchObject({ + candidateId: 'receipt-crash', + cellsFailed: 0, + }) + expect(result.rows[0]?.totalCostUsd).toBeCloseTo(0.3) + expect(result.totalCostUsd).toBeCloseTo(0.3) + expect(recoveryClears).toBe(1) + expect(costLedger.summary()).toMatchObject({ unresolvedCalls: 0, accountingComplete: true }) + expect(costLedger.summary().totalCostUsd).toBeCloseTo(0.3) + }) + + it('bounds direct benchmark cleanup and preserves its recovery record', async () => { + const storage = inMemoryCampaignStorage() + const adapter = createInMemoryBenchmarkAdapter({ id: 'hung-cleanup' }) + let closeCalls = 0 + adapter.clear = () => new Promise(() => {}) + adapter.close = async () => { + closeCalls += 1 + } + const startedAt = Date.now() + + await expect( + runMemoryAdapterBenchmark({ + cases: buildFirstPartyMemoryLifecycleBenchmarkCases().slice(0, 1), + runDir: '/runs/direct-benchmark-cleanup-timeout', + storage, + cleanupTimeoutMs: 10, + candidates: [ + { + id: 'hung-cleanup', + ref: 'hung-cleanup:v1', + createAdapter: () => adapter, + }, + ], + }), + ).rejects.toThrow('memory adapter benchmark cleanup failed') + + expect(Date.now() - startedAt).toBeLessThan(500) + expect( + storage + .read('/runs/direct-benchmark-cleanup-timeout/memory-adapter-attempts.jsonl') + ?.trim() + .split('\n'), + ).toHaveLength(1) + expect(closeCalls).toBe(0) + }) +}) diff --git a/tests/memory-systems.test.ts b/tests/memory-systems.test.ts deleted file mode 100644 index cabfb70..0000000 --- a/tests/memory-systems.test.ts +++ /dev/null @@ -1,4876 +0,0 @@ -import type { MemoryClient as Neo4jMemoryClient } from '@neo4j-labs/agent-memory' -import { - createRunCostLedger, - inMemoryCampaignStorage, - type SurfaceProposer, -} from '@tangle-network/agent-eval/campaign' -import type { MemoryClient } from 'mem0ai' -import type { Memory as OssMemory } from 'mem0ai/oss' -import { describe, expect, it } from 'vitest' -import { - type AgentMemoryAdapter, - type AgentMemoryHit, - type AgentMemoryScope, - agentMemorySequenceJudge, - buildAgentMemorySequenceScenarios, - buildAgentMemorySequencesFromBenchmarkCases, - createAgentMemoryBranch, - createGraphitiMemoryAdapter, - createMem0MemoryAdapter, - createNeo4jAgentMemoryAdapter, - forkAgentMemoryBranchSnapshot, - type GraphitiMcpClientLike, - graphitiMemoryAdapterIdentity, - type Mem0ClientLike, - mem0MemoryAdapterIdentity, - type RunAgentMemoryExperimentOptions, - type RunAgentMemoryImprovementOptions, - runAgentMemoryExperiment as runAgentMemoryExperimentRaw, - runAgentMemoryImprovement as runAgentMemoryImprovementRaw, - runBoundedMemoryLifecycle, -} from '../src/memory/index' -import { mergeRankedMemoryHits } from '../src/memory/rank' - -function withProcessLocalController< - T extends { - storage?: unknown - controllerMode?: 'process-local' - acquireRunLease?: unknown - }, ->(options: T): T { - if (!options.storage || options.controllerMode || options.acquireRunLease) return options - return { ...options, controllerMode: 'process-local' } -} - -function runAgentMemoryExperiment(options: RunAgentMemoryExperimentOptions) { - return runAgentMemoryExperimentRaw(withProcessLocalController(options)) -} - -function runAgentMemoryImprovement(options: RunAgentMemoryImprovementOptions) { - return runAgentMemoryImprovementRaw(withProcessLocalController(options)) -} - -describe('memory lifecycle', () => { - it('blocks later operations until timed-out work on the same adapter settles', async () => { - const resource = {} - const abortController = new AbortController() - let release: (() => void) | undefined - const pending = new Promise((resolve) => { - release = resolve - }) - let laterRuns = 0 - - await expect( - runBoundedMemoryLifecycle({ - operation: 'slow clear', - timeoutMs: 5, - resource, - abortController, - run: () => pending, - }), - ).rejects.toThrow('slow clear did not finish within 5ms') - expect(abortController.signal.aborted).toBe(true) - await expect( - runBoundedMemoryLifecycle({ - operation: 'unsafe close', - timeoutMs: 5, - resource, - run: () => { - laterRuns += 1 - }, - }), - ).rejects.toThrow("unsafe close cannot start because 'slow clear' is still running") - expect(laterRuns).toBe(0) - - release?.() - await pending - await new Promise((resolve) => setTimeout(resolve, 0)) - await runBoundedMemoryLifecycle({ - operation: 'settled close', - timeoutMs: 5, - resource, - run: () => { - laterRuns += 1 - }, - }) - expect(laterRuns).toBe(1) - }) - - it('keeps URI and text pairs distinct when newline-delimited identities collide', () => { - const first: AgentMemoryHit = { - id: 'first', - uri: 'memory://provider/a\nb', - kind: 'fact', - text: 'c', - } - const second: AgentMemoryHit = { - id: 'second', - uri: 'memory://provider/a', - kind: 'fact', - text: 'b\nc', - } - - expect(mergeRankedMemoryHits([[first], [second]]).map((hit) => hit.id)).toEqual([ - 'first', - 'second', - ]) - }) -}) - -describe('Mem0 adapter', () => { - it('is type-compatible with both current Mem0 clients', () => { - const hosted = null as unknown as MemoryClient - const oss = null as unknown as OssMemory - - expect(createMem0MemoryAdapter({ client: hosted, mode: 'hosted' }).id).toBe('mem0-hosted') - expect(createMem0MemoryAdapter({ client: oss, mode: 'oss' }).id).toBe('mem0-oss') - }) - - it('writes and searches with the same provider scope', async () => { - const calls: Array<{ method: string; options?: Record }> = [] - const client: Mem0ClientLike = { - async add(_messages, options) { - calls.push({ method: 'add', options }) - return [{ id: 'memory-1', event: 'ADD' }] - }, - async search(_query, options) { - calls.push({ method: 'search', options }) - return { - results: [ - { - id: 'memory-1', - memory: 'Use concise status updates.', - score: 0.91, - metadata: { - memoryKind: 'preference', - memoryTitle: 'Writing style', - eventId: 'event-1', - }, - }, - ], - } - }, - } - const adapter = createMem0MemoryAdapter({ client, mode: 'hosted', rerank: true }) - const scope = { - tenantId: 'tenant-1', - userId: 'user-1', - agentId: 'agent-1', - teamId: 'team-1', - namespace: 'branch-1/private', - sessionId: 'session-1', - tags: { task: 'support' }, - } - - await adapter.write({ - id: 'event-1', - kind: 'preference', - title: 'Writing style', - text: 'Use concise status updates.', - scope, - metadata: { run_id: 'attempted-override', memoryKind: 'entity' }, - }) - const hits = await adapter.search('status style', { scope, limit: 3 }) - - const add = calls[0]?.options - const search = calls[1]?.options - expect(add).toMatchObject({ - userId: 'user-1', - agentId: 'agent-1', - runId: 'branch-1/private', - infer: true, - metadata: { - memoryKind: 'preference', - tenant_id: 'tenant-1', - user_id: 'user-1', - agent_id: 'agent-1', - team_id: 'team-1', - run_id: 'branch-1/private', - session_id: 'session-1', - tag_task: 'support', - }, - }) - expect(search).toMatchObject({ - filters: { - user_id: 'user-1', - agent_id: 'agent-1', - run_id: 'branch-1/private', - tenant_id: 'tenant-1', - team_id: 'team-1', - session_id: 'session-1', - tag_task: 'support', - }, - topK: 3, - rerank: true, - }) - expect(hits[0]).toMatchObject({ - id: 'memory-1', - kind: 'preference', - text: 'Use concise status updates.', - score: 0.91, - normalizedScore: 0.91, - metadata: { eventId: 'event-1' }, - }) - }) - - it('pushes memory kinds into provider filters before top-k truncation', async () => { - const searchOptions: Record[] = [] - const adapter = createMem0MemoryAdapter({ - mode: 'hosted', - client: { - async add() { - return [] - }, - async search(_query, options) { - searchOptions.push(options ?? {}) - const filters = options?.filters as Record | undefined - if (filters?.memoryKind === 'preference') { - return { - results: [ - { - id: 'preference-1', - memory: 'Use concise updates.', - metadata: { memoryKind: 'preference' }, - }, - ], - } - } - return { - results: [ - { - id: 'fact-1', - memory: 'Higher-ranked unrelated fact.', - metadata: { memoryKind: 'fact' }, - }, - ], - } - }, - }, - }) - - const filtered = await adapter.search('updates', { - scope: { userId: 'user-1' }, - kinds: ['preference'], - limit: 1, - }) - const unfiltered = await adapter.search('updates', { - scope: { userId: 'user-1' }, - kinds: [], - limit: 1, - }) - - expect(filtered.map((hit) => hit.id)).toEqual(['preference-1']) - expect(unfiltered.map((hit) => hit.id)).toEqual(['fact-1']) - expect(searchOptions[0]).toMatchObject({ - filters: { user_id: 'user-1', memoryKind: 'preference' }, - topK: 1, - }) - expect(searchOptions[1]).toMatchObject({ filters: { user_id: 'user-1' }, topK: 1 }) - }) - - it('rejects legacy asynchronous hosted responses the current SDK cannot produce', async () => { - const adapter = createMem0MemoryAdapter({ - mode: 'hosted', - client: { - async add() { - return { status: 'PENDING', eventId: 'event-1' } - }, - async search() { - return { results: [] } - }, - }, - }) - - await expect( - adapter.write({ - kind: 'fact', - text: 'Remember this.', - scope: { userId: 'user-1' }, - }), - ).rejects.toThrow('mem0ai 3.x must return a memory array') - }) - - it('requires fresh attempt branches for hosted Mem0', async () => { - const client: Mem0ClientLike = { - async add() { - return [] - }, - async search() { - return { results: [] } - }, - } - const visible = createMem0MemoryAdapter({ mode: 'hosted', client }) - - expect(() => createAgentMemoryBranch({ adapter: visible, branchId: 'mem0-resumable' })).toThrow( - "must use lifetime='attempt'", - ) - const attempt = createAgentMemoryBranch({ - adapter: visible, - branchId: 'mem0-attempt', - lifetime: 'attempt', - }) - const snapshot = await attempt.snapshot() - await attempt.close() - expect(() => - createAgentMemoryBranch({ - adapter: visible, - branchId: 'mem0-attempt', - snapshot, - }), - ).toThrow('attempt snapshots cannot be resumed') - }) - - it('does not journal an empty provider result as an accepted write', async () => { - const adapter = createMem0MemoryAdapter({ - mode: 'hosted', - client: { - async add() { - return [] - }, - async search() { - return { results: [] } - }, - }, - }) - - await expect( - adapter.write({ - kind: 'fact', - text: 'No provider mutation', - scope: { userId: 'user-1' }, - }), - ).resolves.toMatchObject({ accepted: false }) - }) - - it('journals and replays Mem0 deletion mutations', async () => { - const parentCalls: string[] = [] - const childCalls: string[] = [] - const adapter = (calls: string[]) => - createMem0MemoryAdapter({ - mode: 'oss', - client: { - async add(messages) { - const text = messages[0]!.content - calls.push(text) - return [ - { - id: text.startsWith('Forget') ? 'memory-old' : 'memory-new', - metadata: { event: text.startsWith('Forget') ? 'DELETE' : 'ADD' }, - }, - ] - }, - async search() { - return { results: [] } - }, - }, - }) - const parent = createAgentMemoryBranch({ - adapter: adapter(parentCalls), - branchId: 'mem0-parent', - baseScope: { agentId: 'agent-1' }, - }) - - await parent.write({ kind: 'fact', text: 'Remember the old launch date.' }) - const deletion = await parent.write({ kind: 'fact', text: 'Forget the old launch date.' }) - const snapshot = await parent.snapshot() - await parent.fork({ branchId: 'mem0-child', adapter: adapter(childCalls) }) - - expect(deletion).toMatchObject({ accepted: true, metadata: { event: 'DELETE' } }) - expect(snapshot.journal.map((entry) => entry.input.text)).toEqual([ - 'Remember the old launch date.', - 'Forget the old launch date.', - ]) - expect(childCalls).toEqual(parentCalls) - }) - - it('includes the default scope in the Mem0 adapter identity', () => { - const base = { mode: 'hosted' as const, id: 'mem0', backendRef: 'mem0-account-a' } - expect(mem0MemoryAdapterIdentity({ ...base, defaultScope: { tenantId: 'tenant-a' } })).not.toBe( - mem0MemoryAdapterIdentity({ ...base, defaultScope: { tenantId: 'tenant-b' } }), - ) - expect(mem0MemoryAdapterIdentity({ ...base, ingestionTimeoutMs: 100 })).not.toBe( - mem0MemoryAdapterIdentity({ ...base, ingestionTimeoutMs: 200 }), - ) - }) - - it('passes OSS entity filters on add', async () => { - let addOptions: Record | undefined - const adapter = createMem0MemoryAdapter({ - mode: 'oss', - client: { - async add(_messages, options) { - addOptions = options - return { results: [{ id: 'oss-1', memory: 'Remembered' }] } - }, - async search() { - return { results: [] } - }, - }, - }) - - await adapter.write({ - kind: 'fact', - text: 'Remembered', - scope: { userId: 'user-1', agentId: 'agent-1', namespace: 'run-1' }, - }) - - expect(addOptions).toMatchObject({ - userId: 'user-1', - agentId: 'agent-1', - runId: 'run-1', - filters: { user_id: 'user-1', agent_id: 'agent-1', run_id: 'run-1' }, - }) - }) - - it('rejects direct OSS operations without a provider entity scope', async () => { - let providerCalls = 0 - const adapter = createMem0MemoryAdapter({ - mode: 'oss', - client: { - async add() { - providerCalls += 1 - return { results: [] } - }, - async search() { - providerCalls += 1 - return { results: [] } - }, - }, - }) - - await expect(adapter.write({ kind: 'fact', text: 'No owner.' })).rejects.toThrow( - 'requires scope.userId, scope.agentId, scope.runId, or scope.namespace', - ) - await expect(adapter.search('No owner.')).rejects.toThrow( - 'requires scope.userId, scope.agentId, scope.runId, or scope.namespace', - ) - expect(providerCalls).toBe(0) - }) - - it('rejects unscoped hosted operations before calling Mem0', async () => { - let providerCalls = 0 - const adapter = createMem0MemoryAdapter({ - mode: 'hosted', - appId: 'support-app', - client: { - async add() { - providerCalls += 1 - return [] - }, - async search() { - providerCalls += 1 - return { results: [] } - }, - async getAll() { - providerCalls += 1 - return { results: [] } - }, - async delete() { - providerCalls += 1 - return { message: 'deleted' } - }, - }, - }) - - await expect(adapter.write({ kind: 'fact', text: 'No owner.' })).rejects.toThrow( - 'Mem0 hosted write requires', - ) - await expect(adapter.search('No owner.')).rejects.toThrow('Mem0 hosted search requires') - await expect(adapter.clear?.()).rejects.toThrow('refusing an unscoped Mem0 clear') - expect(providerCalls).toBe(0) - }) - - it('deletes only memories matching a complete scoped filter', async () => { - const rows = new Set(['memory-1', 'memory-2']) - const getAllOptions: Record[] = [] - let deleteAllCalls = 0 - const adapter = createMem0MemoryAdapter({ - mode: 'hosted', - appId: 'support-app', - client: { - async add() { - return [] - }, - async search() { - return { results: [] } - }, - async getAll(options) { - getAllOptions.push(options ?? {}) - return { results: [...rows].map((id) => ({ id, memory: id })) } - }, - async delete(memoryId) { - rows.delete(memoryId) - return { message: 'deleted' } - }, - async deleteAll() { - deleteAllCalls += 1 - return { message: 'deleted all' } - }, - }, - }) - - await adapter.clear?.({ - tenantId: 'tenant-1', - userId: 'user-1', - namespace: 'physical-branch', - runId: 'logical-run', - sessionId: 'session-1', - tags: { visibility: 'private' }, - }) - - expect(rows.size).toBe(0) - expect(deleteAllCalls).toBe(0) - expect(getAllOptions[0]).toMatchObject({ - filters: { - app_id: 'support-app', - user_id: 'user-1', - run_id: 'physical-branch', - tenant_id: 'tenant-1', - logical_run_id: 'logical-run', - session_id: 'session-1', - tag_visibility: 'private', - }, - page: 1, - pageSize: 100, - showExpired: true, - }) - }) - - it('waits for hosted deletes to disappear without deleting the same memory twice', async () => { - let deletedAt: number | undefined - let getAllCalls = 0 - const deletedIds: string[] = [] - const adapter = createMem0MemoryAdapter({ - mode: 'hosted', - ingestionTimeoutMs: 50, - pollIntervalMs: 1, - client: { - async add() { - return [] - }, - async search() { - return { results: [] } - }, - async getAll() { - getAllCalls += 1 - const visible = deletedAt === undefined || Date.now() - deletedAt < 5 - return { results: visible ? [{ id: 'memory-1', memory: 'stale listing' }] : [] } - }, - async delete(memoryId) { - deletedIds.push(memoryId) - deletedAt = Date.now() - return { message: 'deletion accepted' } - }, - }, - }) - - await adapter.clear?.({ userId: 'user-1', namespace: 'branch-1' }) - - expect(deletedIds).toEqual(['memory-1']) - expect(getAllCalls).toBeGreaterThan(2) - }) - - it('waits for list and search indexes to drop deleted memories before reporting cleanup', async () => { - let deletedAt: number | undefined - let getAllCalls = 0 - let searchCalls = 0 - let deleteCalls = 0 - const adapter = createMem0MemoryAdapter({ - mode: 'hosted', - ingestionTimeoutMs: 50, - pollIntervalMs: 1, - client: { - async add() { - return [] - }, - async search() { - searchCalls += 1 - const visible = deletedAt === undefined || Date.now() - deletedAt < 8 - return { results: visible ? [{ id: 'memory-1', memory: 'stale listing' }] : [] } - }, - async getAll() { - getAllCalls += 1 - const visible = deletedAt === undefined || Date.now() - deletedAt < 5 - return { results: visible ? [{ id: 'memory-1', memory: 'stale listing' }] : [] } - }, - async delete() { - deleteCalls += 1 - deletedAt = Date.now() - return { message: 'deletion accepted' } - }, - }, - }) - - await adapter.clear?.({ userId: 'user-1', namespace: 'branch-1' }) - - expect(deleteCalls).toBe(1) - expect(getAllCalls).toBeGreaterThan(1) - expect(searchCalls).toBeGreaterThan(1) - }) - - it('waits for a fresh narrower id-less write when clearing a broader Mem0 scope', async () => { - const writtenAt = Date.now() - let deleted = false - let getAllCalls = 0 - let deleteCalls = 0 - const visible = () => !deleted && Date.now() - writtenAt >= 5 - const adapter = createMem0MemoryAdapter({ - mode: 'hosted', - ingestionTimeoutMs: 15, - pollIntervalMs: 1, - client: { - async add() { - return [{ event: 'ADD' }] - }, - async search() { - return { results: visible() ? [{ id: 'late-memory', memory: 'late fact' }] : [] } - }, - async getAll() { - getAllCalls += 1 - return { results: visible() ? [{ id: 'late-memory', memory: 'late fact' }] : [] } - }, - async delete(memoryId) { - expect(memoryId).toBe('late-memory') - deleteCalls += 1 - deleted = true - return { message: 'deleted' } - }, - }, - }) - - await adapter.write({ - kind: 'fact', - text: 'late fact', - scope: { userId: 'user-1', agentId: 'agent-1', namespace: 'branch-1' }, - }) - await adapter.clear?.({ userId: 'user-1' }) - - expect(deleteCalls).toBe(1) - expect(getAllCalls).toBeGreaterThan(1) - }) - - it('expires pending write probes after the configured visibility window', async () => { - let searchCalls = 0 - const adapter = createMem0MemoryAdapter({ - mode: 'hosted', - ingestionTimeoutMs: 1, - pollIntervalMs: 1, - client: { - async add() { - return [{ event: 'ADD' }] - }, - async search() { - searchCalls += 1 - return { results: [] } - }, - async getAll() { - return { results: [] } - }, - async delete() { - return { message: 'deleted' } - }, - }, - }) - - await adapter.write({ - kind: 'fact', - text: 'bounded pending fact', - scope: { userId: 'user-1' }, - }) - await new Promise((resolve) => setTimeout(resolve, 5)) - await adapter.clear?.({ userId: 'user-1' }) - - expect(searchCalls).toBe(0) - }) - - it('refuses to clear Mem0 without an exact identity scope', async () => { - let providerCalls = 0 - const adapter = createMem0MemoryAdapter({ - mode: 'hosted', - client: { - async add() { - return [] - }, - async search() { - providerCalls += 1 - return { results: [] } - }, - async getAll() { - providerCalls += 1 - return { results: [] } - }, - async delete() { - providerCalls += 1 - return { message: 'deleted' } - }, - }, - }) - - await expect(adapter.clear?.()).rejects.toThrow('refusing an unscoped Mem0 clear') - expect(providerCalls).toBe(0) - }) -}) - -describe('Graphiti adapter', () => { - it('uses the full default ingestion window before recovering an abandoned branch', () => { - const adapter = createGraphitiMemoryAdapter({ - client: { - async callTool() { - return { structuredContent: {} } - }, - }, - }) - - expect(adapter.branchIsolation).toEqual({ - mode: 'scoped', - processExitSafe: false, - recoveryDelayMs: 120_000, - }) - }) - - it('uses official MCP request shapes and rejoins fact provenance to episodes', async () => { - const requests: Array<{ name: string; arguments?: Record }> = [] - const episodes = new Map>() - const client: GraphitiMcpClientLike = { - async callTool(request) { - requests.push(request) - const args = request.arguments ?? {} - if (request.name === 'add_memory') { - episodes.set(String(args.uuid), { - uuid: args.uuid, - name: args.name, - content: args.episode_body, - source_description: args.source_description, - group_id: args.group_id, - }) - return { structuredContent: { message: 'queued' } } - } - if (request.name === 'get_episodes') { - return { - content: [{ type: 'text', text: JSON.stringify({ episodes: [...episodes.values()] }) }], - } - } - if (request.name === 'search_memory_facts') { - const episode = [...episodes.values()][0]! - return { - structuredContent: { - facts: [ - { - uuid: 'fact-1', - name: 'PREFERS', - fact: 'The user prefers concise status updates.', - group_id: episode.group_id, - episodes: [episode.uuid], - }, - ], - }, - } - } - if (request.name === 'search_nodes') { - return { structuredContent: { nodes: [] } } - } - throw new Error(`unexpected tool ${request.name}`) - }, - } - const adapter = createGraphitiMemoryAdapter({ - client, - search: ['facts'], - pollIntervalMs: 1, - ingestionTimeoutMs: 50, - }) - - const write = await adapter.write({ - id: 'event-1', - kind: 'message', - role: 'user', - text: 'I prefer concise status updates.', - scope: { userId: 'user-1', namespace: 'branch-1' }, - metadata: { actorId: 'user', timestamp: '2026-07-17T12:00:00Z' }, - }) - const hits = await adapter.search('status preference', { - scope: { userId: 'user-1', namespace: 'branch-1' }, - limit: 5, - }) - - expect(requests[0]).toMatchObject({ - name: 'add_memory', - arguments: { - episode_body: 'I prefer concise status updates.', - source: 'message', - reference_time: '2026-07-17T12:00:00Z', - }, - }) - expect(String(requests[0]?.arguments?.group_id)).toMatch(/^ak_[a-f0-9]{32}$/) - expect(String(requests[0]?.arguments?.uuid)).toMatch( - /^[a-f0-9]{8}-[a-f0-9]{4}-5[a-f0-9]{3}-a[a-f0-9]{3}-[a-f0-9]{12}$/, - ) - expect(write.metadata).toMatchObject({ consistency: 'visible', queued: false }) - expect(hits[0]).toMatchObject({ - id: 'fact-1', - kind: 'fact', - text: 'The user prefers concise status updates.', - metadata: { eventId: 'event-1', actorId: 'user' }, - }) - }) - - it('surfaces Graphiti tool errors', async () => { - const adapter = createGraphitiMemoryAdapter({ - client: { - async callTool() { - return { structuredContent: { error: 'database unavailable' } } - }, - }, - }) - - await expect(adapter.search('anything')).rejects.toThrow('database unavailable') - }) - - it('expands the episode scan when Graphiti truncates a long group', async () => { - const episodes = Array.from({ length: 150 }, (_, index) => ({ - uuid: `older-${index.toString().padStart(3, '0')}`, - })) - const limits: number[] = [] - const adapter = createGraphitiMemoryAdapter({ - pollIntervalMs: 1, - ingestionTimeoutMs: 50, - client: { - async callTool(request) { - if (request.name === 'add_memory') { - episodes.push({ uuid: String(request.arguments?.uuid) }) - return { structuredContent: { message: 'queued' } } - } - if (request.name === 'get_episodes') { - const limit = Number(request.arguments?.max_episodes) - limits.push(limit) - return { structuredContent: { episodes: episodes.slice(0, limit) } } - } - throw new Error(`unexpected tool ${request.name}`) - }, - }, - }) - - await adapter.write({ kind: 'fact', text: 'Visible beyond the first page.' }) - - expect(limits).toEqual([100, 200]) - }) - - it('keeps polling a full Graphiti group while the new episode becomes visible', async () => { - const older = Array.from({ length: 100 }, (_, index) => ({ uuid: `older-${index}` })) - let pendingUuid = '' - let visible = false - let polls = 0 - const adapter = createGraphitiMemoryAdapter({ - episodeScanLimit: 100, - pollIntervalMs: 1, - ingestionTimeoutMs: 50, - client: { - async callTool(request) { - if (request.name === 'add_memory') { - pendingUuid = String(request.arguments?.uuid) - setTimeout(() => { - visible = true - }, 5) - return { structuredContent: { message: 'queued' } } - } - if (request.name === 'get_episodes') { - polls += 1 - return { - structuredContent: { - episodes: visible ? [...older.slice(1), { uuid: pendingUuid }] : older, - }, - } - } - throw new Error(`unexpected tool ${request.name}`) - }, - }, - }) - - await expect( - adapter.write({ kind: 'fact', text: 'Eventually visible.' }), - ).resolves.toMatchObject({ accepted: true }) - expect(polls).toBeGreaterThan(1) - }) - - it('fails explicitly when visibility cannot be proved within the episode scan limit', async () => { - const episodes = Array.from({ length: 100 }, (_, index) => ({ uuid: `older-${index}` })) - const adapter = createGraphitiMemoryAdapter({ - episodeScanLimit: 100, - pollIntervalMs: 1, - ingestionTimeoutMs: 50, - client: { - async callTool(request) { - if (request.name === 'add_memory') { - episodes.push({ uuid: String(request.arguments?.uuid) }) - return { structuredContent: { message: 'queued' } } - } - if (request.name === 'get_episodes') { - return { structuredContent: { episodes: episodes.slice(0, 100) } } - } - throw new Error(`unexpected tool ${request.name}`) - }, - }, - }) - - await expect(adapter.write({ kind: 'fact', text: 'Outside the scan window.' })).rejects.toThrow( - 'episodeScanLimit=100', - ) - }) - - it('supports documented tool aliases and does not invent missing relevance scores', async () => { - const names: string[] = [] - const adapter = createGraphitiMemoryAdapter({ - search: ['facts'], - toolNames: { searchFacts: 'search_facts' }, - client: { - async callTool(request) { - names.push(request.name) - return { - structuredContent: { - facts: [{ uuid: 'fact-1', fact: 'An unscored fact', episodes: [] }], - }, - } - }, - }, - }) - - const hits = await adapter.search('fact', { minScore: 0.5 }) - - expect(names).toEqual(['search_facts']) - expect(hits).toEqual([]) - }) - - it('isolates run, session, and tag scopes into different Graphiti groups', async () => { - const groupIds: string[] = [] - const adapter = createGraphitiMemoryAdapter({ - search: ['facts'], - client: { - async callTool(request) { - if (request.name === 'search_memory_facts') { - groupIds.push(String((request.arguments?.group_ids as string[] | undefined)?.[0])) - return { structuredContent: { facts: [] } } - } - return { structuredContent: { episodes: [] } } - }, - }, - }) - - await adapter.search('query', { - scope: { namespace: 'same', runId: 'run-a', sessionId: 'session', tags: { arm: 'a' } }, - }) - await adapter.search('query', { - scope: { namespace: 'same', runId: 'run-b', sessionId: 'session', tags: { arm: 'b' } }, - }) - - expect(new Set(groupIds).size).toBe(2) - }) - - it('keeps caller-identified episode ids stable when groups execute in a different order', async () => { - const run = async (scopes: AgentMemoryScope[]) => { - const ids = new Map() - const adapter = createGraphitiMemoryAdapter({ - consistency: 'queued', - client: { - async callTool(request) { - if (request.name === 'add_memory') { - ids.set(String(request.arguments?.group_id), String(request.arguments?.uuid)) - } - return { structuredContent: { message: 'queued' } } - }, - }, - }) - for (const scope of scopes) { - await adapter.write({ id: 'source-1', kind: 'fact', text: 'same fact', scope }) - } - return ids - } - const first = await run([{ namespace: 'a' }, { namespace: 'b' }]) - const reversed = await run([{ namespace: 'b' }, { namespace: 'a' }]) - - expect(first).toEqual(reversed) - }) - - it('uses a new episode id when the content of one source event changes', async () => { - const episodeIds: string[] = [] - const adapter = createGraphitiMemoryAdapter({ - consistency: 'queued', - client: { - async callTool(request) { - if (request.name === 'add_memory') { - episodeIds.push(String(request.arguments?.uuid)) - } - return { structuredContent: { message: 'queued' } } - }, - }, - }) - - await adapter.write({ id: 'source-1', kind: 'fact', text: 'Launch is Friday.' }) - await adapter.write({ id: 'source-1', kind: 'fact', text: 'Launch is Monday.' }) - - expect(episodeIds).toHaveLength(2) - expect(episodeIds[0]).not.toBe(episodeIds[1]) - }) - - it('keeps repeated id-less Graphiti writes as distinct episodes', async () => { - const episodeIds: string[] = [] - const adapter = createGraphitiMemoryAdapter({ - consistency: 'queued', - client: { - async callTool(request) { - if (request.name === 'add_memory') { - episodeIds.push(String(request.arguments?.uuid)) - } - return { structuredContent: { message: 'queued' } } - }, - }, - }) - - await adapter.write({ kind: 'fact', text: 'The same observation.' }) - await adapter.write({ kind: 'fact', text: 'The same observation.' }) - - expect(episodeIds).toHaveLength(2) - expect(episodeIds[0]).not.toBe(episodeIds[1]) - }) - - it('keeps id-less Graphiti writes distinct across adapter restarts', async () => { - const episodeIds: string[] = [] - const client: GraphitiMcpClientLike = { - async callTool(request) { - if (request.name === 'add_memory') episodeIds.push(String(request.arguments?.uuid)) - return { structuredContent: { message: 'queued' } } - }, - } - - await createGraphitiMemoryAdapter({ client, consistency: 'queued' }).write({ - kind: 'fact', - text: 'The same observation.', - }) - await createGraphitiMemoryAdapter({ client, consistency: 'queued' }).write({ - kind: 'fact', - text: 'The same observation.', - }) - - expect(episodeIds).toHaveLength(2) - expect(episodeIds[0]).not.toBe(episodeIds[1]) - }) - - it('treats an empty Graphiti kind list as unfiltered search', async () => { - const names: string[] = [] - const adapter = createGraphitiMemoryAdapter({ - client: { - async callTool(request) { - names.push(request.name) - if (request.name === 'search_memory_facts') { - return { structuredContent: { facts: [] } } - } - if (request.name === 'search_nodes') return { structuredContent: { nodes: [] } } - throw new Error(`unexpected tool ${request.name}`) - }, - }, - }) - - await adapter.search('anything', { kinds: [] }) - - expect(names).toEqual(['search_memory_facts', 'search_nodes']) - }) - - it('fuses unscored Graphiti fact and node rankings before applying the limit', async () => { - const adapter = createGraphitiMemoryAdapter({ - client: { - async callTool(request) { - if (request.name === 'search_memory_facts') { - return { - structuredContent: { - facts: [ - { uuid: 'fact-1', fact: 'first fact' }, - { uuid: 'fact-2', fact: 'second fact' }, - ], - }, - } - } - if (request.name === 'search_nodes') { - return { - structuredContent: { - nodes: [ - { uuid: 'node-1', name: 'first node' }, - { uuid: 'node-2', name: 'second node' }, - ], - }, - } - } - throw new Error(`unexpected tool ${request.name}`) - }, - }, - }) - - const hits = await adapter.search('anything', { limit: 2 }) - - expect(hits.map((hit) => hit.id)).toEqual(['fact-1', 'node-1']) - }) - - it('uses Graphiti clear_graph for exact group cleanup', async () => { - const requests: Array<{ name: string; arguments?: Record }> = [] - const adapter = createGraphitiMemoryAdapter({ - client: { - async callTool(request) { - requests.push(request) - return { structuredContent: { message: 'Graph cleared successfully' } } - }, - }, - }) - - await adapter.clear?.({ userId: 'user-1', namespace: 'candidate-1' }) - - expect(requests).toHaveLength(1) - expect(requests[0]).toMatchObject({ - name: 'clear_graph', - arguments: { group_ids: [expect.stringMatching(/^ak_[a-f0-9]{32}$/)] }, - }) - }) - - it('rejects queued ingestion for resumable branch experiments', () => { - const adapter = createGraphitiMemoryAdapter({ - consistency: 'queued', - client: { - async callTool() { - return { structuredContent: { message: 'queued' } } - }, - }, - }) - - expect(() => createAgentMemoryBranch({ adapter, branchId: 'candidate-1' })).toThrow( - 'queued Graphiti writes can become visible after a process restart', - ) - }) - - it('includes the default scope in the Graphiti adapter identity', () => { - expect( - graphitiMemoryAdapterIdentity({ - id: 'graphiti', - backendRef: 'graphiti-cluster-a', - defaultScope: { namespace: 'one' }, - }), - ).not.toBe( - graphitiMemoryAdapterIdentity({ - id: 'graphiti', - backendRef: 'graphiti-cluster-a', - defaultScope: { namespace: 'two' }, - }), - ) - }) -}) - -describe('Neo4j Agent Memory adapter', () => { - it('is type-compatible with the current official client', () => { - const client = {} as unknown as Neo4jMemoryClient - expect( - createNeo4jAgentMemoryAdapter({ client, transport: 'rest', branchId: 'branch-1' }).id, - ).toBe('neo4j-agent-memory') - }) - - it('stores observations as searchable messages without retrying a successful void write', async () => { - let camelWrites = 0 - let snakeWrites = 0 - let role: unknown - let closes = 0 - const adapter = createNeo4jAgentMemoryAdapter({ - transport: 'rest', - branchId: 'branch-1', - client: { - shortTerm: { - async addMessage(_sessionId: unknown, inputRole: unknown) { - camelWrites += 1 - role = inputRole - return undefined - }, - async add_message() { - snakeWrites += 1 - return { id: 'duplicate' } - }, - async searchMessages() { - return [ - { - id: 'observation-1', - role: 'assistant', - content: 'The deployment completed.', - metadata: { agentKnowledgeKind: 'observation' }, - }, - ] - }, - }, - async close() { - closes += 1 - }, - }, - }) - - const write = await adapter.write({ - kind: 'observation', - role: 'tool', - text: 'The deployment completed.', - scope: { sessionId: 'session-1' }, - }) - const hits = await adapter.search('deployment', { - kinds: ['observation'], - scope: { sessionId: 'session-1' }, - }) - await adapter.close?.() - - expect(write.accepted).toBe(true) - expect(camelWrites).toBe(1) - expect(snakeWrites).toBe(0) - expect(role).toBe('assistant') - expect(hits).toMatchObject([{ id: 'observation-1', kind: 'observation' }]) - expect(closes).toBe(1) - }) - - it('fails clearly when REST is asked for bridge-only facts and preferences', async () => { - const adapter = createNeo4jAgentMemoryAdapter({ - transport: 'rest', - client: { longTerm: {} }, - }) - - await expect(adapter.write({ kind: 'fact', text: 'A fact' })).rejects.toThrow( - 'Neo4j REST cannot write fact', - ) - await expect(adapter.search('preference', { kinds: ['preference'] })).rejects.toThrow( - 'Neo4j rest cannot search memory kinds: preference', - ) - }) - - it('rejects an incompatible Neo4j client instead of reporting an empty search', async () => { - const adapter = createNeo4jAgentMemoryAdapter({ - client: { longTerm: {} }, - transport: 'rest', - }) - - await expect(adapter.search('company', { kinds: ['entity'] })).rejects.toThrow( - 'missing method searchEntities', - ) - }) - - it('rejects non-throwing Neo4j write errors', async () => { - const adapter = createNeo4jAgentMemoryAdapter({ - transport: 'rest', - client: { - shortTerm: { - async addMessage() { - return { success: false, error: { message: 'database unavailable' } } - }, - }, - }, - }) - - await expect( - adapter.write({ - kind: 'message', - text: 'Remember this.', - scope: { sessionId: 'session-1' }, - }), - ).rejects.toThrow('database unavailable') - }) - - it('maps hosted reasoning memories to recordStep', async () => { - let recorded: Record | undefined - const adapter = createNeo4jAgentMemoryAdapter({ - transport: 'rest', - client: { - reasoning: { - async recordStep(input: Record) { - recorded = input - return { id: 'step-1', ...input } - }, - }, - }, - }) - - const result = await adapter.write({ - kind: 'reasoning-trace', - text: 'Check the deployment status.', - title: 'status-check', - scope: { sessionId: 'conversation-1' }, - metadata: { action: 'query_status', result: 'healthy' }, - }) - - expect(recorded).toEqual({ - conversationId: 'conversation-1', - reasoning: 'Check the deployment status.', - actionTaken: 'query_status', - result: 'healthy', - }) - expect(result).toMatchObject({ accepted: true, id: 'step-1', kind: 'reasoning-trace' }) - }) - - it('uses bridge preference APIs only in bridge mode', async () => { - const adapter = createNeo4jAgentMemoryAdapter({ - transport: 'bridge', - client: { - longTerm: { - async addPreference(category: string, preference: string) { - return { id: 'preference-1', category, preference } - }, - async searchPreferences() { - return [{ id: 'preference-1', category: 'style', preference: 'Be concise.' }] - }, - }, - }, - }) - - await expect( - adapter.write({ kind: 'preference', category: 'style', text: 'Be concise.' }), - ).resolves.toMatchObject({ id: 'preference-1' }) - await expect(adapter.search('concise', { kinds: ['preference'] })).resolves.toMatchObject([ - { id: 'preference-1', kind: 'preference', text: 'Be concise.' }, - ]) - }) - - it('fuses unscored Neo4j memory-type rankings before applying the limit', async () => { - const adapter = createNeo4jAgentMemoryAdapter({ - transport: 'bridge', - client: { - shortTerm: { - async searchMessages() { - return [ - { id: 'message-1', role: 'user', content: 'first message' }, - { id: 'message-2', role: 'user', content: 'second message' }, - ] - }, - }, - longTerm: { - async searchEntities() { - return [ - { id: 'entity-1', name: 'first entity', type: 'custom' }, - { id: 'entity-2', name: 'second entity', type: 'custom' }, - ] - }, - async searchPreferences() { - return [] - }, - }, - reasoning: { - async getSimilarTraces() { - return [] - }, - }, - }, - }) - - const hits = await adapter.search('anything', { limit: 2 }) - - expect(hits.map((hit) => hit.id)).toEqual(['message-1', 'entity-1']) - }) -}) - -describe('memory branches', () => { - it('rejects two live handles for the same adapter branch', async () => { - const adapter = createScopedTestAdapter('duplicate-handle') - const first = createAgentMemoryBranch({ adapter, branchId: 'same-branch' }) - - expect(() => createAgentMemoryBranch({ adapter, branchId: 'same-branch' })).toThrow( - "memory branch 'same-branch' already has an open handle", - ) - - await first.close?.() - const resumed = createAgentMemoryBranch({ adapter, branchId: 'same-branch' }) - await resumed.close?.() - }) - - it('rejects adapters that do not declare how branches are isolated', () => { - const { branchIsolation: _branchIsolation, ...legacy } = createScopedTestAdapter('legacy') - - expect(() => - createAgentMemoryBranch({ - adapter: legacy, - branchId: 'candidate-a', - }), - ).toThrow(/adapter must declare branchIsolation/) - }) - - it('rejects Neo4j clients that were not isolated for the exact branch', () => { - const unscoped = createNeo4jAgentMemoryAdapter({ client: {}, transport: 'rest' }) - expect(() => createAgentMemoryBranch({ adapter: unscoped, branchId: 'candidate-a' })).toThrow( - 'create a separate MemoryClient namespace per branch', - ) - - const scoped = createNeo4jAgentMemoryAdapter({ - client: {}, - transport: 'rest', - branchId: 'candidate-a', - }) - expect( - createAgentMemoryBranch({ - adapter: scoped, - branchId: 'candidate-a', - policy: { read: ['shared'], write: 'shared' }, - }).branchId, - ).toBe('candidate-a') - expect(() => createAgentMemoryBranch({ adapter: scoped, branchId: 'candidate-a' })).toThrow( - 'only shared memory policy', - ) - expect(() => createAgentMemoryBranch({ adapter: scoped, branchId: 'candidate-b' })).toThrow( - "adapter instance belongs to branch 'candidate-a'", - ) - }) - - it('isolates branches and private agents while allowing team sharing', async () => { - const storage = createScopedTestAdapter('scoped') - const alpha = createAgentMemoryBranch({ - adapter: storage, - branchId: 'alpha', - policy: { read: ['private'], write: 'private' }, - baseScope: { tenantId: 'tenant', userId: 'user' }, - }) - const beta = createAgentMemoryBranch({ - adapter: storage, - branchId: 'beta', - policy: { read: ['private'], write: 'private' }, - baseScope: { tenantId: 'tenant', userId: 'user' }, - }) - - await alpha.write({ - kind: 'fact', - text: 'alpha private', - scope: { agentId: 'agent-a', teamId: 'team-1' }, - }) - await beta.write({ - kind: 'fact', - text: 'beta private', - scope: { agentId: 'agent-a', teamId: 'team-1' }, - }) - - expect((await alpha.search('private', { scope: { agentId: 'agent-a' } })).map(hitText)).toEqual( - ['alpha private'], - ) - expect((await alpha.search('private', { scope: { agentId: 'agent-b' } })).map(hitText)).toEqual( - [], - ) - expect((await beta.search('private', { scope: { agentId: 'agent-a' } })).map(hitText)).toEqual([ - 'beta private', - ]) - - const team = createAgentMemoryBranch({ - adapter: storage, - branchId: 'team-branch', - policy: { read: ['team'], write: 'team' }, - baseScope: { tenantId: 'tenant', userId: 'user' }, - }) - await team.write({ - kind: 'observation', - text: 'shared with the team', - scope: { agentId: 'agent-a', teamId: 'team-1' }, - }) - const teamHits = await team.search('team', { - scope: { agentId: 'agent-b', teamId: 'team-1' }, - }) - expect(teamHits.map(hitText)).toEqual(['shared with the team']) - }) - - it('removes every journal entry in a provider partition cleared through a narrower scope', async () => { - const branch = createAgentMemoryBranch({ - adapter: createScopedTestAdapter('team-clear'), - branchId: 'team-clear', - policy: { read: ['team'], write: 'team' }, - baseScope: { tenantId: 'tenant', teamId: 'team-1' }, - }) - await branch.write({ kind: 'fact', text: 'from a', scope: { agentId: 'agent-a' } }) - await branch.write({ kind: 'fact', text: 'from b', scope: { agentId: 'agent-b' } }) - - await branch.clear?.({ agentId: 'agent-a' }) - - expect((await branch.snapshot()).journal).toEqual([]) - await expect(branch.search('from b', { scope: { agentId: 'agent-b' } })).resolves.toEqual([]) - }) - - it('preserves provider order for unscored branch hits', async () => { - const hits: AgentMemoryHit[] = [ - { id: 'z-top', uri: 'memory://rank/z', kind: 'fact', text: 'provider first' }, - { id: 'a-lower', uri: 'memory://rank/a', kind: 'fact', text: 'provider second' }, - ] - const adapter: AgentMemoryAdapter = { - id: 'provider-ranked', - branchIsolation: { mode: 'scoped' }, - async search() { - return hits - }, - async getContext(query) { - return { query, text: '', hits: [], sourceRecords: [] } - }, - async write(input) { - return { accepted: true, id: 'write', uri: 'memory://rank/write', kind: input.kind } - }, - } - const branch = createAgentMemoryBranch({ - adapter, - branchId: 'provider-ranked', - baseScope: { agentId: 'agent-a' }, - }) - - const ranked = await branch.search('anything', { limit: 1 }) - - expect(ranked.map((hit) => hit.id)).toEqual(['z-top']) - }) - - it('fuses unscored visibility lists by provider rank', async () => { - const adapter: AgentMemoryAdapter = { - id: 'visibility-ranked', - branchIsolation: { mode: 'scoped' }, - async search(_query, options) { - return options?.scope?.tags?.memoryVisibility === 'private' - ? [ - { id: 'private-1', uri: 'memory://rank/private-1', kind: 'fact', text: 'p1' }, - { id: 'private-2', uri: 'memory://rank/private-2', kind: 'fact', text: 'p2' }, - ] - : [{ id: 'team-1', uri: 'memory://rank/team-1', kind: 'fact', text: 't1' }] - }, - async getContext(query) { - return { query, text: '', hits: [], sourceRecords: [] } - }, - async write(input) { - return { accepted: true, id: 'write', uri: 'memory://rank/write', kind: input.kind } - }, - } - const branch = createAgentMemoryBranch({ - adapter, - branchId: 'visibility-ranked', - policy: { read: ['private', 'team'], write: 'private' }, - baseScope: { agentId: 'agent-a', teamId: 'team-a' }, - }) - - const ranked = await branch.search('anything', { limit: 2 }) - - expect(ranked.map((hit) => hit.id)).toEqual(['private-1', 'team-1']) - }) - - it('uses rank fusion before incomparable scores from separate provider searches', async () => { - const adapter: AgentMemoryAdapter = { - id: 'visibility-scored', - branchIsolation: { mode: 'scoped' }, - async search(_query, options) { - return options?.scope?.tags?.memoryVisibility === 'private' - ? [ - { - id: 'private-first', - uri: 'memory://rank/private-first', - kind: 'fact', - text: 'private first', - score: 0.01, - }, - { - id: 'private-second', - uri: 'memory://rank/private-second', - kind: 'fact', - text: 'private second', - score: 1, - }, - ] - : [ - { - id: 'team-first', - uri: 'memory://rank/team-first', - kind: 'fact', - text: 'team first', - score: 0.02, - }, - ] - }, - async getContext(query) { - return { query, text: '', hits: [], sourceRecords: [] } - }, - async write(input) { - return { accepted: true, id: 'write', uri: 'memory://rank/write', kind: input.kind } - }, - } - const branch = createAgentMemoryBranch({ - adapter, - branchId: 'visibility-scored', - policy: { read: ['private', 'team'], write: 'private' }, - baseScope: { agentId: 'agent-a', teamId: 'team-a' }, - }) - - const ranked = await branch.search('anything', { limit: 2 }) - - expect(ranked.map((hit) => hit.id)).toEqual(['team-first', 'private-first']) - }) - - it('serializes writes per actor and permits independent actors in parallel', async () => { - let active = 0 - let maxActive = 0 - const starts: string[] = [] - const storage = createScopedTestAdapter('ordered', async (scope, text) => { - starts.push(`${scope.agentId}:${text}`) - active += 1 - maxActive = Math.max(maxActive, active) - await new Promise((resolve) => setTimeout(resolve, 10)) - active -= 1 - }) - const branch = createAgentMemoryBranch({ - adapter: storage, - branchId: 'parallel', - policy: { read: ['private'], write: 'private' }, - }) - - await Promise.all([ - branch.write({ kind: 'message', text: 'a1', scope: { agentId: 'a' } }), - branch.write({ kind: 'message', text: 'a2', scope: { agentId: 'a' } }), - branch.write({ kind: 'message', text: 'b1', scope: { agentId: 'b' } }), - ]) - - expect(maxActive).toBe(2) - expect(starts.indexOf('a:a1')).toBeLessThan(starts.indexOf('a:a2')) - expect((await branch.snapshot()).journal.map((entry) => entry.input.text)).toEqual([ - 'a1', - 'a2', - 'b1', - ]) - }) - - it('preserves one actor ordering across sessions and tags', async () => { - const completed: string[] = [] - const storage = createScopedTestAdapter('actor-order', async (_scope, text) => { - if (text === 'first') await new Promise((resolve) => setTimeout(resolve, 20)) - completed.push(text) - }) - const branch = createAgentMemoryBranch({ - adapter: storage, - branchId: 'actor-order', - baseScope: { tenantId: 'tenant', agentId: 'agent-a' }, - }) - - await Promise.all([ - branch.write({ - kind: 'fact', - text: 'first', - scope: { sessionId: 'session-1', tags: { task: 'one' } }, - }), - branch.write({ - kind: 'fact', - text: 'second', - scope: { sessionId: 'session-2', tags: { task: 'two' } }, - }), - ]) - - expect(completed).toEqual(['first', 'second']) - }) - - it('takes a point-in-time snapshot while later writes wait at the boundary', async () => { - let releaseFirst!: () => void - const firstMayFinish = new Promise((resolve) => { - releaseFirst = resolve - }) - let reportFirstStarted!: () => void - const firstStarted = new Promise((resolve) => { - reportFirstStarted = resolve - }) - const providerStarts: string[] = [] - const storage = createScopedTestAdapter('snapshot-boundary', async (_scope, text) => { - providerStarts.push(text) - if (text === 'first') { - reportFirstStarted() - await firstMayFinish - } - }) - const branch = createAgentMemoryBranch({ - adapter: storage, - branchId: 'snapshot-boundary', - baseScope: { agentId: 'worker' }, - }) - - const firstWrite = branch.write({ kind: 'fact', text: 'first' }) - await firstStarted - const snapshotPromise = branch.snapshot() - const secondWrite = branch.write({ kind: 'fact', text: 'second' }) - await Promise.resolve() - - expect(providerStarts).toEqual(['first']) - releaseFirst() - await firstWrite - const snapshot = await snapshotPromise - await secondWrite - - expect(snapshot.journal.map((entry) => entry.input.text)).toEqual(['first']) - expect(providerStarts).toEqual(['first', 'second']) - }) - - it('does not clear storage while a preceding read is still using it', async () => { - let finishRead!: () => void - const readMayFinish = new Promise((resolve) => { - finishRead = resolve - }) - let reportReadStarted!: () => void - const readStarted = new Promise((resolve) => { - reportReadStarted = resolve - }) - let clearStarted = false - const adapter: AgentMemoryAdapter = { - id: 'read-clear-boundary', - branchIsolation: { mode: 'scoped' }, - async search() { - reportReadStarted() - await readMayFinish - return [] - }, - async getContext(query) { - return { query, text: '', hits: [], sourceRecords: [] } - }, - async write(input) { - return { accepted: true, id: 'write', uri: 'memory://write', kind: input.kind } - }, - async clear() { - clearStarted = true - }, - } - const branch = createAgentMemoryBranch({ - adapter, - branchId: 'read-clear-boundary', - baseScope: { agentId: 'worker' }, - }) - - const read = branch.search('active read') - await readStarted - const clear = branch.clear?.() - await Promise.resolve() - expect(clearStarted).toBe(false) - - finishRead() - await read - await clear - expect(clearStarted).toBe(true) - }) - - it('clears a touched scope after the provider commits and then throws', async () => { - let dirty = false - const clearedScopes: AgentMemoryScope[] = [] - const adapter: AgentMemoryAdapter = { - id: 'commit-then-error', - branchIsolation: { mode: 'scoped' }, - async search() { - return [] - }, - async getContext(query) { - return { query, text: '', hits: [], sourceRecords: [] } - }, - async write() { - dirty = true - throw new Error('connection lost after commit') - }, - async clear(scope) { - clearedScopes.push(scope ?? {}) - dirty = false - }, - } - const branch = createAgentMemoryBranch({ - adapter, - branchId: 'commit-then-error', - baseScope: { tenantId: 'tenant' }, - }) - - await expect( - branch.write({ kind: 'fact', text: 'partial', scope: { agentId: 'worker' } }), - ).rejects.toThrow('connection lost after commit') - await branch.clear?.() - - expect(dirty).toBe(false) - expect(clearedScopes).toHaveLength(1) - expect(clearedScopes[0]).toMatchObject({ tenantId: 'tenant', agentId: 'worker' }) - }) - - it('rejects non-durable journal data before it can corrupt resume state', async () => { - let providerWrites = 0 - const adapter = createScopedTestAdapter('durable-journal', async () => { - providerWrites += 1 - }) - const branch = createAgentMemoryBranch({ - adapter, - branchId: 'durable-journal', - baseScope: { agentId: 'worker' }, - }) - const cyclic: Record = {} - cyclic.self = cyclic - - await expect( - branch.write({ kind: 'fact', text: 'bad metadata', metadata: cyclic }), - ).rejects.toThrow('must not contain cycles') - expect(providerWrites).toBe(0) - expect((await branch.snapshot()).journal).toEqual([]) - }) - - it('returns snapshots detached from live branch state', async () => { - const adapter = createScopedTestAdapter('detached-snapshot') - const branch = createAgentMemoryBranch({ - adapter, - branchId: 'detached-snapshot', - baseScope: { tenantId: 'tenant-original', agentId: 'agent-a' }, - }) - await branch.write({ - kind: 'fact', - text: 'original text', - metadata: { nested: { status: 'original' } }, - }) - - const exposed = await branch.snapshot() - exposed.baseScope.tenantId = 'tenant-mutated' - const nested = exposed.journal[0]!.input.metadata!.nested as Record - nested.status = 'mutated' - const current = await branch.snapshot() - - expect(current.baseScope.tenantId).toBe('tenant-original') - expect(current.journal[0]?.input.metadata).toEqual({ nested: { status: 'original' } }) - expect(await branch.search('original')).toHaveLength(1) - }) - - it('closes a shared adapter once after every branch releases it', async () => { - let closes = 0 - const base = createScopedTestAdapter('shared-lifecycle') - const adapter: AgentMemoryAdapter = { - ...base, - async close() { - closes += 1 - }, - } - const first = createAgentMemoryBranch({ - adapter, - branchId: 'first', - baseScope: { agentId: 'agent-1' }, - }) - const second = createAgentMemoryBranch({ - adapter, - branchId: 'second', - baseScope: { agentId: 'agent-2' }, - }) - - await first.close?.() - await first.close?.() - expect(closes).toBe(0) - await second.close?.() - await second.close?.() - - expect(closes).toBe(1) - await expect(first.search('after close')).rejects.toThrow('closed branch') - }) - - it('waits for a concurrent fork to finish before closing the parent', async () => { - let releaseSnapshot!: () => void - let releaseReplay!: () => void - let reportReplayStarted!: () => void - const snapshotBlocked = new Promise((resolve) => { - releaseSnapshot = resolve - }) - const replayBlocked = new Promise((resolve) => { - releaseReplay = resolve - }) - const replayStarted = new Promise((resolve) => { - reportReplayStarted = resolve - }) - const base = createScopedTestAdapter('fork-close-race') - let writes = 0 - let flushes = 0 - let closes = 0 - const adapter: AgentMemoryAdapter = { - ...base, - async write(input) { - writes += 1 - if (writes === 2) { - reportReplayStarted() - await replayBlocked - } - return base.write(input) - }, - async flush() { - flushes += 1 - if (flushes === 1) await snapshotBlocked - }, - async close() { - closes += 1 - }, - } - const parent = createAgentMemoryBranch({ - adapter, - branchId: 'fork-close-parent', - baseScope: { agentId: 'worker' }, - }) - await parent.write({ kind: 'fact', text: 'replay me' }) - - const fork = parent.fork({ branchId: 'fork-close-child' }) - const close = parent.close!() - let closeSettled = false - void close.finally(() => { - closeSettled = true - }) - releaseSnapshot() - await replayStarted - await Promise.resolve() - await Promise.resolve() - - expect(closeSettled).toBe(false) - expect(closes).toBe(0) - releaseReplay() - const child = await fork - await close - expect((await child.search('replay')).map(hitText)).toEqual(['replay me']) - expect(closes).toBe(0) - await child.close?.() - expect(closes).toBe(1) - }) - - it('forks by replaying accepted writes into another adapter and resumes exactly', async () => { - const parentStorage = createScopedTestAdapter('parent') - const childStorage = createScopedTestAdapter('child') - const parent = createAgentMemoryBranch({ - adapter: parentStorage, - branchId: 'parent', - policy: { read: ['private'], write: 'private' }, - baseScope: { tenantId: 'tenant', agentId: 'agent-a' }, - }) - await parent.write({ kind: 'fact', text: 'seed fact' }) - - await expect(parent.fork({ branchId: 'parent' })).rejects.toThrow( - 'child branchId must differ from its parent', - ) - expect((await parent.search('fact')).map(hitText)).toEqual(['seed fact']) - - const child = await parent.fork({ branchId: 'child', adapter: childStorage }) - await child.write({ kind: 'fact', text: 'child only' }) - - expect((await parent.search('fact')).map(hitText)).toEqual(['seed fact']) - expect((await child.search('fact')).map(hitText)).toEqual(['seed fact', 'child only']) - const snapshot = await child.snapshot() - expect(snapshot.journal.every((entry) => Boolean(entry.input.id))).toBe(true) - expect(new Set(snapshot.journal.map((entry) => entry.input.id)).size).toBe(2) - await child.close?.() - const resumed = createAgentMemoryBranch({ - adapter: childStorage, - branchId: 'child', - snapshot, - }) - snapshot.baseScope.tenantId = 'mutated-after-resume' - expect((await resumed.search('fact')).map(hitText)).toEqual(['seed fact', 'child only']) - expect((await resumed.snapshot()).baseScope.tenantId).toBe('tenant') - expect((await resumed.snapshot()).digest).toBe(snapshot.digest) - - await expect( - Promise.resolve().then(() => - createAgentMemoryBranch({ - adapter: childStorage, - branchId: 'child', - snapshot: { ...snapshot, digest: 'sha256:bad' }, - }), - ), - ).rejects.toThrow('digest mismatch') - }) - - it('replays an attempt snapshot into a fresh branch', async () => { - const adapter: AgentMemoryAdapter = { - ...createScopedTestAdapter('attempt-source'), - branchIsolation: { mode: 'scoped', processExitSafe: false, recoveryDelayMs: 1 }, - } - const source = createAgentMemoryBranch({ - adapter, - branchId: 'attempt-source', - lifetime: 'attempt', - baseScope: { agentId: 'worker' }, - }) - await source.write({ kind: 'fact', text: 'durable observation' }) - const snapshot = await source.snapshot() - const target = await forkAgentMemoryBranchSnapshot({ - snapshot, - adapter: createScopedTestAdapter('resumable-target'), - branchId: 'resumable-target', - lifetime: 'resumable', - }) - - expect((await target.search('observation')).map(hitText)).toEqual(['durable observation']) - expect(target.parentBranchId).toBe('attempt-source') - expect(target.lifetime).toBe('resumable') - }) -}) - -describe('agent memory experiments', () => { - it('versions memory scoring for resumable cache safety', () => { - expect(agentMemorySequenceJudge().judgeVersion).toBe('agent-knowledge:memory-sequence:v2') - }) - - it('requires an explicit controller policy for custom storage', async () => { - await expect( - runAgentMemoryExperimentRaw({ - experimentId: 'custom-storage-controller', - sequences: [ - { - id: 'history', - family: 'first-party', - steps: [{ id: 'probe', probes: [{ id: 'probe', query: 'x', referenceAnswer: 'x' }] }], - }, - ], - candidates: [ - { - id: 'memory', - ref: 'memory:v1', - createAdapter: () => createScopedTestAdapter('memory'), - }, - ], - runDir: '/runs/custom-storage-controller', - storage: inMemoryCampaignStorage(), - }), - ).rejects.toThrow("requires acquireRunLease or controllerMode='process-local'") - }) - - it('bounds provider cleanup and leaves the attempt available for recovery', async () => { - const storage = inMemoryCampaignStorage() - let clearCalls = 0 - const adapter = createScopedTestAdapter('bounded-cleanup') - const clear = adapter.clear! - adapter.clear = async (scope) => { - clearCalls += 1 - if (clearCalls === 1) return new Promise(() => {}) - await clear(scope) - } - const startedAt = Date.now() - - await expect( - runAgentMemoryExperiment({ - experimentId: 'bounded-cleanup', - sequences: [ - { - id: 'history', - family: 'first-party', - steps: [ - { - id: 'remember', - scope: { agentId: 'worker' }, - writes: [{ kind: 'fact', text: 'bounded fact' }], - probes: [{ id: 'probe', query: 'bounded', referenceAnswer: 'bounded fact' }], - }, - ], - }, - ], - candidates: [{ id: 'memory', ref: 'memory:v1', createAdapter: () => adapter }], - runDir: '/runs/bounded-cleanup', - storage, - cleanupTimeoutMs: 10, - }), - ).rejects.toThrow('memory experiment cleanup failed after dispatch') - - expect(Date.now() - startedAt).toBeLessThan(500) - expect( - storage.read('/runs/bounded-cleanup/memory-attempts.jsonl')?.trim().split('\n'), - ).toHaveLength(1) - }) - - it('does not charge provider cost when side-effect-free adapter construction fails', async () => { - const storage = inMemoryCampaignStorage() - const costLedger = createRunCostLedger({ - storage, - runDir: '/runs/create-adapter-cost', - costCeilingUsd: 1, - }) - - await expect( - runAgentMemoryExperiment({ - experimentId: 'create-adapter-cost', - sequences: [ - { - id: 'history', - family: 'first-party', - steps: [{ id: 'probe', probes: [{ id: 'probe', query: 'x', referenceAnswer: 'x' }] }], - }, - ], - candidates: [ - { - id: 'paid-memory', - ref: 'paid-memory:v1', - externalCostUsdPerSequence: 0.25, - createAdapter() { - throw new Error('provider setup rejected') - }, - }, - ], - runDir: '/runs/create-adapter-cost', - storage, - costLedger, - }), - ).rejects.toThrow('memory experiment cleanup failed after dispatch') - - expect(costLedger.summary().totalCostUsd).toBe(0) - }) - - it('charges a factory failure after dedicated provider provisioning starts', async () => { - const storage = inMemoryCampaignStorage() - const costLedger = createRunCostLedger({ - storage, - runDir: '/runs/create-adapter-provider-cost', - costCeilingUsd: 1, - }) - - await expect( - runAgentMemoryExperiment({ - experimentId: 'create-adapter-provider-cost', - sequences: [ - { - id: 'history', - family: 'first-party', - steps: [{ id: 'probe', probes: [{ id: 'probe', query: 'x', referenceAnswer: 'x' }] }], - }, - ], - candidates: [ - { - id: 'paid-memory', - ref: 'paid-memory:v1', - externalCostUsdPerSequence: 0.25, - createAdapter({ markExternalCall }) { - markExternalCall() - throw new Error('provider provisioning failed') - }, - }, - ], - runDir: '/runs/create-adapter-provider-cost', - storage, - costLedger, - }), - ).rejects.toThrow('memory experiment cleanup failed after dispatch') - - expect(costLedger.summary()).toMatchObject({ totalCalls: 1, totalCostUsd: 0.25 }) - }) - - it('records provider cleanup before a paid-call receipt can be interrupted', async () => { - const storage = inMemoryCampaignStorage() - const append = storage.append!.bind(storage) - let interruptReceipt = true - storage.append = (path, value, expectedBytes) => { - if ( - interruptReceipt && - path.endsWith('/cost-ledger.jsonl') && - value.includes('"status":"settled"') && - value.includes('agent-knowledge:memory-experiment') - ) { - interruptReceipt = false - throw new Error('simulated process exit before execute receipt') - } - return append(path, value, expectedBytes) - } - const purposes: string[] = [] - const sequence = { - id: 'history', - family: 'first-party' as const, - steps: [ - { - id: 'remember', - scope: { agentId: 'worker' }, - writes: [{ kind: 'fact' as const, text: 'durable fact' }], - probes: [{ id: 'recall', query: 'durable', referenceAnswer: 'durable fact' }], - }, - ], - } - const run = () => - runAgentMemoryExperiment({ - experimentId: 'execute-receipt-crash', - sequences: [sequence], - candidates: [ - { - id: 'memory', - ref: 'memory:v1', - externalCostUsdPerSequence: 0.1, - createAdapter({ purpose }) { - purposes.push(purpose) - return createScopedTestAdapter(`memory:${purpose}`) - }, - }, - ], - runDir: '/runs/execute-receipt-crash', - storage, - costCeiling: 1, - }) - - const interrupted = await run() - expect(interrupted.campaign.aggregates).toMatchObject({ - cellsFailed: 1, - cost: { accountingComplete: false, unresolvedCalls: 1 }, - }) - expect( - storage.read('/runs/execute-receipt-crash/memory-attempts.jsonl')?.trim().split('\n'), - ).toHaveLength(2) - - const result = await run() - expect(purposes).toEqual(['execute', 'execute']) - expect(result.rows[0]).toMatchObject({ cellsFailed: 0, totalCostUsd: 0.2 }) - }) - - it('accounts for parallel candidates against one shared dollar limit', async () => { - let releaseFirstCalls: (() => void) | undefined - const firstCallsReady = new Promise((resolve) => { - releaseFirstCalls = resolve - }) - let activeCalls = 0 - let firstCalls = 0 - let maxActiveCalls = 0 - const createAdapter = (id: string): AgentMemoryAdapter => { - const adapter = createScopedTestAdapter(id) - const clear = adapter.clear! - let entered = false - adapter.clear = async (scope) => { - if (!entered) { - entered = true - firstCalls += 1 - activeCalls += 1 - maxActiveCalls = Math.max(maxActiveCalls, activeCalls) - if (firstCalls === 2) releaseFirstCalls?.() - await firstCallsReady - activeCalls -= 1 - } - await clear(scope) - } - return adapter - } - - const result = await runAgentMemoryExperiment({ - experimentId: 'parallel-shared-cost', - sequences: [ - { - id: 'history', - family: 'first-party', - steps: [ - { - id: 'remember', - scope: { agentId: 'worker' }, - writes: [{ kind: 'fact', text: 'parallel fact' }], - probes: [ - { - id: 'recall', - query: 'parallel', - scope: { agentId: 'worker' }, - referenceAnswer: 'parallel fact', - }, - ], - }, - ], - }, - ], - candidates: ['first', 'second'].map((id) => ({ - id, - ref: `${id}:v1`, - externalCostUsdPerSequence: 0.1, - createAdapter: () => createAdapter(id), - })), - runDir: '/runs/parallel-shared-cost', - storage: inMemoryCampaignStorage(), - costCeiling: 0.2, - maxConcurrency: 2, - }) - - expect(maxActiveCalls).toBe(2) - expect(result.rows.map((row) => row.totalCostUsd).sort()).toEqual([0.1, 0.1]) - expect(result.campaign.aggregates.totalCostUsd).toBe(0.2) - }) - - it('runs sequential paid histories after exact external-cost receipts', async () => { - const storage = inMemoryCampaignStorage() - const costLedger = createRunCostLedger({ - storage, - runDir: '/runs/sequential-shared-cost', - costCeilingUsd: 0.2, - }) - const sequences = ['first', 'second'].map((id) => ({ - id, - family: 'first-party', - steps: [ - { - id: 'remember', - scope: { agentId: 'worker' }, - writes: [{ kind: 'fact' as const, text: `${id} fact` }], - probes: [ - { - id: 'recall', - query: id, - scope: { agentId: 'worker' }, - referenceAnswer: `${id} fact`, - }, - ], - }, - ], - })) - - const result = await runAgentMemoryExperiment({ - experimentId: 'sequential-shared-cost', - sequences, - candidates: [ - { - id: 'memory', - ref: 'memory:v1', - externalCostUsdPerSequence: 0.1, - createAdapter: ({ sequence }) => createScopedTestAdapter(sequence.id), - }, - ], - runDir: '/runs/sequential-shared-cost', - storage, - costLedger, - maxConcurrency: 1, - }) - - expect(result.rows[0]).toMatchObject({ cellsFailed: 0, totalCostUsd: 0.2 }) - expect(costLedger.summary()).toMatchObject({ - totalCalls: 2, - totalCostUsd: 0.2, - accountingComplete: true, - }) - }) - - it('reports the reserved cost of an interrupted prior cell attempt', async () => { - const storage = inMemoryCampaignStorage() - const runDir = '/runs/interrupted-cell-cost-reporting' - const sequence = { - id: 'history', - family: 'first-party' as const, - steps: [ - { - id: 'remember', - scope: { agentId: 'worker' }, - writes: [{ kind: 'fact' as const, text: 'resumed fact' }], - probes: [{ id: 'recall', query: 'resumed', referenceAnswer: 'resumed fact' }], - }, - ], - } - const candidate = { - id: 'memory', - ref: 'memory:v1', - externalCostUsdPerSequence: 0.1, - createAdapter: () => createScopedTestAdapter('memory'), - } - const scenarioId = buildAgentMemorySequenceScenarios([sequence], [candidate])[0]!.id - const abandonedLedger = createRunCostLedger({ storage, runDir, costCeilingUsd: 1 }) - const controller = new AbortController() - let releaseProvider!: () => void - let reportStarted!: () => void - const started = new Promise((resolve) => { - reportStarted = resolve - }) - const abandoned = abandonedLedger.runPaidCall({ - callId: 'prior-cell-attempt', - channel: 'agent', - phase: 'memory.experiment', - actor: 'agent-knowledge:memory-experiment:memory', - model: 'memory', - signal: controller.signal, - tags: { - runDir, - scenarioId, - cellId: 'prior-cell', - rep: '0', - runAttemptId: 'prior-attempt', - }, - maximumCharge: { externallyEnforcedMaximumUsd: 0.1 }, - async execute() { - reportStarted() - return await new Promise((resolve) => { - releaseProvider = () => resolve('late result') - }) - }, - receipt: () => ({ - model: 'memory', - inputTokens: 0, - outputTokens: 0, - actualCostUsd: 0.05, - }), - }) - await started - controller.abort(new Error('simulated process exit')) - await abandoned - - const resumedLedger = createRunCostLedger({ storage, runDir, costCeilingUsd: 1 }) - const result = await runAgentMemoryExperiment({ - experimentId: 'interrupted-cell-cost-reporting', - sequences: [sequence], - candidates: [candidate], - runDir, - storage, - costLedger: resumedLedger, - }) - - expect(result.rows[0]).toMatchObject({ candidateId: 'memory', totalCostUsd: 0.2 }) - expect(resumedLedger.summary()).toMatchObject({ totalCalls: 2, totalCostUsd: 0.2 }) - releaseProvider() - await abandonedLedger.waitForIdle() - }) - - it('reconciles interrupted paid calls from every parallel memory branch before resuming', async () => { - const storage = inMemoryCampaignStorage() - const runDir = '/runs/parallel-interrupted-cost-recovery' - const abandonedLedger = createRunCostLedger({ storage, runDir, costCeilingUsd: 1 }) - const controllers = [new AbortController(), new AbortController()] - const releases: Array<() => void> = [] - let started = 0 - let reportStarted: (() => void) | undefined - const bothStarted = new Promise((resolve) => { - reportStarted = resolve - }) - const abandonedCalls = ['one', 'two'].map((branch, index) => - abandonedLedger.runPaidCall({ - callId: `abandoned-${branch}`, - channel: 'agent', - phase: 'memory.train', - actor: `agent-knowledge:memory-experiment:${branch}`, - model: `provider-${branch}`, - signal: controllers[index]!.signal, - maximumCharge: { externallyEnforcedMaximumUsd: 0.1 }, - async execute() { - started += 1 - if (started === 2) reportStarted?.() - return await new Promise((resolve) => { - releases[index] = () => resolve('late provider result') - }) - }, - receipt: () => ({ - model: `provider-${branch}`, - inputTokens: 0, - outputTokens: 0, - actualCostUsd: 0.05, - }), - }), - ) - - await bothStarted - for (const controller of controllers) { - controller.abort(new Error('simulated process exit')) - } - await Promise.all(abandonedCalls) - expect(abandonedLedger.listPending().map((call) => call.state)).toEqual(['late', 'late']) - - const resumedLedger = createRunCostLedger({ storage, runDir, costCeilingUsd: 1 }) - expect(resumedLedger.listPending().map((call) => call.state)).toEqual([ - 'interrupted', - 'interrupted', - ]) - - const result = await runAgentMemoryExperiment({ - experimentId: 'parallel-interrupted-cost-recovery', - sequences: [ - { - id: 'history', - family: 'first-party', - steps: [ - { - id: 'remember', - scope: { agentId: 'worker' }, - writes: [{ kind: 'fact', text: 'resumed fact' }], - probes: [ - { - id: 'recall', - query: 'resumed', - referenceAnswer: 'resumed fact', - }, - ], - }, - ], - }, - ], - candidates: [ - { - id: 'resumed', - ref: 'resumed:v1', - createAdapter: () => createScopedTestAdapter('resumed'), - }, - ], - runDir, - storage, - costLedger: resumedLedger, - }) - - expect(result.rows[0]).toMatchObject({ candidateId: 'resumed', cellsFailed: 0 }) - expect(resumedLedger.summary()).toMatchObject({ - totalCalls: 2, - unresolvedCalls: 0, - totalCostUsd: 0.2, - accountingComplete: true, - }) - expect(resumedLedger.list().map((receipt) => receipt.error)).toEqual([ - expect.stringContaining('charged the reserved maximum'), - expect.stringContaining('charged the reserved maximum'), - ]) - - for (const release of releases) release() - await abandonedLedger.waitForIdle() - }) - - it('does not reuse cells after the candidate implementation reference changes', async () => { - const storage = inMemoryCampaignStorage() - const sequence = { - id: 'candidate-ref', - family: 'first-party' as const, - steps: [ - { - id: 'remember', - scope: { agentId: 'worker' }, - writes: [{ kind: 'fact' as const, text: 'remember me' }], - probes: [ - { - id: 'recall', - query: 'remember', - requiredFacts: [{ id: 'fact', anyOf: ['remember me'] }], - }, - ], - }, - ], - } - let creates = 0 - const run = (ref: string, visible: boolean) => - runAgentMemoryExperiment({ - experimentId: 'candidate-ref-cache', - sequences: [sequence], - candidates: [ - { - id: 'memory', - ref, - createAdapter() { - creates += 1 - const adapter = createScopedTestAdapter(`memory:${ref}`) - if (visible) return adapter - return { - ...adapter, - async search() { - return [] - }, - } - }, - }, - ], - runDir: '/runs/candidate-ref-cache', - storage, - }) - - const first = await run('memory:v1', true) - const cached = await run('memory:v1', false) - const changed = await run('memory:v2', false) - - expect(first.rows[0]?.scoreMean).toBe(1) - expect(cached.campaign.cells[0]?.cached).toBe(true) - expect(cached.rows[0]?.scoreMean).toBe(1) - expect(changed.campaign.cells[0]?.cached).toBe(false) - expect(changed.rows[0]?.scoreMean).toBe(0) - expect(creates).toBe(2) - }) - - it('runs existing ordered memory benchmark cases without reshaping the dataset', async () => { - const sequences = buildAgentMemorySequencesFromBenchmarkCases([ - { - id: 'launch-update', - family: 'longmemeval', - taskKind: 'memory-temporal', - split: 'holdout', - events: [ - { id: 'old', actorId: 'pm', text: 'Launch is April 3.' }, - { id: 'current', actorId: 'pm', text: 'Launch moved to April 17.' }, - ], - prompt: 'When is launch?', - requiredFacts: [{ id: 'current-date', anyOf: ['April 17'] }], - forbiddenFacts: [{ id: 'old-date', anyOf: ['April 3'], obsolete: true }], - expectedEventIds: ['current'], - expectedActorIds: ['pm'], - }, - ]) - const result = await runAgentMemoryExperiment({ - experimentId: 'benchmark-conversion', - sequences, - candidates: [ - { - id: 'literal-memory', - ref: 'literal-memory:v1', - createAdapter: () => createScopedTestAdapter('literal-memory'), - }, - ], - runDir: '/runs/benchmark-conversion', - storage: inMemoryCampaignStorage(), - }) - - expect(sequences[0]?.steps.map((step) => step.id)).toEqual([ - 'event:old', - 'event:current', - 'probe', - ]) - expect(result.rows[0]).toMatchObject({ - scoreMean: 0.75, - totalSequences: 1, - totalProbes: 1, - cellsFailed: 0, - }) - expect(result.rows[0]?.dimensions).toMatchObject({ memory_stale_safe: 0 }) - }) - - it('keeps each history ordered while comparing candidate branches in parallel', async () => { - const storage = inMemoryCampaignStorage() - const snapshots: string[] = [] - const stepOrder = new Map() - let active = 0 - let maxActive = 0 - let release: (() => void) | undefined - const twoActive = new Promise((resolve) => { - release = resolve - }) - const sequence = (id: string) => ({ - id, - family: 'first-party' as const, - split: 'holdout' as const, - steps: [ - { - id: 'research', - scope: { agentId: 'researcher', teamId: 'team-1' }, - writes: [ - { - id: `${id}-event`, - kind: 'fact' as const, - text: `${id} launch date is Friday`, - metadata: { eventId: `${id}-event`, actorId: 'researcher' }, - }, - ], - }, - { - id: 'delivery', - scope: { agentId: 'builder', teamId: 'team-1' }, - probes: [ - { - id: 'launch-date', - query: `${id} launch date`, - requiredFacts: [{ id: 'current', anyOf: [`${id} launch date is Friday`] }], - expectedEventIds: [`${id}-event`], - expectedActorIds: ['researcher'], - }, - ], - }, - ], - }) - - const result = await runAgentMemoryExperiment({ - experimentId: 'team-sharing-vs-private', - sequences: [sequence('alpha'), sequence('beta')], - candidates: [ - { - id: 'private', - ref: 'private:v1', - policy: { read: ['private'], write: 'private' }, - createAdapter: ({ branchId }) => createScopedTestAdapter(`private:${branchId}`), - }, - { - id: 'team', - ref: 'team:v1', - policy: { read: ['team'], write: 'team' }, - createAdapter: ({ branchId }) => createScopedTestAdapter(`team:${branchId}`), - }, - ], - runDir: '/runs/team-sharing-vs-private', - storage, - maxConcurrency: 4, - cleanupBranches: true, - executeStepRef: 'test-runtime/v1', - executeStep: async ({ memory, step }) => { - const order = stepOrder.get(memory.branchId) ?? [] - order.push(step.id) - stepOrder.set(memory.branchId, order) - if (step.id === 'research') { - active += 1 - maxActive = Math.max(maxActive, active) - if (active === 2) release?.() - await twoActive - active -= 1 - } - }, - onBranchSnapshot: ({ snapshot }) => { - snapshots.push(snapshot.digest) - }, - }) - - expect(maxActive).toBeGreaterThanOrEqual(2) - expect(result.rows[0]).toMatchObject({ - rank: 1, - candidateId: 'team', - scoreMean: 1, - passRate: 1, - totalSequences: 2, - totalCells: 2, - totalProbes: 2, - cellsFailed: 0, - }) - expect(result.rows[1]?.candidateId).toBe('private') - expect(result.rows[1]?.scoreMean).toBeLessThan(0.3) - expect(result.campaign.cells).toHaveLength(4) - expect(snapshots).toHaveLength(4) - expect([...stepOrder.values()].every((steps) => steps.join(',') === 'research,delivery')).toBe( - true, - ) - expect(storage.read(result.rankingJsonPath)).toContain('"candidateId": "team"') - expect(storage.read(result.rankingMarkdownPath)).toContain('| 1 | team |') - }) - - it('uses distinct external branch ids for distinct run directories', async () => { - const branchIds: string[] = [] - const sequence = { - id: 'branch-id', - family: 'first-party' as const, - steps: [ - { - id: 'probe', - scope: { agentId: 'worker' }, - probes: [{ id: 'state', query: 'state', referenceAnswer: 'state' }], - }, - ], - } - const run = (runDir: string) => - runAgentMemoryExperiment({ - experimentId: 'same-experiment', - sequences: [sequence], - candidates: [ - { - id: 'memory', - ref: 'memory:v1', - createAdapter({ branchId }) { - branchIds.push(branchId) - return createScopedTestAdapter(branchId) - }, - }, - ], - runDir, - storage: inMemoryCampaignStorage(), - }) - - await run('/runs/branch-id-a') - await run('/runs/branch-id-b') - - expect(branchIds).toHaveLength(2) - expect(branchIds[0]).not.toBe(branchIds[1]) - }) - - it('uses a fresh external branch id for each distributed execution attempt', async () => { - const branchIds: string[] = [] - const run = (runDir: string) => - runAgentMemoryExperiment({ - experimentId: 'distributed-experiment', - experimentRunId: 'distributed-run-17', - sequences: [ - { - id: 'shared-history', - family: 'first-party', - steps: [ - { - id: 'probe', - scope: { agentId: 'worker' }, - probes: [{ id: 'state', query: 'state', referenceAnswer: 'state' }], - }, - ], - }, - ], - candidates: [ - { - id: 'memory', - ref: 'memory:v1', - createAdapter({ branchId }) { - branchIds.push(branchId) - return createScopedTestAdapter(branchId) - }, - }, - ], - runDir, - storage: inMemoryCampaignStorage(), - }) - - await run('/worker-a/run') - await run('/worker-b/run') - - expect(branchIds).toHaveLength(2) - expect(branchIds[0]).not.toBe(branchIds[1]) - }) - - it('waits for timed-out provider work to finish cleanup before returning', async () => { - let releaseWrite!: () => void - const writeMayFinish = new Promise((resolve) => { - releaseWrite = resolve - }) - let reportWriteStarted!: () => void - const writeStarted = new Promise((resolve) => { - reportWriteStarted = resolve - }) - const rows: AgentMemoryHit[] = [] - let clears = 0 - const adapter: AgentMemoryAdapter = { - id: 'slow-provider', - branchIsolation: { mode: 'scoped' }, - async search() { - return [...rows] - }, - async getContext(query) { - return { query, text: rows.map(hitText).join('\n'), hits: [...rows], sourceRecords: [] } - }, - async write(input) { - reportWriteStarted() - await writeMayFinish - const hit = { - id: 'late-write', - uri: 'memory://slow-provider/late-write', - kind: input.kind, - text: input.text, - } - rows.push(hit) - return { accepted: true, id: hit.id, uri: hit.uri, kind: hit.kind } - }, - async clear() { - clears += 1 - rows.length = 0 - }, - } - const run = runAgentMemoryExperiment({ - experimentId: 'timeout-cleanup', - sequences: [ - { - id: 'slow-history', - family: 'first-party', - steps: [ - { - id: 'write', - scope: { agentId: 'worker' }, - writes: [{ kind: 'fact', text: 'late fact' }], - probes: [{ id: 'fact', query: 'fact', referenceAnswer: 'late fact' }], - }, - ], - }, - ], - candidates: [{ id: 'slow', ref: 'slow:v1', createAdapter: () => adapter }], - runDir: '/runs/timeout-cleanup', - storage: inMemoryCampaignStorage(), - dispatchTimeoutMs: 5, - }) - await writeStarted - let settled = false - void run.then( - () => { - settled = true - }, - () => { - settled = true - }, - ) - await new Promise((resolve) => setTimeout(resolve, 15)) - - expect(settled).toBe(false) - releaseWrite() - const result = await run - - expect(result.rows[0]).toMatchObject({ cellsFailed: 1 }) - expect(result.campaign.cells[0]?.error).toContain('dispatch exceeded 5ms') - expect(clears).toBe(1) - expect(rows).toEqual([]) - }) - - it('fails when cleanup after a timed-out provider write fails', async () => { - let releaseWrite!: () => void - const writeMayFinish = new Promise((resolve) => { - releaseWrite = resolve - }) - let reportWriteStarted!: () => void - const writeStarted = new Promise((resolve) => { - reportWriteStarted = resolve - }) - let clears = 0 - const adapter: AgentMemoryAdapter = { - id: 'cleanup-failure', - branchIsolation: { mode: 'scoped' }, - async search() { - return [] - }, - async getContext(query) { - return { query, text: '', hits: [], sourceRecords: [] } - }, - async write(input) { - reportWriteStarted() - await writeMayFinish - return { - accepted: true, - id: 'late-write', - uri: 'memory://cleanup-failure/late-write', - kind: input.kind, - } - }, - async clear() { - clears += 1 - throw new Error('provider cleanup unavailable') - }, - } - const run = runAgentMemoryExperiment({ - experimentId: 'timeout-cleanup-failure', - sequences: [ - { - id: 'slow-history', - family: 'first-party', - steps: [ - { - id: 'write', - scope: { agentId: 'worker' }, - writes: [{ kind: 'fact', text: 'late fact' }], - probes: [{ id: 'fact', query: 'fact', referenceAnswer: 'late fact' }], - }, - ], - }, - ], - candidates: [{ id: 'slow', ref: 'slow:v1', createAdapter: () => adapter }], - runDir: '/runs/timeout-cleanup-failure', - storage: inMemoryCampaignStorage(), - dispatchTimeoutMs: 5, - }) - await writeStarted - await new Promise((resolve) => setTimeout(resolve, 10)) - releaseWrite() - - await expect(run).rejects.toThrow('memory experiment cleanup failed after dispatch') - }) - - it('recovers an unfinished provider branch before retrying the history', async () => { - const storage = inMemoryCampaignStorage() - const rows = new Map() - const operations: string[] = [] - const branchIds: Record<'first' | 'recovery' | 'retry', string | undefined> = { - first: undefined, - recovery: undefined, - retry: undefined, - } - let firstExecution = true - const sequence = { - id: 'recover-history', - family: 'first-party' as const, - steps: [ - { - id: 'remember', - scope: { agentId: 'worker' }, - writes: [{ kind: 'fact' as const, text: 'sensitive provider fact' }], - probes: [ - { - id: 'recall', - query: 'provider fact', - referenceAnswer: 'sensitive provider fact', - }, - ], - }, - ], - } - const candidate = { - id: 'recoverable', - ref: 'recoverable:v1', - externalRecoveryCostUsdPerAttempt: 0.1, - createAdapter({ branchId, purpose }: { branchId: string; purpose: 'execute' | 'recovery' }) { - const executionLabel = - purpose === 'recovery' ? 'recovery' : firstExecution ? 'first' : 'retry' - branchIds[executionLabel] = branchId - operations.push(`create:${executionLabel}`) - let clearCalls = 0 - const adapter: AgentMemoryAdapter = { - id: 'recoverable-provider', - branchIsolation: { mode: 'scoped' }, - async search(_query, options) { - return [...(rows.get(options.scope?.namespace ?? '') ?? [])] - }, - async getContext(query, options) { - const hits = await adapter.search(query, options) - return { query, text: hits.map(hitText).join('\n'), hits, sourceRecords: [] } - }, - async write(input) { - operations.push(`write:${executionLabel}`) - const namespace = input.scope?.namespace ?? '' - const hit = { - id: input.id ?? `${executionLabel}-fact`, - uri: `memory://recoverable/${executionLabel}`, - kind: input.kind, - text: input.text, - } - rows.set(namespace, [...(rows.get(namespace) ?? []), hit]) - return { accepted: true, id: hit.id, uri: hit.uri, kind: hit.kind } - }, - async clear(scope) { - clearCalls += 1 - operations.push(`clear:${executionLabel}`) - if (executionLabel === 'first' && clearCalls === 1) { - throw new Error('provider cleanup unavailable') - } - rows.delete(scope?.namespace ?? '') - }, - } - if (purpose === 'execute') firstExecution = false - return adapter - }, - } - const run = () => - runAgentMemoryExperiment({ - experimentId: 'restart-recovery', - sequences: [sequence], - candidates: [candidate], - runDir: '/runs/restart-recovery', - storage, - costCeiling: 1, - }) - - await expect(run()).rejects.toThrow('memory experiment cleanup failed after dispatch') - expect(rows.size).toBe(1) - - const result = await run() - - expect(result.rows[0]).toMatchObject({ candidateId: 'recoverable', cellsFailed: 0 }) - expect(result.rows[0]?.totalCostUsd).toBe(0.1) - expect(result.campaign.aggregates.totalCostUsd).toBe(0.1) - expect(branchIds.recovery).toBe(branchIds.first) - expect(branchIds.retry).not.toBe(branchIds.first) - expect(operations.indexOf('clear:recovery')).toBeLessThan(operations.indexOf('write:retry')) - expect(rows.size).toBe(0) - const attemptEvents = storage - .read(result.attemptLogPath)! - .trim() - .split('\n') - .map((line) => JSON.parse(line) as { status: string; recovery: boolean }) - expect(attemptEvents.map(({ status, recovery }) => ({ status, recovery }))).toEqual([ - { status: 'started', recovery: false }, - { status: 'cleaned', recovery: true }, - { status: 'started', recovery: false }, - { status: 'cleaned', recovery: false }, - ]) - }) - - it('closes and disposes a recovery adapter that arrives after its factory timeout', async () => { - const storage = inMemoryCampaignStorage() - const runDir = '/runs/late-recovery-adapter' - storage.write( - `${runDir}/memory-attempts.jsonl`, - `${JSON.stringify({ - schema: 2, - status: 'started', - branchId: 'unfinished-branch', - candidateId: 'memory', - candidateRef: 'memory:v1', - sequenceId: 'history', - rep: 0, - seed: 42, - cleanupBranches: true, - externalCostUsdPerSequence: 0, - externalRecoveryCostUsdPerAttempt: 0, - recordedAt: '2026-01-01T00:00:00.000Z', - recovery: false, - })}\n`, - ) - const sequence = { - id: 'history', - family: 'first-party' as const, - steps: [ - { - id: 'remember', - scope: { agentId: 'worker' }, - writes: [{ kind: 'fact' as const, text: 'fact' }], - probes: [{ id: 'recall', query: 'fact', referenceAnswer: 'fact' }], - }, - ], - } - let resolveCreation!: (adapter: AgentMemoryAdapter) => void - const creation = new Promise((resolve) => { - resolveCreation = resolve - }) - let reportDisposed!: () => void - const disposed = new Promise((resolve) => { - reportDisposed = resolve - }) - let closeCalls = 0 - let disposeCalls = 0 - const lateAdapter = createScopedTestAdapter('memory-provider') - lateAdapter.close = async () => { - closeCalls += 1 - } - - await expect( - runAgentMemoryExperiment({ - experimentId: 'late-recovery-adapter', - sequences: [sequence], - candidates: [ - { - id: 'memory', - ref: 'memory:v1', - createAdapter: ({ purpose }) => - purpose === 'recovery' ? creation : createScopedTestAdapter('memory-provider'), - async disposeAdapter(adapter) { - expect(adapter).toBe(lateAdapter) - disposeCalls += 1 - reportDisposed() - }, - }, - ], - runDir, - storage, - cleanupTimeoutMs: 10, - }), - ).rejects.toThrow("memory: abandoned memory branch 'unfinished-branch' recovery failed") - - resolveCreation(lateAdapter) - await disposed - expect({ closeCalls, disposeCalls }).toEqual({ closeCalls: 1, disposeCalls: 1 }) - }) - - it('reconciles a crash after provider recovery but before its cost receipt', async () => { - const storage = inMemoryCampaignStorage() - const append = storage.append!.bind(storage) - let failRecoveryReceipt = false - storage.append = (path, value, expectedBytes) => { - if ( - failRecoveryReceipt && - path.endsWith('/cost-ledger.jsonl') && - value.includes('"status":"settled"') && - value.includes('agent-knowledge:memory-recovery') - ) { - failRecoveryReceipt = false - throw new Error('simulated process exit before recovery receipt') - } - return append(path, value, expectedBytes) - } - - let firstExecution = true - let recoveryClears = 0 - const candidate = { - id: 'crash-recoverable', - ref: 'crash-recoverable:v1', - externalCostUsdPerSequence: 0.1, - externalRecoveryCostUsdPerAttempt: 0.1, - createAdapter({ purpose }: { purpose: 'execute' | 'recovery' }) { - const adapter = createScopedTestAdapter(`crash-recoverable:${purpose}`) - const clear = adapter.clear! - let clearCalls = 0 - const failThisExecution = purpose === 'execute' && firstExecution - if (purpose === 'execute') firstExecution = false - adapter.clear = async (scope) => { - clearCalls += 1 - if (purpose === 'recovery') recoveryClears += 1 - if (failThisExecution && clearCalls === 1) { - throw new Error('leave the first branch active') - } - await clear(scope) - } - return adapter - }, - } - const run = () => - runAgentMemoryExperiment({ - experimentId: 'recovery-receipt-crash', - sequences: [ - { - id: 'history', - family: 'first-party', - steps: [ - { - id: 'remember', - scope: { agentId: 'worker' }, - writes: [{ kind: 'fact', text: 'durable recovery fact' }], - probes: [ - { - id: 'recall', - query: 'durable', - referenceAnswer: 'durable recovery fact', - }, - ], - }, - ], - }, - ], - candidates: [candidate], - runDir: '/runs/recovery-receipt-crash', - storage, - costCeiling: 1, - }) - - await expect(run()).rejects.toThrow('memory experiment cleanup failed after dispatch') - failRecoveryReceipt = true - await expect(run()).rejects.toThrow('failed to persist') - expect( - storage.read('/runs/recovery-receipt-crash/memory-attempts.jsonl')?.trim().split('\n'), - ).toHaveLength(2) - - const result = await run() - const costLedger = createRunCostLedger({ - storage, - runDir: '/runs/recovery-receipt-crash', - costCeilingUsd: 1, - }) - - expect(result.rows[0]).toMatchObject({ - candidateId: 'crash-recoverable', - cellsFailed: 0, - }) - expect(result.rows[0]?.totalCostUsd).toBeCloseTo(0.3) - expect(recoveryClears).toBe(1) - expect(costLedger.summary()).toMatchObject({ unresolvedCalls: 0, accountingComplete: true }) - expect(costLedger.summary().totalCostUsd).toBeCloseTo(0.3) - }) - - it('recovers independent abandoned branches in parallel before retrying them', async () => { - const storage = inMemoryCampaignStorage() - let phase: 'leave-active' | 'recover' = 'leave-active' - let activeRecoveries = 0 - let maxActiveRecoveries = 0 - let releaseRecoveries: (() => void) | undefined - const recoveriesStarted = new Promise((resolve) => { - releaseRecoveries = resolve - }) - const sequences = ['one', 'two'].map((id) => ({ - id, - family: 'first-party' as const, - steps: [ - { - id: 'remember', - scope: { agentId: 'worker' }, - writes: [{ kind: 'fact' as const, text: `${id} fact` }], - probes: [{ id: 'recall', query: id, referenceAnswer: `${id} fact` }], - }, - ], - })) - const run = () => - runAgentMemoryExperiment({ - experimentId: 'parallel-recovery', - sequences, - candidates: [ - { - id: 'memory', - ref: 'memory:v1', - async createAdapter({ purpose }) { - if (purpose === 'recovery') { - activeRecoveries += 1 - maxActiveRecoveries = Math.max(maxActiveRecoveries, activeRecoveries) - if (activeRecoveries === 2) releaseRecoveries?.() - await recoveriesStarted - activeRecoveries -= 1 - return null - } - if (phase === 'leave-active') return null - return createScopedTestAdapter('parallel-recovery') - }, - }, - ], - runDir: '/runs/parallel-recovery', - storage, - maxConcurrency: 2, - }) - - await expect(run()).rejects.toThrow('memory experiment cleanup failed after dispatch') - phase = 'recover' - const result = await run() - - expect(maxActiveRecoveries).toBe(2) - expect(result.rows[0]).toMatchObject({ candidateId: 'memory', cellsFailed: 0 }) - const recovered = storage - .read(result.attemptLogPath)! - .trim() - .split('\n') - .map((line) => JSON.parse(line) as { status: string; recovery: boolean }) - .filter((event) => event.status === 'cleaned' && event.recovery) - expect(recovered).toHaveLength(2) - }) - - it('closes an attempt whose provider was never created before retrying it', async () => { - const storage = inMemoryCampaignStorage() - const purposes: Array<'execute' | 'recovery'> = [] - let executionCalls = 0 - const sequence = { - id: 'provider-creation', - family: 'first-party' as const, - steps: [ - { - id: 'remember', - scope: { agentId: 'worker' }, - writes: [{ kind: 'fact' as const, text: 'created provider fact' }], - probes: [ - { - id: 'recall', - query: 'provider fact', - referenceAnswer: 'created provider fact', - }, - ], - }, - ], - } - const run = () => - runAgentMemoryExperiment({ - experimentId: 'provider-creation-recovery', - sequences: [sequence], - candidates: [ - { - id: 'sometimes-created', - ref: 'sometimes-created:v1', - externalRecoveryCostUsdPerAttempt: 0.1, - createAdapter({ purpose }) { - purposes.push(purpose) - if (purpose === 'recovery') return null - executionCalls += 1 - if (executionCalls === 1) return null - return createScopedTestAdapter('created-provider') - }, - }, - ], - runDir: '/runs/provider-creation-recovery', - storage, - costCeiling: 1, - }) - - await expect(run()).rejects.toThrow('memory experiment cleanup failed after dispatch') - const result = await run() - - expect(result.rows[0]).toMatchObject({ candidateId: 'sometimes-created', cellsFailed: 0 }) - expect(result.rows[0]?.totalCostUsd).toBe(0) - expect(purposes).toEqual(['execute', 'recovery', 'execute']) - const attemptEvents = storage - .read(result.attemptLogPath)! - .trim() - .split('\n') - .map((line) => JSON.parse(line) as { status: string; recovery: boolean }) - expect(attemptEvents.map(({ status, recovery }) => ({ status, recovery }))).toEqual([ - { status: 'started', recovery: false }, - { status: 'cleaned', recovery: true }, - { status: 'started', recovery: false }, - { status: 'cleaned', recovery: false }, - ]) - }) - - it('uses a retired candidate only to clean its unfinished branch', async () => { - const storage = inMemoryCampaignStorage() - const runDir = '/runs/retired-recovery-candidate' - const sequence = { - id: 'history', - family: 'first-party' as const, - steps: [ - { - id: 'remember', - scope: { agentId: 'worker' }, - writes: [{ kind: 'fact' as const, text: 'active fact' }], - probes: [{ id: 'recall', query: 'active', referenceAnswer: 'active fact' }], - }, - ], - } - storage.write( - `${runDir}/memory-attempts.jsonl`, - `${JSON.stringify({ - schema: 2, - status: 'started', - branchId: 'retired-branch', - candidateId: 'retired', - candidateRef: 'retired:v1', - sequenceId: sequence.id, - rep: 0, - seed: 1, - cleanupBranches: true, - externalCostUsdPerSequence: 0, - externalRecoveryCostUsdPerAttempt: 0.1, - recordedAt: '2026-01-01T00:00:00.000Z', - recovery: false, - })}\n`, - ) - const purposes: string[] = [] - let retiredClears = 0 - const result = await runAgentMemoryExperiment({ - experimentId: 'retired-recovery-candidate', - sequences: [sequence], - candidates: [ - { - id: 'active', - ref: 'active:v1', - createAdapter({ purpose }) { - purposes.push(`active:${purpose}`) - return createScopedTestAdapter('active') - }, - }, - ], - recoveryCandidates: [ - { - id: 'retired', - ref: 'retired:v1', - externalRecoveryCostUsdPerAttempt: 0.1, - createAdapter({ purpose }) { - purposes.push(`retired:${purpose}`) - const adapter = createScopedTestAdapter('retired') - adapter.clear = async () => { - retiredClears += 1 - } - return adapter - }, - }, - ], - runDir, - storage, - resumable: false, - costCeiling: 1, - }) - - expect(result.rows).toHaveLength(1) - expect(result.rows[0]?.candidateId).toBe('active') - expect(result.rows[0]?.totalCostUsd).toBe(0) - expect(result).toMatchObject({ totalCostUsd: 0.1, unrankedRecoveryCostUsd: 0.1 }) - expect(purposes).toEqual(['retired:recovery', 'active:execute']) - expect(retiredClears).toBeGreaterThan(0) - }) - - it('refuses to hide an unfinished branch when candidate cost settings change', async () => { - const storage = inMemoryCampaignStorage() - const runDir = '/runs/changed-recovery-costs' - const sequence = { - id: 'history', - family: 'first-party' as const, - steps: [{ id: 'probe', probes: [{ id: 'recall', query: 'fact', referenceAnswer: 'fact' }] }], - } - storage.write( - `${runDir}/memory-attempts.jsonl`, - `${JSON.stringify({ - schema: 2, - status: 'started', - branchId: 'unfinished-branch', - candidateId: 'memory', - candidateRef: 'memory:v1', - sequenceId: sequence.id, - rep: 0, - seed: 1, - cleanupBranches: true, - externalCostUsdPerSequence: 0, - externalRecoveryCostUsdPerAttempt: 0, - recordedAt: '2026-01-01T00:00:00.000Z', - recovery: false, - })}\n`, - ) - let adapterCreates = 0 - - await expect( - runAgentMemoryExperiment({ - experimentId: 'changed-recovery-costs', - sequences: [sequence], - candidates: [ - { - id: 'memory', - ref: 'memory:v1', - externalCostUsdPerSequence: 0.1, - createAdapter() { - adapterCreates += 1 - return createScopedTestAdapter('memory') - }, - }, - ], - runDir, - storage, - costCeiling: 1, - }), - ).rejects.toThrow('candidate cost settings changed') - - expect(adapterCreates).toBe(0) - expect(storage.read(`${runDir}/memory-attempts.jsonl`)?.trim().split('\n')).toHaveLength(1) - }) - - it('refuses a recovery backlog larger than maxRecoveryAttempts', async () => { - const storage = inMemoryCampaignStorage() - const runDir = '/runs/recovery-backlog-limit' - const sequence = { - id: 'history', - family: 'first-party' as const, - steps: [ - { - id: 'probe', - scope: { agentId: 'worker' }, - probes: [{ id: 'recall', query: 'fact', referenceAnswer: 'fact' }], - }, - ], - } - storage.write( - `${runDir}/memory-attempts.jsonl`, - `${[ - { - schema: 2, - status: 'started', - branchId: 'branch-1', - candidateId: 'memory', - candidateRef: 'memory:v1', - sequenceId: sequence.id, - rep: 0, - seed: 1, - cleanupBranches: true, - externalCostUsdPerSequence: 0, - externalRecoveryCostUsdPerAttempt: 0, - recordedAt: '2026-01-01T00:00:00.000Z', - recovery: false, - }, - { - schema: 2, - status: 'started', - branchId: 'branch-2', - candidateId: 'memory', - candidateRef: 'memory:v1', - sequenceId: sequence.id, - rep: 0, - seed: 2, - cleanupBranches: true, - externalCostUsdPerSequence: 0, - externalRecoveryCostUsdPerAttempt: 0, - recordedAt: '2026-01-01T00:00:00.000Z', - recovery: false, - }, - ] - .map((event) => JSON.stringify(event)) - .join('\n')}\n`, - ) - let providerCreates = 0 - - await expect( - runAgentMemoryExperiment({ - experimentId: 'recovery-backlog-limit', - sequences: [sequence], - candidates: [ - { - id: 'memory', - ref: 'memory:v1', - createAdapter() { - providerCreates += 1 - return createScopedTestAdapter('memory') - }, - }, - ], - runDir, - storage, - maxRecoveryAttempts: 1, - }), - ).rejects.toThrow('2 unfinished attempts; maxRecoveryAttempts is 1') - expect(providerCreates).toBe(0) - }) - - it('bounds repeated provider recovery across process restarts', async () => { - const storage = inMemoryCampaignStorage() - const runDir = '/runs/recovery-retry-limit' - const sequence = { - id: 'history', - family: 'first-party' as const, - steps: [{ id: 'probe', probes: [{ id: 'recall', query: 'fact', referenceAnswer: 'fact' }] }], - } - storage.write( - `${runDir}/memory-attempts.jsonl`, - `${JSON.stringify({ - schema: 2, - status: 'started', - branchId: 'unfinished-branch', - candidateId: 'memory', - candidateRef: 'memory:v1', - sequenceId: sequence.id, - rep: 0, - seed: 1, - cleanupBranches: true, - externalCostUsdPerSequence: 0, - externalRecoveryCostUsdPerAttempt: 0, - recordedAt: '2026-01-01T00:00:00.000Z', - recovery: false, - })}\n`, - ) - let recoveryCreates = 0 - const run = () => - runAgentMemoryExperiment({ - experimentId: 'recovery-retry-limit', - sequences: [sequence], - candidates: [ - { - id: 'memory', - ref: 'memory:v1', - createAdapter({ purpose }) { - if (purpose === 'recovery') recoveryCreates += 1 - throw new Error('provider recovery unavailable') - }, - }, - ], - runDir, - storage, - maxRecoveryRetriesPerAttempt: 2, - }) - - await expect(run()).rejects.toThrow( - "abandoned memory branch 'unfinished-branch' recovery failed", - ) - await expect(run()).rejects.toThrow( - "abandoned memory branch 'unfinished-branch' recovery failed", - ) - await expect(run()).rejects.toThrow('exhausted 2 recovery attempts') - expect(recoveryCreates).toBe(2) - expect( - storage.read(`${runDir}/memory-recovery-attempts.jsonl`)?.trim().split('\n'), - ).toHaveLength(2) - }) - - it('allows one controller per run while its history workers run in parallel', async () => { - const storage = inMemoryCampaignStorage() - let reportStarted: (() => void) | undefined - const started = new Promise((resolve) => { - reportStarted = resolve - }) - let releaseWorker: (() => void) | undefined - const continueWorker = new Promise((resolve) => { - releaseWorker = resolve - }) - const options: RunAgentMemoryExperimentOptions = { - experimentId: 'single-controller', - sequences: [ - { - id: 'one', - family: 'first-party', - steps: [ - { - id: 'work', - scope: { agentId: 'worker' }, - writes: [{ kind: 'fact', text: 'parallel worker fact' }], - probes: [ - { - id: 'recall', - query: 'worker fact', - referenceAnswer: 'parallel worker fact', - }, - ], - }, - ], - }, - ], - candidates: [ - { - id: 'memory', - ref: 'memory:v1', - createAdapter: () => createScopedTestAdapter('single-controller'), - }, - ], - runDir: '/runs/single-controller', - storage, - executeStepRef: 'blocking-step:v1', - async executeStep() { - reportStarted?.() - await continueWorker - }, - } - - const firstRun = runAgentMemoryExperiment(options) - await started - await expect( - runAgentMemoryExperiment({ ...options, storage: inMemoryCampaignStorage() }), - ).rejects.toThrow('active controller') - releaseWorker?.() - const result = await firstRun - - expect(result.rows[0]).toMatchObject({ candidateId: 'memory', cellsFailed: 0 }) - }) - - it('leaves a lost controller branch for the next owner and accepts duplicate cleanup receipts', async () => { - const storage = inMemoryCampaignStorage() - let firstControllerOwned = true - let expireFirstController = true - let controllerCount = 0 - let searches = 0 - let clears = 0 - let closes = 0 - let factVisible = false - const purposes: string[] = [] - const options: RunAgentMemoryExperimentOptions = { - experimentId: 'lost-controller', - sequences: [ - { - id: 'one', - family: 'first-party', - steps: [ - { - id: 'work', - scope: { agentId: 'worker' }, - writes: [{ kind: 'fact', text: 'must be cleaned' }], - probes: [{ id: 'probe', query: 'must', referenceAnswer: 'must be cleaned' }], - }, - ], - }, - ], - candidates: [ - { - id: 'memory', - ref: 'memory:v1', - createAdapter({ purpose }) { - purposes.push(purpose) - const adapter: AgentMemoryAdapter = { - id: 'lost-controller', - branchIsolation: { mode: 'scoped' }, - async search() { - searches += 1 - return factVisible - ? [ - { - id: 'write', - uri: 'memory://lost-controller/write', - kind: 'fact', - text: 'must be cleaned', - }, - ] - : [] - }, - async getContext(query, searchOptions) { - const hits = await adapter.search(query, searchOptions) - return { query, text: hits.map(hitText).join('\n'), hits, sourceRecords: [] } - }, - async write(input) { - factVisible = true - return { - accepted: true, - id: 'write', - uri: 'memory://lost-controller/write', - kind: input.kind, - } - }, - async clear() { - clears += 1 - factVisible = false - }, - async close() { - closes += 1 - }, - } - return adapter - }, - }, - ], - runDir: '/runs/lost-controller', - storage, - acquireRunLease: async () => { - controllerCount += 1 - const controller = controllerCount - return { - assertOwned() { - if (controller === 1 && !firstControllerOwned) { - throw new Error('controller ownership expired') - } - }, - release() {}, - } - }, - executeStepRef: 'expire-controller:v1', - async executeStep() { - if (expireFirstController) { - expireFirstController = false - firstControllerOwned = false - } - }, - } - const run = () => runAgentMemoryExperiment(options) - - await expect(run()).rejects.toThrow('controller ownership expired') - expect(searches).toBe(0) - expect(clears).toBe(0) - expect(closes).toBe(1) - expect(factVisible).toBe(true) - - const result = await run() - expect(result.rows[0]).toMatchObject({ candidateId: 'memory', cellsFailed: 0 }) - expect(purposes).toEqual(['execute', 'recovery', 'execute']) - expect(clears).toBe(2) - expect(closes).toBe(3) - expect(factVisible).toBe(false) - - const journal = storage.read(result.attemptLogPath)! - const lines = journal.trim().split('\n') - const last = lines.at(-1)! - expect(storage.append!(result.attemptLogPath, `${last}\n`, Buffer.byteLength(journal))).toBe( - Buffer.byteLength(journal) + Buffer.byteLength(`${last}\n`), - ) - const callsBeforeCachedRun = { - purposes: purposes.length, - searches, - clears, - closes, - } - const cached = await run() - expect(cached.campaign.aggregates.cellsCached).toBe(1) - expect({ purposes: purposes.length, searches, clears, closes }).toEqual(callsBeforeCachedRun) - }) - - it('preserves both the run failure and controller release failure', async () => { - const error = await runAgentMemoryExperiment({ - experimentId: 'run-and-release-failure', - sequences: [ - { - id: 'one', - family: 'first-party', - steps: [ - { - id: 'probe', - probes: [{ id: 'probe', query: 'state', referenceAnswer: 'state' }], - }, - ], - }, - ], - candidates: [ - { - id: 'unused', - ref: 'unused:v1', - createAdapter: () => createScopedTestAdapter('unused'), - }, - ], - runDir: '/runs/run-and-release-failure', - storage: inMemoryCampaignStorage(), - acquireRunLease: async () => ({ - assertOwned() { - throw new Error('run ownership failed') - }, - release() { - throw new Error('release failed') - }, - }), - }).catch((caught: unknown) => caught) - - expect(error).toBeInstanceOf(AggregateError) - expect((error as AggregateError).errors.map((item) => String(item))).toEqual([ - 'Error: run ownership failed', - 'Error: release failed', - ]) - }) - - it('uses the same random seed for every candidate on one history and repetition', async () => { - const seeds = new Map() - const sequence = { - id: 'paired-history', - family: 'first-party' as const, - steps: [ - { - id: 'probe', - scope: { agentId: 'worker' }, - probes: [{ id: 'state', query: 'state', referenceAnswer: 'state' }], - }, - ], - } - await runAgentMemoryExperiment({ - experimentId: 'paired-seeds', - sequences: [sequence], - candidates: ['a', 'b'].map((candidateId) => ({ - id: candidateId, - ref: `${candidateId}:v1`, - createAdapter: ({ sequence: candidateSequence, rep, seed }) => { - seeds.set(`${candidateId}:${candidateSequence.id}:${rep}`, seed) - return createScopedTestAdapter(`${candidateId}:${rep}`) - }, - })), - reps: 2, - runDir: '/runs/paired-seeds', - storage: inMemoryCampaignStorage(), - maxConcurrency: 4, - }) - - expect(seeds.get('a:paired-history:0')).toBe(seeds.get('b:paired-history:0')) - expect(seeds.get('a:paired-history:1')).toBe(seeds.get('b:paired-history:1')) - expect(seeds.get('a:paired-history:0')).not.toBe(seeds.get('a:paired-history:1')) - }) - - it('fails the experiment when accepted writes cannot be cleared after a failed step', async () => { - let clears = 0 - let closes = 0 - let disposals = 0 - const run = runAgentMemoryExperiment({ - experimentId: 'failed-step-cleanup', - sequences: [ - { - id: 'failure', - family: 'first-party', - steps: [ - { - id: 'write-then-fail', - scope: { agentId: 'worker' }, - writes: [{ kind: 'fact', text: 'partial state' }], - probes: [{ id: 'state', query: 'partial state', referenceAnswer: 'partial state' }], - }, - ], - }, - ], - candidates: [ - { - id: 'scoped', - ref: 'scoped:v1', - createAdapter() { - const base = createScopedTestAdapter('failure') - return { - ...base, - async clear(scope) { - clears += 1 - await base.clear?.(scope) - throw new Error('provider clear failed') - }, - async close() { - closes += 1 - }, - } - }, - async disposeAdapter() { - disposals += 1 - }, - }, - ], - runDir: '/runs/failed-step-cleanup', - storage: inMemoryCampaignStorage(), - cleanupBranches: true, - executeStepRef: 'failing-worker/v1', - async executeStep() { - throw new Error('worker failed') - }, - }) - - await expect(run).rejects.toThrow('memory experiment cleanup failed after dispatch') - expect(clears).toBe(1) - expect(closes).toBe(1) - expect(disposals).toBe(1) - }) - - it('cleans an unjournaled provider side effect and ranks failed histories below complete ones', async () => { - let dirtyRows = 0 - let clears = 0 - const result = await runAgentMemoryExperiment({ - experimentId: 'provider-side-effect-cleanup', - sequences: [ - { - id: 'one', - family: 'first-party', - steps: [ - { - id: 'remember', - scope: { agentId: 'worker' }, - writes: [{ kind: 'fact', text: 'Launch is Friday.' }], - probes: [{ id: 'launch', query: 'launch', referenceAnswer: 'Launch is Friday.' }], - }, - ], - }, - ], - candidates: [ - { - id: 'side-effect-then-error', - ref: 'side-effect-then-error:v1', - createAdapter: () => ({ - id: 'side-effect-then-error', - branchIsolation: { mode: 'scoped' }, - async search() { - return [] - }, - async getContext(query) { - return { query, text: '', hits: [], sourceRecords: [] } - }, - async write() { - dirtyRows += 1 - throw new Error('provider disconnected after commit') - }, - async clear() { - clears += 1 - dirtyRows = 0 - }, - }), - }, - { - id: 'complete', - ref: 'complete:v1', - createAdapter: () => createScopedTestAdapter('complete'), - }, - ], - runDir: '/runs/provider-side-effect-cleanup', - storage: inMemoryCampaignStorage(), - cleanupBranches: true, - maxConcurrency: 2, - }) - - expect(dirtyRows).toBe(0) - expect(clears).toBe(1) - expect(result.rows[0]).toMatchObject({ - candidateId: 'complete', - cellsFailed: 0, - scoreMean: 1, - }) - expect(result.rows[1]).toMatchObject({ - candidateId: 'side-effect-then-error', - cellsFailed: 1, - scoreMean: 0, - passRate: 0, - }) - }) - - it('closes and disposes an adapter when branch validation fails', async () => { - let closes = 0 - let disposals = 0 - const result = await runAgentMemoryExperiment({ - experimentId: 'invalid-branch-cleanup', - sequences: [ - { - id: 'one', - family: 'first-party', - steps: [ - { - id: 'probe', - scope: { agentId: 'worker' }, - probes: [{ id: 'state', query: 'state', referenceAnswer: 'state' }], - }, - ], - }, - ], - candidates: [ - { - id: 'legacy', - ref: 'legacy:v1', - createAdapter() { - const { branchIsolation: _branchIsolation, ...legacy } = - createScopedTestAdapter('legacy') - return { - ...legacy, - async close() { - closes += 1 - }, - } - }, - async disposeAdapter() { - disposals += 1 - }, - }, - ], - runDir: '/runs/invalid-branch-cleanup', - storage: inMemoryCampaignStorage(), - cleanupBranches: false, - }) - - expect(result.rows[0]).toMatchObject({ cellsFailed: 1 }) - expect(result.campaign.cells[0]?.error).toContain('adapter must declare branchIsolation') - expect(closes).toBe(1) - expect(disposals).toBe(1) - }) - - it('fails closed when cleanup is requested for an adapter without scoped clear', async () => { - let disposals = 0 - const result = await runAgentMemoryExperiment({ - experimentId: 'unsupported-cleanup', - sequences: [ - { - id: 'one', - family: 'first-party', - steps: [ - { - id: 'probe', - scope: { agentId: 'worker' }, - probes: [{ id: 'state', query: 'state', referenceAnswer: 'state' }], - }, - ], - }, - ], - candidates: [ - { - id: 'no-clear', - ref: 'no-clear:v1', - createAdapter() { - const { clear: _clear, ...adapter } = createScopedTestAdapter('no-clear') - return adapter - }, - async disposeAdapter() { - disposals += 1 - }, - }, - ], - runDir: '/runs/unsupported-cleanup', - storage: inMemoryCampaignStorage(), - cleanupBranches: true, - }) - - expect(result.rows[0]).toMatchObject({ cellsFailed: 1 }) - expect(result.campaign.cells[0]?.error).toContain('requires an adapter with scoped clear') - expect(disposals).toBe(1) - }) - - it('rejects dynamic writes whose scope cannot be cleaned after a crash', async () => { - let providerWrites = 0 - const result = await runAgentMemoryExperiment({ - experimentId: 'undeclared-dynamic-scope', - sequences: [ - { - id: 'one', - family: 'first-party', - steps: [ - { - id: 'agent-step', - scope: { agentId: 'declared' }, - probes: [{ id: 'state', query: 'state', referenceAnswer: 'state' }], - }, - ], - }, - ], - candidates: [ - { - id: 'scoped', - ref: 'scoped:v1', - createAdapter: () => - createScopedTestAdapter('scoped', async () => { - providerWrites += 1 - }), - }, - ], - runDir: '/runs/undeclared-dynamic-scope', - storage: inMemoryCampaignStorage(), - cleanupBranches: true, - executeStepRef: 'dynamic-scope-test/v1', - executeStep: async ({ memory }) => { - await memory.write({ - kind: 'fact', - text: 'must not reach the provider', - scope: { agentId: 'undeclared' }, - }) - }, - }) - - expect(result.rows[0]).toMatchObject({ cellsFailed: 1 }) - expect(result.campaign.cells[0]?.error).toContain( - 'write scope was not declared in the experiment sequence cleanupScopes', - ) - expect(providerWrites).toBe(0) - }) - - it('requires an external-state disposer when scoped cleanup is disabled', async () => { - let creates = 0 - await expect( - runAgentMemoryExperiment({ - experimentId: 'missing-disposer', - sequences: [ - { - id: 'one', - family: 'first-party', - steps: [ - { - id: 'probe', - scope: { agentId: 'worker' }, - probes: [{ id: 'state', query: 'state', referenceAnswer: 'state' }], - }, - ], - }, - ], - candidates: [ - { - id: 'persistent', - ref: 'persistent:v1', - createAdapter() { - creates += 1 - return createScopedTestAdapter('persistent') - }, - }, - ], - runDir: '/runs/missing-disposer', - storage: inMemoryCampaignStorage(), - cleanupBranches: false, - }), - ).rejects.toThrow('requires disposeAdapter') - - expect(creates).toBe(0) - }) - - it('rejects probes that cannot distinguish a useful memory system', async () => { - await expect( - runAgentMemoryExperiment({ - experimentId: 'no-target', - sequences: [ - { - id: 'one', - family: 'first-party', - steps: [ - { - id: 'probe', - scope: { agentId: 'worker' }, - probes: [{ id: 'state', query: 'state' }], - }, - ], - }, - ], - candidates: [ - { - id: 'memory', - ref: 'memory:v1', - createAdapter: () => createScopedTestAdapter('memory'), - }, - ], - runDir: '/runs/no-target', - storage: inMemoryCampaignStorage(), - }), - ).rejects.toThrow('has no measurable target') - }) -}) - -describe('agent memory improvement', () => { - it('fails fast when a JavaScript caller uses the removed onPromote option', async () => { - await expect( - runAgentMemoryImprovementRaw({ - onPromote() {}, - } as unknown as RunAgentMemoryImprovementOptions), - ).rejects.toThrow( - 'onPromote was removed; use activation.readCurrent and activation.compareAndSet', - ) - }) - - it('requires an explicit controller policy for custom improvement storage', async () => { - await expect( - runAgentMemoryImprovementRaw({ - experimentId: 'custom-improvement-storage', - trainSequences: [improvementSequence('train', 'train')], - holdoutSequences: [improvementSequence('holdout', 'holdout')], - seeds: [ - { - config: { mode: 'baseline' }, - track: 'baseline', - proposer: 'default', - }, - ], - createCandidate: () => ({ - ref: 'memory:v1', - createAdapter: () => createScopedTestAdapter('memory'), - }), - proposer: { kind: 'noop', propose: async () => [] }, - improvementRef: 'custom-improvement-storage:v1', - budget: { maxSteps: 1 }, - runDir: '/runs/custom-improvement-storage', - storage: inMemoryCampaignStorage(), - }), - ).rejects.toThrow("requires acquireRunLease or controllerMode='process-local'") - }) - - it('searches isolated configs and activates only a fresh holdout win', async () => { - type Config = { visibility: 'private' | 'team' | 'shared' } - const storage = inMemoryCampaignStorage() - const promoted: Config[] = [] - let activeConfig: Config = { visibility: 'private' } - const activationIds: string[] = [] - const contenderIds = new Set() - const activeContenderCalls = new Map() - let maxConcurrentConfigs = 0 - let reportContendersActive: (() => void) | undefined - const contendersActive = new Promise((resolve) => { - reportContendersActive = resolve - }) - let releaseContenders: (() => void) | undefined - const continueContenders = new Promise((resolve) => { - releaseContenders = resolve - }) - let proposalCalls = 0 - const proposer: SurfaceProposer = { - kind: 'team-sharing-proposer', - async propose() { - proposalCalls += 1 - return [ - { - surface: JSON.stringify({ visibility: 'team' }), - label: 'share within the team', - rationale: "the second agent needs the first agent's accepted fact", - }, - { - surface: JSON.stringify({ visibility: 'shared' }), - label: 'share globally', - rationale: 'compare a broader sharing policy under the same histories', - }, - ] - }, - } - - const options: RunAgentMemoryImprovementOptions = { - experimentId: 'improve-team-memory', - trainSequences: [ - improvementSequence('train-a', 'train'), - improvementSequence('train-b', 'train'), - ], - holdoutSequences: [ - improvementSequence('holdout-a', 'holdout'), - improvementSequence('holdout-b', 'holdout'), - ], - seeds: [ - { - config: { visibility: 'private' }, - track: 'baseline', - proposer: 'seed', - }, - ], - proposer, - improvementRef: 'team-memory-policy/v1', - budget: { maxSteps: 1 }, - populationSize: 2, - candidateConcurrency: 2, - sequenceConcurrency: 4, - runDir: '/runs/improve-team-memory', - storage, - significance: { minProductiveRuns: 2, resamples: 200, seed: 7 }, - createCandidate: ({ config, candidateId }) => { - if (config.visibility !== 'private') contenderIds.add(candidateId) - return { - ref: `visibility:${config.visibility}:v1`, - label: config.visibility, - policy: { read: [config.visibility], write: config.visibility }, - createAdapter: ({ branchId }) => createScopedTestAdapter(`${candidateId}:${branchId}`), - } - }, - executeStepRef: 'parallel-config-proof/v1', - executeStep: async ({ candidateId, step }) => { - if (step.id !== 'research' || !contenderIds.has(candidateId)) return - activeContenderCalls.set(candidateId, (activeContenderCalls.get(candidateId) ?? 0) + 1) - maxConcurrentConfigs = Math.max(maxConcurrentConfigs, activeContenderCalls.size) - if (activeContenderCalls.size === 2) reportContendersActive?.() - try { - await continueContenders - } finally { - const remaining = (activeContenderCalls.get(candidateId) ?? 1) - 1 - if (remaining === 0) activeContenderCalls.delete(candidateId) - else activeContenderCalls.set(candidateId, remaining) - } - }, - activation: { - ref: 'memory-policy/live:v1', - async readCurrent() { - return structuredClone(activeConfig) - }, - async compareAndSet({ activationId, expectedConfig, config }) { - expect(activeConfig).toEqual(expectedConfig) - activationIds.push(activationId) - activeConfig = structuredClone(config) - promoted.push(structuredClone(config)) - }, - }, - } - const firstRun = runAgentMemoryImprovement(options) - await contendersActive - await expect(runAgentMemoryImprovement(options)).rejects.toThrow('active controller') - releaseContenders?.() - const result = await firstRun - - expect(result.decision).toMatchObject({ - status: 'promote', - reasons: [], - baselineScore: 0.25, - winnerScore: 1, - lift: 0.75, - }) - expect(result.decision.significance).toMatchObject({ n: 2, significant: true }) - expect(result.winnerConfig).toEqual({ visibility: 'team' }) - expect(result.holdout?.campaign.cells).toHaveLength(4) - expect(maxConcurrentConfigs).toBe(2) - expect(promoted).toEqual([{ visibility: 'team' }]) - expect(result.activation).toMatchObject({ status: 'activated' }) - expect(storage.read(result.resultJsonPath)).toContain('"status": "promote"') - expect( - JSON.parse(storage.read('/runs/improve-team-memory/memory-improvement-manifest.json') ?? '{}') - .identity?.schema, - ).toBe(6) - - const resumed = await runAgentMemoryImprovement(options) - expect(proposalCalls).toBe(1) - expect(promoted).toEqual([{ visibility: 'team' }]) - expect(activationIds).toEqual([result.activation.id]) - expect(resumed.activation).toEqual({ - ...result.activation, - status: 'already-activated', - }) - expect(activeConfig).toEqual({ visibility: 'team' }) - await expect( - runAgentMemoryImprovement({ ...options, budget: { maxSteps: 2 } }), - ).rejects.toThrow('does not match its persisted inputs or implementationRef') - await expect(runAgentMemoryImprovement({ ...options, minHoldoutScore: 0.99 })).rejects.toThrow( - 'does not match its persisted inputs or implementationRef', - ) - await expect( - runAgentMemoryImprovement({ ...options, improvementRef: 'team-memory-policy/v2' }), - ).rejects.toThrow('does not match its persisted inputs or implementationRef') - }) - - it('recovers when the live config changes before the activation event is persisted', async () => { - type Config = { visibility: 'private' | 'team' } - const storage = inMemoryCampaignStorage() - const append = storage.append!.bind(storage) - let rejectActivatedEvent = true - storage.append = (path, value, expectedBytes) => { - if ( - rejectActivatedEvent && - path.includes('/activations/') && - value.includes('"status":"activated"') - ) { - rejectActivatedEvent = false - throw new Error('activation journal unavailable') - } - return append(path, value, expectedBytes) - } - let activeConfig: Config = { visibility: 'private' } - let compareAndSetCalls = 0 - const options: RunAgentMemoryImprovementOptions = { - experimentId: 'recover-memory-activation', - trainSequences: [ - improvementSequence('activation-train-a', 'train'), - improvementSequence('activation-train-b', 'train'), - ], - holdoutSequences: [ - improvementSequence('activation-holdout-a', 'holdout'), - improvementSequence('activation-holdout-b', 'holdout'), - ], - seeds: [ - { - config: { visibility: 'private' }, - track: 'baseline', - proposer: 'seed', - }, - ], - proposer: { - kind: 'team-sharing-proposer', - async propose() { - return [ - { - surface: JSON.stringify({ visibility: 'team' }), - label: 'share with team', - rationale: 'the second agent needs the accepted fact', - }, - ] - }, - }, - improvementRef: 'recover-memory-activation:v1', - budget: { maxSteps: 1 }, - runDir: '/runs/recover-memory-activation', - storage, - significance: { minProductiveRuns: 2, resamples: 200, seed: 11 }, - createCandidate: ({ config, candidateId }) => ({ - ref: `visibility:${config.visibility}:v1`, - policy: { read: [config.visibility], write: config.visibility }, - createAdapter: ({ branchId }) => createScopedTestAdapter(`${candidateId}:${branchId}`), - }), - activation: { - ref: 'memory-policy/live:v1', - async readCurrent() { - return structuredClone(activeConfig) - }, - async compareAndSet({ expectedConfig, config }) { - expect(activeConfig).toEqual(expectedConfig) - compareAndSetCalls += 1 - activeConfig = structuredClone(config) - }, - }, - } - - await expect(runAgentMemoryImprovement(options)).rejects.toThrow( - 'activation journal unavailable', - ) - expect(activeConfig).toEqual({ visibility: 'team' }) - expect(compareAndSetCalls).toBe(1) - - const recovered = await runAgentMemoryImprovement(options) - expect(recovered.activation.status).toBe('recovered') - expect(compareAndSetCalls).toBe(1) - const resumed = await runAgentMemoryImprovement(options) - expect(resumed.activation.status).toBe('already-activated') - expect(compareAndSetCalls).toBe(1) - }) - - it('routes independent tracks to their named proposers with track context', async () => { - type Config = { visibility: 'private' | 'team' } - const trackContexts: Array<{ - id?: string - operation?: string - vision?: string - generation: number - costPhase?: string - hasCostLedger: boolean - }> = [] - let governorCostPhase: string | undefined - let governorHasCostLedger = false - const trackProposer: SurfaceProposer = { - kind: 'team-memory-researcher', - async propose(context) { - trackContexts.push({ - id: context.track?.id, - operation: context.track?.operation, - vision: context.track?.vision, - generation: context.generation, - costPhase: context.costPhase, - hasCostLedger: context.costLedger !== undefined, - }) - return [JSON.stringify({ visibility: 'team' })] - }, - } - - await runAgentMemoryImprovement({ - experimentId: 'named-track-proposers', - trainSequences: [improvementSequence('track-train', 'train')], - holdoutSequences: [improvementSequence('track-holdout', 'holdout')], - seeds: [ - { config: { visibility: 'private' }, track: 'baseline', proposer: 'baseline' }, - { - config: { visibility: 'private' }, - track: 'sharing-research', - proposer: 'team-memory-researcher', - vision: 'test whether team memory transfers accepted facts', - }, - ], - proposer: { - kind: 'unexpected-fallback', - async propose() { - throw new Error('named track should not use the fallback proposer') - }, - }, - proposers: { 'team-memory-researcher': trackProposer }, - governor: { - decide(context) { - governorCostPhase = context.costPhase - governorHasCostLedger = context.costLedger !== undefined - return { op: 'extend', track: 'sharing-research' } - }, - }, - improvementRef: 'named-track-proposers/v1', - budget: { maxSteps: 1 }, - populationSize: 1, - runDir: '/runs/named-track-proposers', - storage: inMemoryCampaignStorage(), - createCandidate: ({ config, candidateId }) => ({ - ref: `visibility:${config.visibility}:v1`, - policy: { read: [config.visibility], write: config.visibility }, - createAdapter: ({ branchId }) => createScopedTestAdapter(`${candidateId}:${branchId}`), - }), - }) - - expect(trackContexts).toEqual([ - { - id: 'sharing-research', - operation: 'extend', - vision: 'test whether team memory transfers accepted facts', - generation: 1, - costPhase: 'memory.proposal.sharing-research', - hasCostLedger: true, - }, - ]) - expect(governorCostPhase).toBe('memory.governor') - expect(governorHasCostLedger).toBe(true) - }) - - it('holds a winner when holdout histories do not test a critical dimension', async () => { - type Config = { visibility: 'private' | 'team' } - const result = await runAgentMemoryImprovement({ - experimentId: 'missing-critical-dimension', - trainSequences: [improvementSequence('critical-train', 'train')], - holdoutSequences: [ - improvementSequence('critical-holdout-a', 'holdout', false), - improvementSequence('critical-holdout-b', 'holdout', false), - improvementSequence('critical-holdout-c', 'holdout', false), - ], - seeds: [{ config: { visibility: 'private' }, track: 'baseline', proposer: 'sharing' }], - proposer: { - kind: 'sharing', - async propose() { - return [JSON.stringify({ visibility: 'team' })] - }, - }, - improvementRef: 'missing-critical-dimension/v1', - budget: { maxSteps: 1 }, - populationSize: 1, - runDir: '/runs/missing-critical-dimension', - storage: inMemoryCampaignStorage(), - significance: { minProductiveRuns: 1, resamples: 100, seed: 9 }, - criticalDimensions: ['memory_stale_safe'], - createCandidate: ({ config, candidateId }) => ({ - ref: `visibility:${config.visibility}:v1`, - policy: { read: [config.visibility], write: config.visibility }, - createAdapter: ({ branchId }) => createScopedTestAdapter(`${candidateId}:${branchId}`), - }), - }) - - expect(result.decision.status).toBe('hold') - expect(result.decision.criticalDimensions).toEqual([ - expect.objectContaining({ - dimension: 'memory_stale_safe', - n: 0, - expectedN: 0, - measured: false, - }), - ]) - expect(result.decision.reasons).toContain( - 'critical dimension memory_stale_safe has no applicable holdout histories', - ) - }) - - it('stops before a proposer call would exceed the run-wide cost limit', async () => { - let proposerExecuted = false - - await expect( - runAgentMemoryImprovement({ - experimentId: 'proposer-cost-limit', - trainSequences: [improvementSequence('cost-train', 'train')], - holdoutSequences: [improvementSequence('cost-holdout', 'holdout')], - seeds: [ - { - config: { visibility: 'private' as const }, - track: 'baseline', - proposer: 'costed', - }, - ], - proposer: { - kind: 'costed', - async propose(context) { - if (!context.costLedger) throw new Error('missing run cost ledger') - const paid = await context.costLedger.runPaidCall({ - actor: 'memory-config-proposer', - channel: 'agent', - phase: context.costPhase, - model: 'fixture-model', - maximumCharge: { externallyEnforcedMaximumUsd: 0.06 }, - execute: async () => { - proposerExecuted = true - return JSON.stringify({ visibility: 'team' }) - }, - receipt: () => ({ - model: 'fixture-model', - inputTokens: 0, - outputTokens: 0, - usageUnknown: true, - actualCostUsd: 0.06, - }), - }) - if (!paid.succeeded) throw paid.error - return [paid.value] - }, - }, - improvementRef: 'proposer-cost-limit/v1', - budget: { maxSteps: 1 }, - maxTotalCostUsd: 0.05, - runDir: '/runs/proposer-cost-limit', - storage: inMemoryCampaignStorage(), - createCandidate: ({ config, candidateId }) => ({ - ref: `visibility:${config.visibility}:v1`, - policy: { read: [config.visibility], write: config.visibility }, - createAdapter: ({ branchId }) => createScopedTestAdapter(`${candidateId}:${branchId}`), - }), - }), - ).rejects.toThrow('would exceed ceiling 0.05') - expect(proposerExecuted).toBe(false) - }) - - it('charges an interrupted proposer reservation before resuming the search', async () => { - type Config = { visibility: 'private' | 'team' } - const storage = inMemoryCampaignStorage() - const runDir = '/runs/interrupted-proposer-recovery' - const append = storage.append!.bind(storage) - let failFirstProposerReceipt = true - storage.append = (path, value, expectedBytes) => { - if ( - failFirstProposerReceipt && - path.endsWith('/cost-ledger.jsonl') && - value.includes('"status":"settled"') && - value.includes('memory-config-proposer') - ) { - failFirstProposerReceipt = false - throw new Error('simulated process exit before proposer receipt') - } - return append(path, value, expectedBytes) - } - let proposerCalls = 0 - const options: RunAgentMemoryImprovementOptions = { - experimentId: 'interrupted-proposer-recovery', - trainSequences: [improvementSequence('proposer-train', 'train')], - holdoutSequences: [improvementSequence('proposer-holdout', 'holdout')], - seeds: [ - { - config: { visibility: 'private' }, - track: 'baseline', - proposer: 'costed', - }, - ], - proposer: { - kind: 'costed', - async propose(context) { - proposerCalls += 1 - if (!context.costLedger) throw new Error('missing run cost ledger') - const paid = await context.costLedger.runPaidCall({ - actor: 'memory-config-proposer', - channel: 'agent', - phase: context.costPhase, - model: 'fixture-model', - maximumCharge: { externallyEnforcedMaximumUsd: 0.1 }, - execute: async () => JSON.stringify({ visibility: 'team' }), - receipt: () => ({ - model: 'fixture-model', - inputTokens: 0, - outputTokens: 0, - actualCostUsd: 0.1, - }), - }) - if (!paid.succeeded) throw paid.error - return [paid.value] - }, - }, - improvementRef: 'interrupted-proposer-recovery/v1', - budget: { maxSteps: 1 }, - maxTotalCostUsd: 0.2, - runDir, - storage, - createCandidate: ({ config, candidateId }) => ({ - ref: `visibility:${config.visibility}:v1`, - policy: { read: [config.visibility], write: config.visibility }, - createAdapter: ({ branchId }) => createScopedTestAdapter(`${candidateId}:${branchId}`), - }), - } - - await expect(runAgentMemoryImprovement(options)).rejects.toThrow('failed to persist') - const interruptedLedger = createRunCostLedger({ - storage, - runDir, - costCeilingUsd: 0.2, - }) - expect(interruptedLedger.listPending()).toEqual([ - expect.objectContaining({ actor: 'memory-config-proposer', state: 'interrupted' }), - ]) - - const result = await runAgentMemoryImprovement(options) - const resumedLedger = createRunCostLedger({ storage, runDir, costCeilingUsd: 0.2 }) - - expect(proposerCalls).toBe(2) - expect(result.totalCostUsd).toBe(0.2) - expect(resumedLedger.summary()).toMatchObject({ - totalCalls: 2, - unresolvedCalls: 0, - totalCostUsd: 0.2, - accountingComplete: true, - }) - expect(resumedLedger.list()[0]).toMatchObject({ - actor: 'memory-config-proposer', - costUsd: 0.1, - error: expect.stringContaining('charged the reserved maximum'), - }) - }) - - it('rejects train and holdout histories with the same id', async () => { - const sequence = improvementSequence('duplicate', 'train') - await expect( - runAgentMemoryImprovement({ - experimentId: 'overlap', - trainSequences: [sequence], - holdoutSequences: [{ ...sequence, split: 'holdout' }], - seeds: [{ config: {}, track: 'baseline', proposer: 'seed' }], - proposer: { - kind: 'unused', - async propose() { - return [] - }, - }, - improvementRef: 'overlap-test/v1', - budget: { maxSteps: 1 }, - runDir: '/runs/overlap', - storage: inMemoryCampaignStorage(), - createCandidate: () => ({ - ref: 'unused:v1', - createAdapter: () => createScopedTestAdapter('unused'), - }), - }), - ).rejects.toThrow('train/holdout overlap: duplicate') - }) - - it('rejects a holdout history copied under a different id', async () => { - const train = improvementSequence('train-original', 'train') - await expect( - runAgentMemoryImprovement({ - experimentId: 'renamed-overlap', - trainSequences: [train], - holdoutSequences: [{ ...train, id: 'renamed-holdout', split: 'holdout' }], - seeds: [{ config: {}, track: 'baseline', proposer: 'seed' }], - proposer: { - kind: 'unused', - async propose() { - return [] - }, - }, - improvementRef: 'renamed-overlap/v1', - budget: { maxSteps: 0 }, - runDir: '/runs/renamed-overlap', - storage: inMemoryCampaignStorage(), - createCandidate: () => ({ - ref: 'unused:v1', - createAdapter: () => createScopedTestAdapter('unused'), - }), - }), - ).rejects.toThrow('histories duplicate content') - }) -}) - -function improvementSequence(id: string, split: 'train' | 'holdout', includeStaleTarget = true) { - return { - id, - family: 'first-party' as const, - split, - steps: [ - { - id: 'research', - scope: { agentId: 'researcher', teamId: 'team-1' }, - writes: [ - { - id: `${id}-event`, - kind: 'fact' as const, - text: `${id} launch date is Friday`, - metadata: { eventId: `${id}-event`, actorId: 'researcher' }, - }, - ], - }, - { - id: 'delivery', - scope: { agentId: 'builder', teamId: 'team-1' }, - probes: [ - { - id: 'launch-date', - query: `${id} launch date`, - requiredFacts: [{ id: 'current', anyOf: [`${id} launch date is Friday`] }], - ...(includeStaleTarget - ? { - forbiddenFacts: [ - { - id: 'stale', - anyOf: [`${id} launch date is Thursday`], - obsolete: true, - }, - ], - } - : {}), - expectedEventIds: [`${id}-event`], - expectedActorIds: ['researcher'], - }, - ], - }, - ], - } -} - -function hitText(hit: AgentMemoryHit): string { - return hit.text -} - -function createScopedTestAdapter( - id: string, - beforeWrite?: (scope: AgentMemoryScope, text: string) => Promise, -): AgentMemoryAdapter { - const rows: Array<{ scope: AgentMemoryScope; hit: AgentMemoryHit }> = [] - let sequence = 0 - return { - id, - branchIsolation: { mode: 'scoped' }, - async search(_query, options = {}) { - return rows.filter((row) => sameScope(row.scope, options.scope)).map((row) => row.hit) - }, - async getContext(query, options = {}) { - const hits = await this.search(query, options) - return { query, hits, text: hits.map(hitText).join('\n'), sourceRecords: [] } - }, - async write(input) { - const scope = input.scope ?? {} - await beforeWrite?.(scope, input.text) - sequence += 1 - const memoryId = input.id ?? `${id}-${sequence}` - const hit: AgentMemoryHit = { - id: memoryId, - uri: `memory://${id}/${memoryId}`, - kind: input.kind, - text: input.text, - metadata: input.metadata, - } - rows.push({ scope, hit }) - return { accepted: true, id: memoryId, uri: hit.uri, kind: input.kind } - }, - async clear(scope) { - for (let index = rows.length - 1; index >= 0; index -= 1) { - if (sameScope(rows[index]!.scope, scope)) rows.splice(index, 1) - } - }, - } -} - -function sameScope(left: AgentMemoryScope, right: AgentMemoryScope = {}): boolean { - for (const key of [ - 'tenantId', - 'userId', - 'agentId', - 'teamId', - 'runId', - 'sessionId', - 'namespace', - ] as const) { - if (right[key] !== undefined && left[key] !== right[key]) return false - } - for (const [key, value] of Object.entries(right.tags ?? {})) { - if (left.tags?.[key] !== value) return false - } - return true -} diff --git a/tests/memory/branches.test.ts b/tests/memory/branches.test.ts new file mode 100644 index 0000000..717fa86 --- /dev/null +++ b/tests/memory/branches.test.ts @@ -0,0 +1,633 @@ +import { describe, expect, it } from 'vitest' +import { + type AgentMemoryAdapter, + type AgentMemoryHit, + type AgentMemoryScope, + createAgentMemoryBranch, + createNeo4jAgentMemoryAdapter, + forkAgentMemoryBranchSnapshot, +} from '../../src/memory/index' +import { createScopedTestAdapter, hitText } from '../support/memory' + +describe('memory branches', () => { + it('rejects two live handles for the same adapter branch', async () => { + const adapter = createScopedTestAdapter('duplicate-handle') + const first = createAgentMemoryBranch({ adapter, branchId: 'same-branch' }) + + expect(() => createAgentMemoryBranch({ adapter, branchId: 'same-branch' })).toThrow( + "memory branch 'same-branch' already has an open handle", + ) + + await first.close?.() + const resumed = createAgentMemoryBranch({ adapter, branchId: 'same-branch' }) + await resumed.close?.() + }) + + it('rejects adapters that do not declare how branches are isolated', () => { + const { branchIsolation: _branchIsolation, ...legacy } = createScopedTestAdapter('legacy') + + expect(() => + createAgentMemoryBranch({ + adapter: legacy, + branchId: 'candidate-a', + }), + ).toThrow(/adapter must declare branchIsolation/) + }) + + it('rejects Neo4j clients that were not isolated for the exact branch', () => { + const unscoped = createNeo4jAgentMemoryAdapter({ client: {}, transport: 'rest' }) + expect(() => createAgentMemoryBranch({ adapter: unscoped, branchId: 'candidate-a' })).toThrow( + 'create a separate MemoryClient namespace per branch', + ) + + const scoped = createNeo4jAgentMemoryAdapter({ + client: {}, + transport: 'rest', + branchId: 'candidate-a', + }) + expect( + createAgentMemoryBranch({ + adapter: scoped, + branchId: 'candidate-a', + policy: { read: ['shared'], write: 'shared' }, + }).branchId, + ).toBe('candidate-a') + expect(() => createAgentMemoryBranch({ adapter: scoped, branchId: 'candidate-a' })).toThrow( + 'only shared memory policy', + ) + expect(() => createAgentMemoryBranch({ adapter: scoped, branchId: 'candidate-b' })).toThrow( + "adapter instance belongs to branch 'candidate-a'", + ) + }) + + it('isolates branches and private agents while allowing team sharing', async () => { + const storage = createScopedTestAdapter('scoped') + const alpha = createAgentMemoryBranch({ + adapter: storage, + branchId: 'alpha', + policy: { read: ['private'], write: 'private' }, + baseScope: { tenantId: 'tenant', userId: 'user' }, + }) + const beta = createAgentMemoryBranch({ + adapter: storage, + branchId: 'beta', + policy: { read: ['private'], write: 'private' }, + baseScope: { tenantId: 'tenant', userId: 'user' }, + }) + + await alpha.write({ + kind: 'fact', + text: 'alpha private', + scope: { agentId: 'agent-a', teamId: 'team-1' }, + }) + await beta.write({ + kind: 'fact', + text: 'beta private', + scope: { agentId: 'agent-a', teamId: 'team-1' }, + }) + + expect((await alpha.search('private', { scope: { agentId: 'agent-a' } })).map(hitText)).toEqual( + ['alpha private'], + ) + expect((await alpha.search('private', { scope: { agentId: 'agent-b' } })).map(hitText)).toEqual( + [], + ) + expect((await beta.search('private', { scope: { agentId: 'agent-a' } })).map(hitText)).toEqual([ + 'beta private', + ]) + + const team = createAgentMemoryBranch({ + adapter: storage, + branchId: 'team-branch', + policy: { read: ['team'], write: 'team' }, + baseScope: { tenantId: 'tenant', userId: 'user' }, + }) + await team.write({ + kind: 'observation', + text: 'shared with the team', + scope: { agentId: 'agent-a', teamId: 'team-1' }, + }) + const teamHits = await team.search('team', { + scope: { agentId: 'agent-b', teamId: 'team-1' }, + }) + expect(teamHits.map(hitText)).toEqual(['shared with the team']) + }) + + it('removes every journal entry in a provider partition cleared through a narrower scope', async () => { + const branch = createAgentMemoryBranch({ + adapter: createScopedTestAdapter('team-clear'), + branchId: 'team-clear', + policy: { read: ['team'], write: 'team' }, + baseScope: { tenantId: 'tenant', teamId: 'team-1' }, + }) + await branch.write({ kind: 'fact', text: 'from a', scope: { agentId: 'agent-a' } }) + await branch.write({ kind: 'fact', text: 'from b', scope: { agentId: 'agent-b' } }) + + await branch.clear?.({ agentId: 'agent-a' }) + + expect((await branch.snapshot()).journal).toEqual([]) + await expect(branch.search('from b', { scope: { agentId: 'agent-b' } })).resolves.toEqual([]) + }) + + it('preserves provider order for unscored branch hits', async () => { + const hits: AgentMemoryHit[] = [ + { id: 'z-top', uri: 'memory://rank/z', kind: 'fact', text: 'provider first' }, + { id: 'a-lower', uri: 'memory://rank/a', kind: 'fact', text: 'provider second' }, + ] + const adapter: AgentMemoryAdapter = { + id: 'provider-ranked', + branchIsolation: { mode: 'scoped' }, + async search() { + return hits + }, + async getContext(query) { + return { query, text: '', hits: [], sourceRecords: [] } + }, + async write(input) { + return { accepted: true, id: 'write', uri: 'memory://rank/write', kind: input.kind } + }, + } + const branch = createAgentMemoryBranch({ + adapter, + branchId: 'provider-ranked', + baseScope: { agentId: 'agent-a' }, + }) + + const ranked = await branch.search('anything', { limit: 1 }) + + expect(ranked.map((hit) => hit.id)).toEqual(['z-top']) + }) + + it('fuses unscored visibility lists by provider rank', async () => { + const adapter: AgentMemoryAdapter = { + id: 'visibility-ranked', + branchIsolation: { mode: 'scoped' }, + async search(_query, options) { + return options?.scope?.tags?.memoryVisibility === 'private' + ? [ + { id: 'private-1', uri: 'memory://rank/private-1', kind: 'fact', text: 'p1' }, + { id: 'private-2', uri: 'memory://rank/private-2', kind: 'fact', text: 'p2' }, + ] + : [{ id: 'team-1', uri: 'memory://rank/team-1', kind: 'fact', text: 't1' }] + }, + async getContext(query) { + return { query, text: '', hits: [], sourceRecords: [] } + }, + async write(input) { + return { accepted: true, id: 'write', uri: 'memory://rank/write', kind: input.kind } + }, + } + const branch = createAgentMemoryBranch({ + adapter, + branchId: 'visibility-ranked', + policy: { read: ['private', 'team'], write: 'private' }, + baseScope: { agentId: 'agent-a', teamId: 'team-a' }, + }) + + const ranked = await branch.search('anything', { limit: 2 }) + + expect(ranked.map((hit) => hit.id)).toEqual(['private-1', 'team-1']) + }) + + it('uses rank fusion before incomparable scores from separate provider searches', async () => { + const adapter: AgentMemoryAdapter = { + id: 'visibility-scored', + branchIsolation: { mode: 'scoped' }, + async search(_query, options) { + return options?.scope?.tags?.memoryVisibility === 'private' + ? [ + { + id: 'private-first', + uri: 'memory://rank/private-first', + kind: 'fact', + text: 'private first', + score: 0.01, + }, + { + id: 'private-second', + uri: 'memory://rank/private-second', + kind: 'fact', + text: 'private second', + score: 1, + }, + ] + : [ + { + id: 'team-first', + uri: 'memory://rank/team-first', + kind: 'fact', + text: 'team first', + score: 0.02, + }, + ] + }, + async getContext(query) { + return { query, text: '', hits: [], sourceRecords: [] } + }, + async write(input) { + return { accepted: true, id: 'write', uri: 'memory://rank/write', kind: input.kind } + }, + } + const branch = createAgentMemoryBranch({ + adapter, + branchId: 'visibility-scored', + policy: { read: ['private', 'team'], write: 'private' }, + baseScope: { agentId: 'agent-a', teamId: 'team-a' }, + }) + + const ranked = await branch.search('anything', { limit: 2 }) + + expect(ranked.map((hit) => hit.id)).toEqual(['team-first', 'private-first']) + }) + + it('serializes writes per actor and permits independent actors in parallel', async () => { + let active = 0 + let maxActive = 0 + const starts: string[] = [] + const storage = createScopedTestAdapter('ordered', async (scope, text) => { + starts.push(`${scope.agentId}:${text}`) + active += 1 + maxActive = Math.max(maxActive, active) + await new Promise((resolve) => setTimeout(resolve, 10)) + active -= 1 + }) + const branch = createAgentMemoryBranch({ + adapter: storage, + branchId: 'parallel', + policy: { read: ['private'], write: 'private' }, + }) + + await Promise.all([ + branch.write({ kind: 'message', text: 'a1', scope: { agentId: 'a' } }), + branch.write({ kind: 'message', text: 'a2', scope: { agentId: 'a' } }), + branch.write({ kind: 'message', text: 'b1', scope: { agentId: 'b' } }), + ]) + + expect(maxActive).toBe(2) + expect(starts.indexOf('a:a1')).toBeLessThan(starts.indexOf('a:a2')) + expect((await branch.snapshot()).journal.map((entry) => entry.input.text)).toEqual([ + 'a1', + 'a2', + 'b1', + ]) + }) + + it('preserves one actor ordering across sessions and tags', async () => { + const completed: string[] = [] + const storage = createScopedTestAdapter('actor-order', async (_scope, text) => { + if (text === 'first') await new Promise((resolve) => setTimeout(resolve, 20)) + completed.push(text) + }) + const branch = createAgentMemoryBranch({ + adapter: storage, + branchId: 'actor-order', + baseScope: { tenantId: 'tenant', agentId: 'agent-a' }, + }) + + await Promise.all([ + branch.write({ + kind: 'fact', + text: 'first', + scope: { sessionId: 'session-1', tags: { task: 'one' } }, + }), + branch.write({ + kind: 'fact', + text: 'second', + scope: { sessionId: 'session-2', tags: { task: 'two' } }, + }), + ]) + + expect(completed).toEqual(['first', 'second']) + }) + + it('takes a point-in-time snapshot while later writes wait at the boundary', async () => { + let releaseFirst!: () => void + const firstMayFinish = new Promise((resolve) => { + releaseFirst = resolve + }) + let reportFirstStarted!: () => void + const firstStarted = new Promise((resolve) => { + reportFirstStarted = resolve + }) + const providerStarts: string[] = [] + const storage = createScopedTestAdapter('snapshot-boundary', async (_scope, text) => { + providerStarts.push(text) + if (text === 'first') { + reportFirstStarted() + await firstMayFinish + } + }) + const branch = createAgentMemoryBranch({ + adapter: storage, + branchId: 'snapshot-boundary', + baseScope: { agentId: 'worker' }, + }) + + const firstWrite = branch.write({ kind: 'fact', text: 'first' }) + await firstStarted + const snapshotPromise = branch.snapshot() + const secondWrite = branch.write({ kind: 'fact', text: 'second' }) + await Promise.resolve() + + expect(providerStarts).toEqual(['first']) + releaseFirst() + await firstWrite + const snapshot = await snapshotPromise + await secondWrite + + expect(snapshot.journal.map((entry) => entry.input.text)).toEqual(['first']) + expect(providerStarts).toEqual(['first', 'second']) + }) + + it('does not clear storage while a preceding read is still using it', async () => { + let finishRead!: () => void + const readMayFinish = new Promise((resolve) => { + finishRead = resolve + }) + let reportReadStarted!: () => void + const readStarted = new Promise((resolve) => { + reportReadStarted = resolve + }) + let clearStarted = false + const adapter: AgentMemoryAdapter = { + id: 'read-clear-boundary', + branchIsolation: { mode: 'scoped' }, + async search() { + reportReadStarted() + await readMayFinish + return [] + }, + async getContext(query) { + return { query, text: '', hits: [], sourceRecords: [] } + }, + async write(input) { + return { accepted: true, id: 'write', uri: 'memory://write', kind: input.kind } + }, + async clear() { + clearStarted = true + }, + } + const branch = createAgentMemoryBranch({ + adapter, + branchId: 'read-clear-boundary', + baseScope: { agentId: 'worker' }, + }) + + const read = branch.search('active read') + await readStarted + const clear = branch.clear?.() + await Promise.resolve() + expect(clearStarted).toBe(false) + + finishRead() + await read + await clear + expect(clearStarted).toBe(true) + }) + + it('clears a touched scope after the provider commits and then throws', async () => { + let dirty = false + const clearedScopes: AgentMemoryScope[] = [] + const adapter: AgentMemoryAdapter = { + id: 'commit-then-error', + branchIsolation: { mode: 'scoped' }, + async search() { + return [] + }, + async getContext(query) { + return { query, text: '', hits: [], sourceRecords: [] } + }, + async write() { + dirty = true + throw new Error('connection lost after commit') + }, + async clear(scope) { + clearedScopes.push(scope ?? {}) + dirty = false + }, + } + const branch = createAgentMemoryBranch({ + adapter, + branchId: 'commit-then-error', + baseScope: { tenantId: 'tenant' }, + }) + + await expect( + branch.write({ kind: 'fact', text: 'partial', scope: { agentId: 'worker' } }), + ).rejects.toThrow('connection lost after commit') + await branch.clear?.() + + expect(dirty).toBe(false) + expect(clearedScopes).toHaveLength(1) + expect(clearedScopes[0]).toMatchObject({ tenantId: 'tenant', agentId: 'worker' }) + }) + + it('rejects non-durable journal data before it can corrupt resume state', async () => { + let providerWrites = 0 + const adapter = createScopedTestAdapter('durable-journal', async () => { + providerWrites += 1 + }) + const branch = createAgentMemoryBranch({ + adapter, + branchId: 'durable-journal', + baseScope: { agentId: 'worker' }, + }) + const cyclic: Record = {} + cyclic.self = cyclic + + await expect( + branch.write({ kind: 'fact', text: 'bad metadata', metadata: cyclic }), + ).rejects.toThrow('must not contain cycles') + expect(providerWrites).toBe(0) + expect((await branch.snapshot()).journal).toEqual([]) + }) + + it('returns snapshots detached from live branch state', async () => { + const adapter = createScopedTestAdapter('detached-snapshot') + const branch = createAgentMemoryBranch({ + adapter, + branchId: 'detached-snapshot', + baseScope: { tenantId: 'tenant-original', agentId: 'agent-a' }, + }) + await branch.write({ + kind: 'fact', + text: 'original text', + metadata: { nested: { status: 'original' } }, + }) + + const exposed = await branch.snapshot() + exposed.baseScope.tenantId = 'tenant-mutated' + const nested = exposed.journal[0]!.input.metadata!.nested as Record + nested.status = 'mutated' + const current = await branch.snapshot() + + expect(current.baseScope.tenantId).toBe('tenant-original') + expect(current.journal[0]?.input.metadata).toEqual({ nested: { status: 'original' } }) + expect(await branch.search('original')).toHaveLength(1) + }) + + it('closes a shared adapter once after every branch releases it', async () => { + let closes = 0 + const base = createScopedTestAdapter('shared-lifecycle') + const adapter: AgentMemoryAdapter = { + ...base, + async close() { + closes += 1 + }, + } + const first = createAgentMemoryBranch({ + adapter, + branchId: 'first', + baseScope: { agentId: 'agent-1' }, + }) + const second = createAgentMemoryBranch({ + adapter, + branchId: 'second', + baseScope: { agentId: 'agent-2' }, + }) + + await first.close?.() + await first.close?.() + expect(closes).toBe(0) + await second.close?.() + await second.close?.() + + expect(closes).toBe(1) + await expect(first.search('after close')).rejects.toThrow('closed branch') + }) + + it('waits for a concurrent fork to finish before closing the parent', async () => { + let releaseSnapshot!: () => void + let releaseReplay!: () => void + let reportReplayStarted!: () => void + const snapshotBlocked = new Promise((resolve) => { + releaseSnapshot = resolve + }) + const replayBlocked = new Promise((resolve) => { + releaseReplay = resolve + }) + const replayStarted = new Promise((resolve) => { + reportReplayStarted = resolve + }) + const base = createScopedTestAdapter('fork-close-race') + let writes = 0 + let flushes = 0 + let closes = 0 + const adapter: AgentMemoryAdapter = { + ...base, + async write(input) { + writes += 1 + if (writes === 2) { + reportReplayStarted() + await replayBlocked + } + return base.write(input) + }, + async flush() { + flushes += 1 + if (flushes === 1) await snapshotBlocked + }, + async close() { + closes += 1 + }, + } + const parent = createAgentMemoryBranch({ + adapter, + branchId: 'fork-close-parent', + baseScope: { agentId: 'worker' }, + }) + await parent.write({ kind: 'fact', text: 'replay me' }) + + const fork = parent.fork({ branchId: 'fork-close-child' }) + const close = parent.close!() + let closeSettled = false + void close.finally(() => { + closeSettled = true + }) + releaseSnapshot() + await replayStarted + await Promise.resolve() + await Promise.resolve() + + expect(closeSettled).toBe(false) + expect(closes).toBe(0) + releaseReplay() + const child = await fork + await close + expect((await child.search('replay')).map(hitText)).toEqual(['replay me']) + expect(closes).toBe(0) + await child.close?.() + expect(closes).toBe(1) + }) + + it('forks by replaying accepted writes into another adapter and resumes exactly', async () => { + const parentStorage = createScopedTestAdapter('parent') + const childStorage = createScopedTestAdapter('child') + const parent = createAgentMemoryBranch({ + adapter: parentStorage, + branchId: 'parent', + policy: { read: ['private'], write: 'private' }, + baseScope: { tenantId: 'tenant', agentId: 'agent-a' }, + }) + await parent.write({ kind: 'fact', text: 'seed fact' }) + + await expect(parent.fork({ branchId: 'parent' })).rejects.toThrow( + 'child branchId must differ from its parent', + ) + expect((await parent.search('fact')).map(hitText)).toEqual(['seed fact']) + + const child = await parent.fork({ branchId: 'child', adapter: childStorage }) + await child.write({ kind: 'fact', text: 'child only' }) + + expect((await parent.search('fact')).map(hitText)).toEqual(['seed fact']) + expect((await child.search('fact')).map(hitText)).toEqual(['seed fact', 'child only']) + const snapshot = await child.snapshot() + expect(snapshot.journal.every((entry) => Boolean(entry.input.id))).toBe(true) + expect(new Set(snapshot.journal.map((entry) => entry.input.id)).size).toBe(2) + await child.close?.() + const resumed = createAgentMemoryBranch({ + adapter: childStorage, + branchId: 'child', + snapshot, + }) + snapshot.baseScope.tenantId = 'mutated-after-resume' + expect((await resumed.search('fact')).map(hitText)).toEqual(['seed fact', 'child only']) + expect((await resumed.snapshot()).baseScope.tenantId).toBe('tenant') + expect((await resumed.snapshot()).digest).toBe(snapshot.digest) + + await expect( + Promise.resolve().then(() => + createAgentMemoryBranch({ + adapter: childStorage, + branchId: 'child', + snapshot: { ...snapshot, digest: 'sha256:bad' }, + }), + ), + ).rejects.toThrow('digest mismatch') + }) + + it('replays an attempt snapshot into a fresh branch', async () => { + const adapter: AgentMemoryAdapter = { + ...createScopedTestAdapter('attempt-source'), + branchIsolation: { mode: 'scoped', processExitSafe: false, recoveryDelayMs: 1 }, + } + const source = createAgentMemoryBranch({ + adapter, + branchId: 'attempt-source', + lifetime: 'attempt', + baseScope: { agentId: 'worker' }, + }) + await source.write({ kind: 'fact', text: 'durable observation' }) + const snapshot = await source.snapshot() + const target = await forkAgentMemoryBranchSnapshot({ + snapshot, + adapter: createScopedTestAdapter('resumable-target'), + branchId: 'resumable-target', + lifetime: 'resumable', + }) + + expect((await target.search('observation')).map(hitText)).toEqual(['durable observation']) + expect(target.parentBranchId).toBe('attempt-source') + expect(target.lifetime).toBe('resumable') + }) +}) diff --git a/tests/memory/experiment-cost.test.ts b/tests/memory/experiment-cost.test.ts new file mode 100644 index 0000000..fb216a2 --- /dev/null +++ b/tests/memory/experiment-cost.test.ts @@ -0,0 +1,619 @@ +import { createRunCostLedger, inMemoryCampaignStorage } from '@tangle-network/agent-eval/campaign' +import { describe, expect, it } from 'vitest' +import { + type AgentMemoryAdapter, + agentMemorySequenceJudge, + buildAgentMemorySequenceScenarios, + buildAgentMemorySequencesFromBenchmarkCases, + runAgentMemoryExperiment as runAgentMemoryExperimentRaw, +} from '../../src/memory/index' +import { createScopedTestAdapter, runAgentMemoryExperiment } from '../support/memory' + +describe('agent memory experiment cost and resume', () => { + it('versions memory scoring for resumable cache safety', () => { + expect(agentMemorySequenceJudge().judgeVersion).toBe('agent-knowledge:memory-sequence:v2') + }) + + it('requires an explicit controller policy for custom storage', async () => { + await expect( + runAgentMemoryExperimentRaw({ + experimentId: 'custom-storage-controller', + sequences: [ + { + id: 'history', + family: 'first-party', + steps: [{ id: 'probe', probes: [{ id: 'probe', query: 'x', referenceAnswer: 'x' }] }], + }, + ], + candidates: [ + { + id: 'memory', + ref: 'memory:v1', + createAdapter: () => createScopedTestAdapter('memory'), + }, + ], + runDir: '/runs/custom-storage-controller', + storage: inMemoryCampaignStorage(), + }), + ).rejects.toThrow("requires acquireRunLease or controllerMode='process-local'") + }) + + it('bounds provider cleanup and leaves the attempt available for recovery', async () => { + const storage = inMemoryCampaignStorage() + let clearCalls = 0 + const adapter = createScopedTestAdapter('bounded-cleanup') + const clear = adapter.clear! + adapter.clear = async (scope) => { + clearCalls += 1 + if (clearCalls === 1) return new Promise(() => {}) + await clear(scope) + } + const startedAt = Date.now() + + await expect( + runAgentMemoryExperiment({ + experimentId: 'bounded-cleanup', + sequences: [ + { + id: 'history', + family: 'first-party', + steps: [ + { + id: 'remember', + scope: { agentId: 'worker' }, + writes: [{ kind: 'fact', text: 'bounded fact' }], + probes: [{ id: 'probe', query: 'bounded', referenceAnswer: 'bounded fact' }], + }, + ], + }, + ], + candidates: [{ id: 'memory', ref: 'memory:v1', createAdapter: () => adapter }], + runDir: '/runs/bounded-cleanup', + storage, + cleanupTimeoutMs: 10, + }), + ).rejects.toThrow('memory experiment cleanup failed after dispatch') + + expect(Date.now() - startedAt).toBeLessThan(500) + expect( + storage.read('/runs/bounded-cleanup/memory-attempts.jsonl')?.trim().split('\n'), + ).toHaveLength(1) + }) + + it('does not charge provider cost when side-effect-free adapter construction fails', async () => { + const storage = inMemoryCampaignStorage() + const costLedger = createRunCostLedger({ + storage, + runDir: '/runs/create-adapter-cost', + costCeilingUsd: 1, + }) + + await expect( + runAgentMemoryExperiment({ + experimentId: 'create-adapter-cost', + sequences: [ + { + id: 'history', + family: 'first-party', + steps: [{ id: 'probe', probes: [{ id: 'probe', query: 'x', referenceAnswer: 'x' }] }], + }, + ], + candidates: [ + { + id: 'paid-memory', + ref: 'paid-memory:v1', + externalCostUsdPerSequence: 0.25, + createAdapter() { + throw new Error('provider setup rejected') + }, + }, + ], + runDir: '/runs/create-adapter-cost', + storage, + costLedger, + }), + ).rejects.toThrow('memory experiment cleanup failed after dispatch') + + expect(costLedger.summary().totalCostUsd).toBe(0) + }) + + it('charges a factory failure after dedicated provider provisioning starts', async () => { + const storage = inMemoryCampaignStorage() + const costLedger = createRunCostLedger({ + storage, + runDir: '/runs/create-adapter-provider-cost', + costCeilingUsd: 1, + }) + + await expect( + runAgentMemoryExperiment({ + experimentId: 'create-adapter-provider-cost', + sequences: [ + { + id: 'history', + family: 'first-party', + steps: [{ id: 'probe', probes: [{ id: 'probe', query: 'x', referenceAnswer: 'x' }] }], + }, + ], + candidates: [ + { + id: 'paid-memory', + ref: 'paid-memory:v1', + externalCostUsdPerSequence: 0.25, + createAdapter({ markExternalCall }) { + markExternalCall() + throw new Error('provider provisioning failed') + }, + }, + ], + runDir: '/runs/create-adapter-provider-cost', + storage, + costLedger, + }), + ).rejects.toThrow('memory experiment cleanup failed after dispatch') + + expect(costLedger.summary()).toMatchObject({ totalCalls: 1, totalCostUsd: 0.25 }) + }) + + it('records provider cleanup before a paid-call receipt can be interrupted', async () => { + const storage = inMemoryCampaignStorage() + const append = storage.append!.bind(storage) + let interruptReceipt = true + storage.append = (path, value, expectedBytes) => { + if ( + interruptReceipt && + path.endsWith('/cost-ledger.jsonl') && + value.includes('"status":"settled"') && + value.includes('agent-knowledge:memory-experiment') + ) { + interruptReceipt = false + throw new Error('simulated process exit before execute receipt') + } + return append(path, value, expectedBytes) + } + const purposes: string[] = [] + const sequence = { + id: 'history', + family: 'first-party' as const, + steps: [ + { + id: 'remember', + scope: { agentId: 'worker' }, + writes: [{ kind: 'fact' as const, text: 'durable fact' }], + probes: [{ id: 'recall', query: 'durable', referenceAnswer: 'durable fact' }], + }, + ], + } + const run = () => + runAgentMemoryExperiment({ + experimentId: 'execute-receipt-crash', + sequences: [sequence], + candidates: [ + { + id: 'memory', + ref: 'memory:v1', + externalCostUsdPerSequence: 0.1, + createAdapter({ purpose }) { + purposes.push(purpose) + return createScopedTestAdapter(`memory:${purpose}`) + }, + }, + ], + runDir: '/runs/execute-receipt-crash', + storage, + costCeiling: 1, + }) + + const interrupted = await run() + expect(interrupted.campaign.aggregates).toMatchObject({ + cellsFailed: 1, + cost: { accountingComplete: false, unresolvedCalls: 1 }, + }) + expect( + storage.read('/runs/execute-receipt-crash/memory-attempts.jsonl')?.trim().split('\n'), + ).toHaveLength(2) + + const result = await run() + expect(purposes).toEqual(['execute', 'execute']) + expect(result.rows[0]).toMatchObject({ cellsFailed: 0, totalCostUsd: 0.2 }) + }) + + it('accounts for parallel candidates against one shared dollar limit', async () => { + let releaseFirstCalls: (() => void) | undefined + const firstCallsReady = new Promise((resolve) => { + releaseFirstCalls = resolve + }) + let activeCalls = 0 + let firstCalls = 0 + let maxActiveCalls = 0 + const createAdapter = (id: string): AgentMemoryAdapter => { + const adapter = createScopedTestAdapter(id) + const clear = adapter.clear! + let entered = false + adapter.clear = async (scope) => { + if (!entered) { + entered = true + firstCalls += 1 + activeCalls += 1 + maxActiveCalls = Math.max(maxActiveCalls, activeCalls) + if (firstCalls === 2) releaseFirstCalls?.() + await firstCallsReady + activeCalls -= 1 + } + await clear(scope) + } + return adapter + } + + const result = await runAgentMemoryExperiment({ + experimentId: 'parallel-shared-cost', + sequences: [ + { + id: 'history', + family: 'first-party', + steps: [ + { + id: 'remember', + scope: { agentId: 'worker' }, + writes: [{ kind: 'fact', text: 'parallel fact' }], + probes: [ + { + id: 'recall', + query: 'parallel', + scope: { agentId: 'worker' }, + referenceAnswer: 'parallel fact', + }, + ], + }, + ], + }, + ], + candidates: ['first', 'second'].map((id) => ({ + id, + ref: `${id}:v1`, + externalCostUsdPerSequence: 0.1, + createAdapter: () => createAdapter(id), + })), + runDir: '/runs/parallel-shared-cost', + storage: inMemoryCampaignStorage(), + costCeiling: 0.2, + maxConcurrency: 2, + }) + + expect(maxActiveCalls).toBe(2) + expect(result.rows.map((row) => row.totalCostUsd).sort()).toEqual([0.1, 0.1]) + expect(result.campaign.aggregates.totalCostUsd).toBe(0.2) + }) + + it('runs sequential paid histories after exact external-cost receipts', async () => { + const storage = inMemoryCampaignStorage() + const costLedger = createRunCostLedger({ + storage, + runDir: '/runs/sequential-shared-cost', + costCeilingUsd: 0.2, + }) + const sequences = ['first', 'second'].map((id) => ({ + id, + family: 'first-party', + steps: [ + { + id: 'remember', + scope: { agentId: 'worker' }, + writes: [{ kind: 'fact' as const, text: `${id} fact` }], + probes: [ + { + id: 'recall', + query: id, + scope: { agentId: 'worker' }, + referenceAnswer: `${id} fact`, + }, + ], + }, + ], + })) + + const result = await runAgentMemoryExperiment({ + experimentId: 'sequential-shared-cost', + sequences, + candidates: [ + { + id: 'memory', + ref: 'memory:v1', + externalCostUsdPerSequence: 0.1, + createAdapter: ({ sequence }) => createScopedTestAdapter(sequence.id), + }, + ], + runDir: '/runs/sequential-shared-cost', + storage, + costLedger, + maxConcurrency: 1, + }) + + expect(result.rows[0]).toMatchObject({ cellsFailed: 0, totalCostUsd: 0.2 }) + expect(costLedger.summary()).toMatchObject({ + totalCalls: 2, + totalCostUsd: 0.2, + accountingComplete: true, + }) + }) + + it('reports the reserved cost of an interrupted prior cell attempt', async () => { + const storage = inMemoryCampaignStorage() + const runDir = '/runs/interrupted-cell-cost-reporting' + const sequence = { + id: 'history', + family: 'first-party' as const, + steps: [ + { + id: 'remember', + scope: { agentId: 'worker' }, + writes: [{ kind: 'fact' as const, text: 'resumed fact' }], + probes: [{ id: 'recall', query: 'resumed', referenceAnswer: 'resumed fact' }], + }, + ], + } + const candidate = { + id: 'memory', + ref: 'memory:v1', + externalCostUsdPerSequence: 0.1, + createAdapter: () => createScopedTestAdapter('memory'), + } + const scenarioId = buildAgentMemorySequenceScenarios([sequence], [candidate])[0]!.id + const abandonedLedger = createRunCostLedger({ storage, runDir, costCeilingUsd: 1 }) + const controller = new AbortController() + let releaseProvider!: () => void + let reportStarted!: () => void + const started = new Promise((resolve) => { + reportStarted = resolve + }) + const abandoned = abandonedLedger.runPaidCall({ + callId: 'prior-cell-attempt', + channel: 'agent', + phase: 'memory.experiment', + actor: 'agent-knowledge:memory-experiment:memory', + model: 'memory', + signal: controller.signal, + tags: { + runDir, + scenarioId, + cellId: 'prior-cell', + rep: '0', + runAttemptId: 'prior-attempt', + }, + maximumCharge: { externallyEnforcedMaximumUsd: 0.1 }, + async execute() { + reportStarted() + return await new Promise((resolve) => { + releaseProvider = () => resolve('late result') + }) + }, + receipt: () => ({ + model: 'memory', + inputTokens: 0, + outputTokens: 0, + actualCostUsd: 0.05, + }), + }) + await started + controller.abort(new Error('simulated process exit')) + await abandoned + + const resumedLedger = createRunCostLedger({ storage, runDir, costCeilingUsd: 1 }) + const result = await runAgentMemoryExperiment({ + experimentId: 'interrupted-cell-cost-reporting', + sequences: [sequence], + candidates: [candidate], + runDir, + storage, + costLedger: resumedLedger, + }) + + expect(result.rows[0]).toMatchObject({ candidateId: 'memory', totalCostUsd: 0.2 }) + expect(resumedLedger.summary()).toMatchObject({ totalCalls: 2, totalCostUsd: 0.2 }) + releaseProvider() + await abandonedLedger.waitForIdle() + }) + + it('reconciles interrupted paid calls from every parallel memory branch before resuming', async () => { + const storage = inMemoryCampaignStorage() + const runDir = '/runs/parallel-interrupted-cost-recovery' + const abandonedLedger = createRunCostLedger({ storage, runDir, costCeilingUsd: 1 }) + const controllers = [new AbortController(), new AbortController()] + const releases: Array<() => void> = [] + let started = 0 + let reportStarted: (() => void) | undefined + const bothStarted = new Promise((resolve) => { + reportStarted = resolve + }) + const abandonedCalls = ['one', 'two'].map((branch, index) => + abandonedLedger.runPaidCall({ + callId: `abandoned-${branch}`, + channel: 'agent', + phase: 'memory.train', + actor: `agent-knowledge:memory-experiment:${branch}`, + model: `provider-${branch}`, + signal: controllers[index]!.signal, + maximumCharge: { externallyEnforcedMaximumUsd: 0.1 }, + async execute() { + started += 1 + if (started === 2) reportStarted?.() + return await new Promise((resolve) => { + releases[index] = () => resolve('late provider result') + }) + }, + receipt: () => ({ + model: `provider-${branch}`, + inputTokens: 0, + outputTokens: 0, + actualCostUsd: 0.05, + }), + }), + ) + + await bothStarted + for (const controller of controllers) { + controller.abort(new Error('simulated process exit')) + } + await Promise.all(abandonedCalls) + expect(abandonedLedger.listPending().map((call) => call.state)).toEqual(['late', 'late']) + + const resumedLedger = createRunCostLedger({ storage, runDir, costCeilingUsd: 1 }) + expect(resumedLedger.listPending().map((call) => call.state)).toEqual([ + 'interrupted', + 'interrupted', + ]) + + const result = await runAgentMemoryExperiment({ + experimentId: 'parallel-interrupted-cost-recovery', + sequences: [ + { + id: 'history', + family: 'first-party', + steps: [ + { + id: 'remember', + scope: { agentId: 'worker' }, + writes: [{ kind: 'fact', text: 'resumed fact' }], + probes: [ + { + id: 'recall', + query: 'resumed', + referenceAnswer: 'resumed fact', + }, + ], + }, + ], + }, + ], + candidates: [ + { + id: 'resumed', + ref: 'resumed:v1', + createAdapter: () => createScopedTestAdapter('resumed'), + }, + ], + runDir, + storage, + costLedger: resumedLedger, + }) + + expect(result.rows[0]).toMatchObject({ candidateId: 'resumed', cellsFailed: 0 }) + expect(resumedLedger.summary()).toMatchObject({ + totalCalls: 2, + unresolvedCalls: 0, + totalCostUsd: 0.2, + accountingComplete: true, + }) + expect(resumedLedger.list().map((receipt) => receipt.error)).toEqual([ + expect.stringContaining('charged the reserved maximum'), + expect.stringContaining('charged the reserved maximum'), + ]) + + for (const release of releases) release() + await abandonedLedger.waitForIdle() + }) + + it('does not reuse cells after the candidate implementation reference changes', async () => { + const storage = inMemoryCampaignStorage() + const sequence = { + id: 'candidate-ref', + family: 'first-party' as const, + steps: [ + { + id: 'remember', + scope: { agentId: 'worker' }, + writes: [{ kind: 'fact' as const, text: 'remember me' }], + probes: [ + { + id: 'recall', + query: 'remember', + requiredFacts: [{ id: 'fact', anyOf: ['remember me'] }], + }, + ], + }, + ], + } + let creates = 0 + const run = (ref: string, visible: boolean) => + runAgentMemoryExperiment({ + experimentId: 'candidate-ref-cache', + sequences: [sequence], + candidates: [ + { + id: 'memory', + ref, + createAdapter() { + creates += 1 + const adapter = createScopedTestAdapter(`memory:${ref}`) + if (visible) return adapter + return { + ...adapter, + async search() { + return [] + }, + } + }, + }, + ], + runDir: '/runs/candidate-ref-cache', + storage, + }) + + const first = await run('memory:v1', true) + const cached = await run('memory:v1', false) + const changed = await run('memory:v2', false) + + expect(first.rows[0]?.scoreMean).toBe(1) + expect(cached.campaign.cells[0]?.cached).toBe(true) + expect(cached.rows[0]?.scoreMean).toBe(1) + expect(changed.campaign.cells[0]?.cached).toBe(false) + expect(changed.rows[0]?.scoreMean).toBe(0) + expect(creates).toBe(2) + }) + + it('runs existing ordered memory benchmark cases without reshaping the dataset', async () => { + const sequences = buildAgentMemorySequencesFromBenchmarkCases([ + { + id: 'launch-update', + family: 'longmemeval', + taskKind: 'memory-temporal', + split: 'holdout', + events: [ + { id: 'old', actorId: 'pm', text: 'Launch is April 3.' }, + { id: 'current', actorId: 'pm', text: 'Launch moved to April 17.' }, + ], + prompt: 'When is launch?', + requiredFacts: [{ id: 'current-date', anyOf: ['April 17'] }], + forbiddenFacts: [{ id: 'old-date', anyOf: ['April 3'], obsolete: true }], + expectedEventIds: ['current'], + expectedActorIds: ['pm'], + }, + ]) + const result = await runAgentMemoryExperiment({ + experimentId: 'benchmark-conversion', + sequences, + candidates: [ + { + id: 'literal-memory', + ref: 'literal-memory:v1', + createAdapter: () => createScopedTestAdapter('literal-memory'), + }, + ], + runDir: '/runs/benchmark-conversion', + storage: inMemoryCampaignStorage(), + }) + + expect(sequences[0]?.steps.map((step) => step.id)).toEqual([ + 'event:old', + 'event:current', + 'probe', + ]) + expect(result.rows[0]).toMatchObject({ + scoreMean: 0.75, + totalSequences: 1, + totalProbes: 1, + cellsFailed: 0, + }) + expect(result.rows[0]?.dimensions).toMatchObject({ memory_stale_safe: 0 }) + }) +}) diff --git a/tests/memory/experiment-execution.test.ts b/tests/memory/experiment-execution.test.ts new file mode 100644 index 0000000..5e08ba4 --- /dev/null +++ b/tests/memory/experiment-execution.test.ts @@ -0,0 +1,330 @@ +import { inMemoryCampaignStorage } from '@tangle-network/agent-eval/campaign' +import { describe, expect, it } from 'vitest' +import type { AgentMemoryAdapter, AgentMemoryHit } from '../../src/memory/index' +import { createScopedTestAdapter, hitText, runAgentMemoryExperiment } from '../support/memory' + +describe('agent memory experiment execution', () => { + it('keeps each history ordered while comparing candidate branches in parallel', async () => { + const storage = inMemoryCampaignStorage() + const snapshots: string[] = [] + const stepOrder = new Map() + let active = 0 + let maxActive = 0 + let release: (() => void) | undefined + const twoActive = new Promise((resolve) => { + release = resolve + }) + const sequence = (id: string) => ({ + id, + family: 'first-party' as const, + split: 'holdout' as const, + steps: [ + { + id: 'research', + scope: { agentId: 'researcher', teamId: 'team-1' }, + writes: [ + { + id: `${id}-event`, + kind: 'fact' as const, + text: `${id} launch date is Friday`, + metadata: { eventId: `${id}-event`, actorId: 'researcher' }, + }, + ], + }, + { + id: 'delivery', + scope: { agentId: 'builder', teamId: 'team-1' }, + probes: [ + { + id: 'launch-date', + query: `${id} launch date`, + requiredFacts: [{ id: 'current', anyOf: [`${id} launch date is Friday`] }], + expectedEventIds: [`${id}-event`], + expectedActorIds: ['researcher'], + }, + ], + }, + ], + }) + + const result = await runAgentMemoryExperiment({ + experimentId: 'team-sharing-vs-private', + sequences: [sequence('alpha'), sequence('beta')], + candidates: [ + { + id: 'private', + ref: 'private:v1', + policy: { read: ['private'], write: 'private' }, + createAdapter: ({ branchId }) => createScopedTestAdapter(`private:${branchId}`), + }, + { + id: 'team', + ref: 'team:v1', + policy: { read: ['team'], write: 'team' }, + createAdapter: ({ branchId }) => createScopedTestAdapter(`team:${branchId}`), + }, + ], + runDir: '/runs/team-sharing-vs-private', + storage, + maxConcurrency: 4, + cleanupBranches: true, + executeStepRef: 'test-runtime/v1', + executeStep: async ({ memory, step }) => { + const order = stepOrder.get(memory.branchId) ?? [] + order.push(step.id) + stepOrder.set(memory.branchId, order) + if (step.id === 'research') { + active += 1 + maxActive = Math.max(maxActive, active) + if (active === 2) release?.() + await twoActive + active -= 1 + } + }, + onBranchSnapshot: ({ snapshot }) => { + snapshots.push(snapshot.digest) + }, + }) + + expect(maxActive).toBeGreaterThanOrEqual(2) + expect(result.rows[0]).toMatchObject({ + rank: 1, + candidateId: 'team', + scoreMean: 1, + passRate: 1, + totalSequences: 2, + totalCells: 2, + totalProbes: 2, + cellsFailed: 0, + }) + expect(result.rows[1]?.candidateId).toBe('private') + expect(result.rows[1]?.scoreMean).toBeLessThan(0.3) + expect(result.campaign.cells).toHaveLength(4) + expect(snapshots).toHaveLength(4) + expect([...stepOrder.values()].every((steps) => steps.join(',') === 'research,delivery')).toBe( + true, + ) + expect(storage.read(result.rankingJsonPath)).toContain('"candidateId": "team"') + expect(storage.read(result.rankingMarkdownPath)).toContain('| 1 | team |') + }) + + it('uses distinct external branch ids for distinct run directories', async () => { + const branchIds: string[] = [] + const sequence = { + id: 'branch-id', + family: 'first-party' as const, + steps: [ + { + id: 'probe', + scope: { agentId: 'worker' }, + probes: [{ id: 'state', query: 'state', referenceAnswer: 'state' }], + }, + ], + } + const run = (runDir: string) => + runAgentMemoryExperiment({ + experimentId: 'same-experiment', + sequences: [sequence], + candidates: [ + { + id: 'memory', + ref: 'memory:v1', + createAdapter({ branchId }) { + branchIds.push(branchId) + return createScopedTestAdapter(branchId) + }, + }, + ], + runDir, + storage: inMemoryCampaignStorage(), + }) + + await run('/runs/branch-id-a') + await run('/runs/branch-id-b') + + expect(branchIds).toHaveLength(2) + expect(branchIds[0]).not.toBe(branchIds[1]) + }) + + it('uses a fresh external branch id for each distributed execution attempt', async () => { + const branchIds: string[] = [] + const run = (runDir: string) => + runAgentMemoryExperiment({ + experimentId: 'distributed-experiment', + experimentRunId: 'distributed-run-17', + sequences: [ + { + id: 'shared-history', + family: 'first-party', + steps: [ + { + id: 'probe', + scope: { agentId: 'worker' }, + probes: [{ id: 'state', query: 'state', referenceAnswer: 'state' }], + }, + ], + }, + ], + candidates: [ + { + id: 'memory', + ref: 'memory:v1', + createAdapter({ branchId }) { + branchIds.push(branchId) + return createScopedTestAdapter(branchId) + }, + }, + ], + runDir, + storage: inMemoryCampaignStorage(), + }) + + await run('/worker-a/run') + await run('/worker-b/run') + + expect(branchIds).toHaveLength(2) + expect(branchIds[0]).not.toBe(branchIds[1]) + }) + + it('waits for timed-out provider work to finish cleanup before returning', async () => { + let releaseWrite!: () => void + const writeMayFinish = new Promise((resolve) => { + releaseWrite = resolve + }) + let reportWriteStarted!: () => void + const writeStarted = new Promise((resolve) => { + reportWriteStarted = resolve + }) + const rows: AgentMemoryHit[] = [] + let clears = 0 + const adapter: AgentMemoryAdapter = { + id: 'slow-provider', + branchIsolation: { mode: 'scoped' }, + async search() { + return [...rows] + }, + async getContext(query) { + return { query, text: rows.map(hitText).join('\n'), hits: [...rows], sourceRecords: [] } + }, + async write(input) { + reportWriteStarted() + await writeMayFinish + const hit = { + id: 'late-write', + uri: 'memory://slow-provider/late-write', + kind: input.kind, + text: input.text, + } + rows.push(hit) + return { accepted: true, id: hit.id, uri: hit.uri, kind: hit.kind } + }, + async clear() { + clears += 1 + rows.length = 0 + }, + } + const run = runAgentMemoryExperiment({ + experimentId: 'timeout-cleanup', + sequences: [ + { + id: 'slow-history', + family: 'first-party', + steps: [ + { + id: 'write', + scope: { agentId: 'worker' }, + writes: [{ kind: 'fact', text: 'late fact' }], + probes: [{ id: 'fact', query: 'fact', referenceAnswer: 'late fact' }], + }, + ], + }, + ], + candidates: [{ id: 'slow', ref: 'slow:v1', createAdapter: () => adapter }], + runDir: '/runs/timeout-cleanup', + storage: inMemoryCampaignStorage(), + dispatchTimeoutMs: 5, + }) + await writeStarted + let settled = false + void run.then( + () => { + settled = true + }, + () => { + settled = true + }, + ) + await new Promise((resolve) => setTimeout(resolve, 15)) + + expect(settled).toBe(false) + releaseWrite() + const result = await run + + expect(result.rows[0]).toMatchObject({ cellsFailed: 1 }) + expect(result.campaign.cells[0]?.error).toContain('dispatch exceeded 5ms') + expect(clears).toBe(1) + expect(rows).toEqual([]) + }) + + it('fails when cleanup after a timed-out provider write fails', async () => { + let releaseWrite!: () => void + const writeMayFinish = new Promise((resolve) => { + releaseWrite = resolve + }) + let reportWriteStarted!: () => void + const writeStarted = new Promise((resolve) => { + reportWriteStarted = resolve + }) + let clears = 0 + const adapter: AgentMemoryAdapter = { + id: 'cleanup-failure', + branchIsolation: { mode: 'scoped' }, + async search() { + return [] + }, + async getContext(query) { + return { query, text: '', hits: [], sourceRecords: [] } + }, + async write(input) { + reportWriteStarted() + await writeMayFinish + return { + accepted: true, + id: 'late-write', + uri: 'memory://cleanup-failure/late-write', + kind: input.kind, + } + }, + async clear() { + clears += 1 + throw new Error('provider cleanup unavailable') + }, + } + const run = runAgentMemoryExperiment({ + experimentId: 'timeout-cleanup-failure', + sequences: [ + { + id: 'slow-history', + family: 'first-party', + steps: [ + { + id: 'write', + scope: { agentId: 'worker' }, + writes: [{ kind: 'fact', text: 'late fact' }], + probes: [{ id: 'fact', query: 'fact', referenceAnswer: 'late fact' }], + }, + ], + }, + ], + candidates: [{ id: 'slow', ref: 'slow:v1', createAdapter: () => adapter }], + runDir: '/runs/timeout-cleanup-failure', + storage: inMemoryCampaignStorage(), + dispatchTimeoutMs: 5, + }) + await writeStarted + await new Promise((resolve) => setTimeout(resolve, 10)) + releaseWrite() + + await expect(run).rejects.toThrow('memory experiment cleanup failed after dispatch') + }) +}) diff --git a/tests/memory/experiment-recovery.test.ts b/tests/memory/experiment-recovery.test.ts new file mode 100644 index 0000000..c3e49b7 --- /dev/null +++ b/tests/memory/experiment-recovery.test.ts @@ -0,0 +1,675 @@ +import { createRunCostLedger, inMemoryCampaignStorage } from '@tangle-network/agent-eval/campaign' +import { describe, expect, it } from 'vitest' +import type { AgentMemoryAdapter, AgentMemoryHit } from '../../src/memory/index' +import { createScopedTestAdapter, hitText, runAgentMemoryExperiment } from '../support/memory' + +describe('agent memory experiment recovery', () => { + it('recovers an unfinished provider branch before retrying the history', async () => { + const storage = inMemoryCampaignStorage() + const rows = new Map() + const operations: string[] = [] + const branchIds: Record<'first' | 'recovery' | 'retry', string | undefined> = { + first: undefined, + recovery: undefined, + retry: undefined, + } + let firstExecution = true + const sequence = { + id: 'recover-history', + family: 'first-party' as const, + steps: [ + { + id: 'remember', + scope: { agentId: 'worker' }, + writes: [{ kind: 'fact' as const, text: 'sensitive provider fact' }], + probes: [ + { + id: 'recall', + query: 'provider fact', + referenceAnswer: 'sensitive provider fact', + }, + ], + }, + ], + } + const candidate = { + id: 'recoverable', + ref: 'recoverable:v1', + externalRecoveryCostUsdPerAttempt: 0.1, + createAdapter({ branchId, purpose }: { branchId: string; purpose: 'execute' | 'recovery' }) { + const executionLabel = + purpose === 'recovery' ? 'recovery' : firstExecution ? 'first' : 'retry' + branchIds[executionLabel] = branchId + operations.push(`create:${executionLabel}`) + let clearCalls = 0 + const adapter: AgentMemoryAdapter = { + id: 'recoverable-provider', + branchIsolation: { mode: 'scoped' }, + async search(_query, options) { + return [...(rows.get(options.scope?.namespace ?? '') ?? [])] + }, + async getContext(query, options) { + const hits = await adapter.search(query, options) + return { query, text: hits.map(hitText).join('\n'), hits, sourceRecords: [] } + }, + async write(input) { + operations.push(`write:${executionLabel}`) + const namespace = input.scope?.namespace ?? '' + const hit = { + id: input.id ?? `${executionLabel}-fact`, + uri: `memory://recoverable/${executionLabel}`, + kind: input.kind, + text: input.text, + } + rows.set(namespace, [...(rows.get(namespace) ?? []), hit]) + return { accepted: true, id: hit.id, uri: hit.uri, kind: hit.kind } + }, + async clear(scope) { + clearCalls += 1 + operations.push(`clear:${executionLabel}`) + if (executionLabel === 'first' && clearCalls === 1) { + throw new Error('provider cleanup unavailable') + } + rows.delete(scope?.namespace ?? '') + }, + } + if (purpose === 'execute') firstExecution = false + return adapter + }, + } + const run = () => + runAgentMemoryExperiment({ + experimentId: 'restart-recovery', + sequences: [sequence], + candidates: [candidate], + runDir: '/runs/restart-recovery', + storage, + costCeiling: 1, + }) + + await expect(run()).rejects.toThrow('memory experiment cleanup failed after dispatch') + expect(rows.size).toBe(1) + + const result = await run() + + expect(result.rows[0]).toMatchObject({ candidateId: 'recoverable', cellsFailed: 0 }) + expect(result.rows[0]?.totalCostUsd).toBe(0.1) + expect(result.campaign.aggregates.totalCostUsd).toBe(0.1) + expect(branchIds.recovery).toBe(branchIds.first) + expect(branchIds.retry).not.toBe(branchIds.first) + expect(operations.indexOf('clear:recovery')).toBeLessThan(operations.indexOf('write:retry')) + expect(rows.size).toBe(0) + const attemptEvents = storage + .read(result.attemptLogPath)! + .trim() + .split('\n') + .map((line) => JSON.parse(line) as { status: string; recovery: boolean }) + expect(attemptEvents.map(({ status, recovery }) => ({ status, recovery }))).toEqual([ + { status: 'started', recovery: false }, + { status: 'cleaned', recovery: true }, + { status: 'started', recovery: false }, + { status: 'cleaned', recovery: false }, + ]) + }) + + it('closes and disposes a recovery adapter that arrives after its factory timeout', async () => { + const storage = inMemoryCampaignStorage() + const runDir = '/runs/late-recovery-adapter' + storage.write( + `${runDir}/memory-attempts.jsonl`, + `${JSON.stringify({ + schema: 2, + status: 'started', + branchId: 'unfinished-branch', + candidateId: 'memory', + candidateRef: 'memory:v1', + sequenceId: 'history', + rep: 0, + seed: 42, + cleanupBranches: true, + externalCostUsdPerSequence: 0, + externalRecoveryCostUsdPerAttempt: 0, + recordedAt: '2026-01-01T00:00:00.000Z', + recovery: false, + })}\n`, + ) + const sequence = { + id: 'history', + family: 'first-party' as const, + steps: [ + { + id: 'remember', + scope: { agentId: 'worker' }, + writes: [{ kind: 'fact' as const, text: 'fact' }], + probes: [{ id: 'recall', query: 'fact', referenceAnswer: 'fact' }], + }, + ], + } + let resolveCreation!: (adapter: AgentMemoryAdapter) => void + const creation = new Promise((resolve) => { + resolveCreation = resolve + }) + let reportDisposed!: () => void + const disposed = new Promise((resolve) => { + reportDisposed = resolve + }) + let closeCalls = 0 + let disposeCalls = 0 + const lateAdapter = createScopedTestAdapter('memory-provider') + lateAdapter.close = async () => { + closeCalls += 1 + } + + await expect( + runAgentMemoryExperiment({ + experimentId: 'late-recovery-adapter', + sequences: [sequence], + candidates: [ + { + id: 'memory', + ref: 'memory:v1', + createAdapter: ({ purpose }) => + purpose === 'recovery' ? creation : createScopedTestAdapter('memory-provider'), + async disposeAdapter(adapter) { + expect(adapter).toBe(lateAdapter) + disposeCalls += 1 + reportDisposed() + }, + }, + ], + runDir, + storage, + cleanupTimeoutMs: 10, + }), + ).rejects.toThrow("memory: abandoned memory branch 'unfinished-branch' recovery failed") + + resolveCreation(lateAdapter) + await disposed + expect({ closeCalls, disposeCalls }).toEqual({ closeCalls: 1, disposeCalls: 1 }) + }) + + it('reconciles a crash after provider recovery but before its cost receipt', async () => { + const storage = inMemoryCampaignStorage() + const append = storage.append!.bind(storage) + let failRecoveryReceipt = false + storage.append = (path, value, expectedBytes) => { + if ( + failRecoveryReceipt && + path.endsWith('/cost-ledger.jsonl') && + value.includes('"status":"settled"') && + value.includes('agent-knowledge:memory-recovery') + ) { + failRecoveryReceipt = false + throw new Error('simulated process exit before recovery receipt') + } + return append(path, value, expectedBytes) + } + + let firstExecution = true + let recoveryClears = 0 + const candidate = { + id: 'crash-recoverable', + ref: 'crash-recoverable:v1', + externalCostUsdPerSequence: 0.1, + externalRecoveryCostUsdPerAttempt: 0.1, + createAdapter({ purpose }: { purpose: 'execute' | 'recovery' }) { + const adapter = createScopedTestAdapter(`crash-recoverable:${purpose}`) + const clear = adapter.clear! + let clearCalls = 0 + const failThisExecution = purpose === 'execute' && firstExecution + if (purpose === 'execute') firstExecution = false + adapter.clear = async (scope) => { + clearCalls += 1 + if (purpose === 'recovery') recoveryClears += 1 + if (failThisExecution && clearCalls === 1) { + throw new Error('leave the first branch active') + } + await clear(scope) + } + return adapter + }, + } + const run = () => + runAgentMemoryExperiment({ + experimentId: 'recovery-receipt-crash', + sequences: [ + { + id: 'history', + family: 'first-party', + steps: [ + { + id: 'remember', + scope: { agentId: 'worker' }, + writes: [{ kind: 'fact', text: 'durable recovery fact' }], + probes: [ + { + id: 'recall', + query: 'durable', + referenceAnswer: 'durable recovery fact', + }, + ], + }, + ], + }, + ], + candidates: [candidate], + runDir: '/runs/recovery-receipt-crash', + storage, + costCeiling: 1, + }) + + await expect(run()).rejects.toThrow('memory experiment cleanup failed after dispatch') + failRecoveryReceipt = true + await expect(run()).rejects.toThrow('failed to persist') + expect( + storage.read('/runs/recovery-receipt-crash/memory-attempts.jsonl')?.trim().split('\n'), + ).toHaveLength(2) + + const result = await run() + const costLedger = createRunCostLedger({ + storage, + runDir: '/runs/recovery-receipt-crash', + costCeilingUsd: 1, + }) + + expect(result.rows[0]).toMatchObject({ + candidateId: 'crash-recoverable', + cellsFailed: 0, + }) + expect(result.rows[0]?.totalCostUsd).toBeCloseTo(0.3) + expect(recoveryClears).toBe(1) + expect(costLedger.summary()).toMatchObject({ unresolvedCalls: 0, accountingComplete: true }) + expect(costLedger.summary().totalCostUsd).toBeCloseTo(0.3) + }) + + it('recovers independent abandoned branches in parallel before retrying them', async () => { + const storage = inMemoryCampaignStorage() + let phase: 'leave-active' | 'recover' = 'leave-active' + let activeRecoveries = 0 + let maxActiveRecoveries = 0 + let releaseRecoveries: (() => void) | undefined + const recoveriesStarted = new Promise((resolve) => { + releaseRecoveries = resolve + }) + const sequences = ['one', 'two'].map((id) => ({ + id, + family: 'first-party' as const, + steps: [ + { + id: 'remember', + scope: { agentId: 'worker' }, + writes: [{ kind: 'fact' as const, text: `${id} fact` }], + probes: [{ id: 'recall', query: id, referenceAnswer: `${id} fact` }], + }, + ], + })) + const run = () => + runAgentMemoryExperiment({ + experimentId: 'parallel-recovery', + sequences, + candidates: [ + { + id: 'memory', + ref: 'memory:v1', + async createAdapter({ purpose }) { + if (purpose === 'recovery') { + activeRecoveries += 1 + maxActiveRecoveries = Math.max(maxActiveRecoveries, activeRecoveries) + if (activeRecoveries === 2) releaseRecoveries?.() + await recoveriesStarted + activeRecoveries -= 1 + return null + } + if (phase === 'leave-active') return null + return createScopedTestAdapter('parallel-recovery') + }, + }, + ], + runDir: '/runs/parallel-recovery', + storage, + maxConcurrency: 2, + }) + + await expect(run()).rejects.toThrow('memory experiment cleanup failed after dispatch') + phase = 'recover' + const result = await run() + + expect(maxActiveRecoveries).toBe(2) + expect(result.rows[0]).toMatchObject({ candidateId: 'memory', cellsFailed: 0 }) + const recovered = storage + .read(result.attemptLogPath)! + .trim() + .split('\n') + .map((line) => JSON.parse(line) as { status: string; recovery: boolean }) + .filter((event) => event.status === 'cleaned' && event.recovery) + expect(recovered).toHaveLength(2) + }) + + it('closes an attempt whose provider was never created before retrying it', async () => { + const storage = inMemoryCampaignStorage() + const purposes: Array<'execute' | 'recovery'> = [] + let executionCalls = 0 + const sequence = { + id: 'provider-creation', + family: 'first-party' as const, + steps: [ + { + id: 'remember', + scope: { agentId: 'worker' }, + writes: [{ kind: 'fact' as const, text: 'created provider fact' }], + probes: [ + { + id: 'recall', + query: 'provider fact', + referenceAnswer: 'created provider fact', + }, + ], + }, + ], + } + const run = () => + runAgentMemoryExperiment({ + experimentId: 'provider-creation-recovery', + sequences: [sequence], + candidates: [ + { + id: 'sometimes-created', + ref: 'sometimes-created:v1', + externalRecoveryCostUsdPerAttempt: 0.1, + createAdapter({ purpose }) { + purposes.push(purpose) + if (purpose === 'recovery') return null + executionCalls += 1 + if (executionCalls === 1) return null + return createScopedTestAdapter('created-provider') + }, + }, + ], + runDir: '/runs/provider-creation-recovery', + storage, + costCeiling: 1, + }) + + await expect(run()).rejects.toThrow('memory experiment cleanup failed after dispatch') + const result = await run() + + expect(result.rows[0]).toMatchObject({ candidateId: 'sometimes-created', cellsFailed: 0 }) + expect(result.rows[0]?.totalCostUsd).toBe(0) + expect(purposes).toEqual(['execute', 'recovery', 'execute']) + const attemptEvents = storage + .read(result.attemptLogPath)! + .trim() + .split('\n') + .map((line) => JSON.parse(line) as { status: string; recovery: boolean }) + expect(attemptEvents.map(({ status, recovery }) => ({ status, recovery }))).toEqual([ + { status: 'started', recovery: false }, + { status: 'cleaned', recovery: true }, + { status: 'started', recovery: false }, + { status: 'cleaned', recovery: false }, + ]) + }) + + it('uses a retired candidate only to clean its unfinished branch', async () => { + const storage = inMemoryCampaignStorage() + const runDir = '/runs/retired-recovery-candidate' + const sequence = { + id: 'history', + family: 'first-party' as const, + steps: [ + { + id: 'remember', + scope: { agentId: 'worker' }, + writes: [{ kind: 'fact' as const, text: 'active fact' }], + probes: [{ id: 'recall', query: 'active', referenceAnswer: 'active fact' }], + }, + ], + } + storage.write( + `${runDir}/memory-attempts.jsonl`, + `${JSON.stringify({ + schema: 2, + status: 'started', + branchId: 'retired-branch', + candidateId: 'retired', + candidateRef: 'retired:v1', + sequenceId: sequence.id, + rep: 0, + seed: 1, + cleanupBranches: true, + externalCostUsdPerSequence: 0, + externalRecoveryCostUsdPerAttempt: 0.1, + recordedAt: '2026-01-01T00:00:00.000Z', + recovery: false, + })}\n`, + ) + const purposes: string[] = [] + let retiredClears = 0 + const result = await runAgentMemoryExperiment({ + experimentId: 'retired-recovery-candidate', + sequences: [sequence], + candidates: [ + { + id: 'active', + ref: 'active:v1', + createAdapter({ purpose }) { + purposes.push(`active:${purpose}`) + return createScopedTestAdapter('active') + }, + }, + ], + recoveryCandidates: [ + { + id: 'retired', + ref: 'retired:v1', + externalRecoveryCostUsdPerAttempt: 0.1, + createAdapter({ purpose }) { + purposes.push(`retired:${purpose}`) + const adapter = createScopedTestAdapter('retired') + adapter.clear = async () => { + retiredClears += 1 + } + return adapter + }, + }, + ], + runDir, + storage, + resumable: false, + costCeiling: 1, + }) + + expect(result.rows).toHaveLength(1) + expect(result.rows[0]?.candidateId).toBe('active') + expect(result.rows[0]?.totalCostUsd).toBe(0) + expect(result).toMatchObject({ totalCostUsd: 0.1, unrankedRecoveryCostUsd: 0.1 }) + expect(purposes).toEqual(['retired:recovery', 'active:execute']) + expect(retiredClears).toBeGreaterThan(0) + }) + + it('refuses to hide an unfinished branch when candidate cost settings change', async () => { + const storage = inMemoryCampaignStorage() + const runDir = '/runs/changed-recovery-costs' + const sequence = { + id: 'history', + family: 'first-party' as const, + steps: [{ id: 'probe', probes: [{ id: 'recall', query: 'fact', referenceAnswer: 'fact' }] }], + } + storage.write( + `${runDir}/memory-attempts.jsonl`, + `${JSON.stringify({ + schema: 2, + status: 'started', + branchId: 'unfinished-branch', + candidateId: 'memory', + candidateRef: 'memory:v1', + sequenceId: sequence.id, + rep: 0, + seed: 1, + cleanupBranches: true, + externalCostUsdPerSequence: 0, + externalRecoveryCostUsdPerAttempt: 0, + recordedAt: '2026-01-01T00:00:00.000Z', + recovery: false, + })}\n`, + ) + let adapterCreates = 0 + + await expect( + runAgentMemoryExperiment({ + experimentId: 'changed-recovery-costs', + sequences: [sequence], + candidates: [ + { + id: 'memory', + ref: 'memory:v1', + externalCostUsdPerSequence: 0.1, + createAdapter() { + adapterCreates += 1 + return createScopedTestAdapter('memory') + }, + }, + ], + runDir, + storage, + costCeiling: 1, + }), + ).rejects.toThrow('candidate cost settings changed') + + expect(adapterCreates).toBe(0) + expect(storage.read(`${runDir}/memory-attempts.jsonl`)?.trim().split('\n')).toHaveLength(1) + }) + + it('refuses a recovery backlog larger than maxRecoveryAttempts', async () => { + const storage = inMemoryCampaignStorage() + const runDir = '/runs/recovery-backlog-limit' + const sequence = { + id: 'history', + family: 'first-party' as const, + steps: [ + { + id: 'probe', + scope: { agentId: 'worker' }, + probes: [{ id: 'recall', query: 'fact', referenceAnswer: 'fact' }], + }, + ], + } + storage.write( + `${runDir}/memory-attempts.jsonl`, + `${[ + { + schema: 2, + status: 'started', + branchId: 'branch-1', + candidateId: 'memory', + candidateRef: 'memory:v1', + sequenceId: sequence.id, + rep: 0, + seed: 1, + cleanupBranches: true, + externalCostUsdPerSequence: 0, + externalRecoveryCostUsdPerAttempt: 0, + recordedAt: '2026-01-01T00:00:00.000Z', + recovery: false, + }, + { + schema: 2, + status: 'started', + branchId: 'branch-2', + candidateId: 'memory', + candidateRef: 'memory:v1', + sequenceId: sequence.id, + rep: 0, + seed: 2, + cleanupBranches: true, + externalCostUsdPerSequence: 0, + externalRecoveryCostUsdPerAttempt: 0, + recordedAt: '2026-01-01T00:00:00.000Z', + recovery: false, + }, + ] + .map((event) => JSON.stringify(event)) + .join('\n')}\n`, + ) + let providerCreates = 0 + + await expect( + runAgentMemoryExperiment({ + experimentId: 'recovery-backlog-limit', + sequences: [sequence], + candidates: [ + { + id: 'memory', + ref: 'memory:v1', + createAdapter() { + providerCreates += 1 + return createScopedTestAdapter('memory') + }, + }, + ], + runDir, + storage, + maxRecoveryAttempts: 1, + }), + ).rejects.toThrow('2 unfinished attempts; maxRecoveryAttempts is 1') + expect(providerCreates).toBe(0) + }) + + it('bounds repeated provider recovery across process restarts', async () => { + const storage = inMemoryCampaignStorage() + const runDir = '/runs/recovery-retry-limit' + const sequence = { + id: 'history', + family: 'first-party' as const, + steps: [{ id: 'probe', probes: [{ id: 'recall', query: 'fact', referenceAnswer: 'fact' }] }], + } + storage.write( + `${runDir}/memory-attempts.jsonl`, + `${JSON.stringify({ + schema: 2, + status: 'started', + branchId: 'unfinished-branch', + candidateId: 'memory', + candidateRef: 'memory:v1', + sequenceId: sequence.id, + rep: 0, + seed: 1, + cleanupBranches: true, + externalCostUsdPerSequence: 0, + externalRecoveryCostUsdPerAttempt: 0, + recordedAt: '2026-01-01T00:00:00.000Z', + recovery: false, + })}\n`, + ) + let recoveryCreates = 0 + const run = () => + runAgentMemoryExperiment({ + experimentId: 'recovery-retry-limit', + sequences: [sequence], + candidates: [ + { + id: 'memory', + ref: 'memory:v1', + createAdapter({ purpose }) { + if (purpose === 'recovery') recoveryCreates += 1 + throw new Error('provider recovery unavailable') + }, + }, + ], + runDir, + storage, + maxRecoveryRetriesPerAttempt: 2, + }) + + await expect(run()).rejects.toThrow( + "abandoned memory branch 'unfinished-branch' recovery failed", + ) + await expect(run()).rejects.toThrow( + "abandoned memory branch 'unfinished-branch' recovery failed", + ) + await expect(run()).rejects.toThrow('exhausted 2 recovery attempts') + expect(recoveryCreates).toBe(2) + expect( + storage.read(`${runDir}/memory-recovery-attempts.jsonl`)?.trim().split('\n'), + ).toHaveLength(2) + }) +}) diff --git a/tests/memory/experiment-safety.test.ts b/tests/memory/experiment-safety.test.ts new file mode 100644 index 0000000..005bfec --- /dev/null +++ b/tests/memory/experiment-safety.test.ts @@ -0,0 +1,597 @@ +import { inMemoryCampaignStorage } from '@tangle-network/agent-eval/campaign' +import { describe, expect, it } from 'vitest' +import type { AgentMemoryAdapter } from '../../src/memory/index' +import { createScopedTestAdapter, hitText, runAgentMemoryExperiment } from '../support/memory' + +describe('agent memory experiment safety', () => { + it('allows one controller per run while its history workers run in parallel', async () => { + const storage = inMemoryCampaignStorage() + let reportStarted: (() => void) | undefined + const started = new Promise((resolve) => { + reportStarted = resolve + }) + let releaseWorker: (() => void) | undefined + const continueWorker = new Promise((resolve) => { + releaseWorker = resolve + }) + const options: RunAgentMemoryExperimentOptions = { + experimentId: 'single-controller', + sequences: [ + { + id: 'one', + family: 'first-party', + steps: [ + { + id: 'work', + scope: { agentId: 'worker' }, + writes: [{ kind: 'fact', text: 'parallel worker fact' }], + probes: [ + { + id: 'recall', + query: 'worker fact', + referenceAnswer: 'parallel worker fact', + }, + ], + }, + ], + }, + ], + candidates: [ + { + id: 'memory', + ref: 'memory:v1', + createAdapter: () => createScopedTestAdapter('single-controller'), + }, + ], + runDir: '/runs/single-controller', + storage, + executeStepRef: 'blocking-step:v1', + async executeStep() { + reportStarted?.() + await continueWorker + }, + } + + const firstRun = runAgentMemoryExperiment(options) + await started + await expect( + runAgentMemoryExperiment({ ...options, storage: inMemoryCampaignStorage() }), + ).rejects.toThrow('active controller') + releaseWorker?.() + const result = await firstRun + + expect(result.rows[0]).toMatchObject({ candidateId: 'memory', cellsFailed: 0 }) + }) + + it('leaves a lost controller branch for the next owner and accepts duplicate cleanup receipts', async () => { + const storage = inMemoryCampaignStorage() + let firstControllerOwned = true + let expireFirstController = true + let controllerCount = 0 + let searches = 0 + let clears = 0 + let closes = 0 + let factVisible = false + const purposes: string[] = [] + const options: RunAgentMemoryExperimentOptions = { + experimentId: 'lost-controller', + sequences: [ + { + id: 'one', + family: 'first-party', + steps: [ + { + id: 'work', + scope: { agentId: 'worker' }, + writes: [{ kind: 'fact', text: 'must be cleaned' }], + probes: [{ id: 'probe', query: 'must', referenceAnswer: 'must be cleaned' }], + }, + ], + }, + ], + candidates: [ + { + id: 'memory', + ref: 'memory:v1', + createAdapter({ purpose }) { + purposes.push(purpose) + const adapter: AgentMemoryAdapter = { + id: 'lost-controller', + branchIsolation: { mode: 'scoped' }, + async search() { + searches += 1 + return factVisible + ? [ + { + id: 'write', + uri: 'memory://lost-controller/write', + kind: 'fact', + text: 'must be cleaned', + }, + ] + : [] + }, + async getContext(query, searchOptions) { + const hits = await adapter.search(query, searchOptions) + return { query, text: hits.map(hitText).join('\n'), hits, sourceRecords: [] } + }, + async write(input) { + factVisible = true + return { + accepted: true, + id: 'write', + uri: 'memory://lost-controller/write', + kind: input.kind, + } + }, + async clear() { + clears += 1 + factVisible = false + }, + async close() { + closes += 1 + }, + } + return adapter + }, + }, + ], + runDir: '/runs/lost-controller', + storage, + acquireRunLease: async () => { + controllerCount += 1 + const controller = controllerCount + return { + assertOwned() { + if (controller === 1 && !firstControllerOwned) { + throw new Error('controller ownership expired') + } + }, + release() {}, + } + }, + executeStepRef: 'expire-controller:v1', + async executeStep() { + if (expireFirstController) { + expireFirstController = false + firstControllerOwned = false + } + }, + } + const run = () => runAgentMemoryExperiment(options) + + await expect(run()).rejects.toThrow('controller ownership expired') + expect(searches).toBe(0) + expect(clears).toBe(0) + expect(closes).toBe(1) + expect(factVisible).toBe(true) + + const result = await run() + expect(result.rows[0]).toMatchObject({ candidateId: 'memory', cellsFailed: 0 }) + expect(purposes).toEqual(['execute', 'recovery', 'execute']) + expect(clears).toBe(2) + expect(closes).toBe(3) + expect(factVisible).toBe(false) + + const journal = storage.read(result.attemptLogPath)! + const lines = journal.trim().split('\n') + const last = lines.at(-1)! + expect(storage.append!(result.attemptLogPath, `${last}\n`, Buffer.byteLength(journal))).toBe( + Buffer.byteLength(journal) + Buffer.byteLength(`${last}\n`), + ) + const callsBeforeCachedRun = { + purposes: purposes.length, + searches, + clears, + closes, + } + const cached = await run() + expect(cached.campaign.aggregates.cellsCached).toBe(1) + expect({ purposes: purposes.length, searches, clears, closes }).toEqual(callsBeforeCachedRun) + }) + + it('preserves both the run failure and controller release failure', async () => { + const error = await runAgentMemoryExperiment({ + experimentId: 'run-and-release-failure', + sequences: [ + { + id: 'one', + family: 'first-party', + steps: [ + { + id: 'probe', + probes: [{ id: 'probe', query: 'state', referenceAnswer: 'state' }], + }, + ], + }, + ], + candidates: [ + { + id: 'unused', + ref: 'unused:v1', + createAdapter: () => createScopedTestAdapter('unused'), + }, + ], + runDir: '/runs/run-and-release-failure', + storage: inMemoryCampaignStorage(), + acquireRunLease: async () => ({ + assertOwned() { + throw new Error('run ownership failed') + }, + release() { + throw new Error('release failed') + }, + }), + }).catch((caught: unknown) => caught) + + expect(error).toBeInstanceOf(AggregateError) + expect((error as AggregateError).errors.map((item) => String(item))).toEqual([ + 'Error: run ownership failed', + 'Error: release failed', + ]) + }) + + it('uses the same random seed for every candidate on one history and repetition', async () => { + const seeds = new Map() + const sequence = { + id: 'paired-history', + family: 'first-party' as const, + steps: [ + { + id: 'probe', + scope: { agentId: 'worker' }, + probes: [{ id: 'state', query: 'state', referenceAnswer: 'state' }], + }, + ], + } + await runAgentMemoryExperiment({ + experimentId: 'paired-seeds', + sequences: [sequence], + candidates: ['a', 'b'].map((candidateId) => ({ + id: candidateId, + ref: `${candidateId}:v1`, + createAdapter: ({ sequence: candidateSequence, rep, seed }) => { + seeds.set(`${candidateId}:${candidateSequence.id}:${rep}`, seed) + return createScopedTestAdapter(`${candidateId}:${rep}`) + }, + })), + reps: 2, + runDir: '/runs/paired-seeds', + storage: inMemoryCampaignStorage(), + maxConcurrency: 4, + }) + + expect(seeds.get('a:paired-history:0')).toBe(seeds.get('b:paired-history:0')) + expect(seeds.get('a:paired-history:1')).toBe(seeds.get('b:paired-history:1')) + expect(seeds.get('a:paired-history:0')).not.toBe(seeds.get('a:paired-history:1')) + }) + + it('fails the experiment when accepted writes cannot be cleared after a failed step', async () => { + let clears = 0 + let closes = 0 + let disposals = 0 + const run = runAgentMemoryExperiment({ + experimentId: 'failed-step-cleanup', + sequences: [ + { + id: 'failure', + family: 'first-party', + steps: [ + { + id: 'write-then-fail', + scope: { agentId: 'worker' }, + writes: [{ kind: 'fact', text: 'partial state' }], + probes: [{ id: 'state', query: 'partial state', referenceAnswer: 'partial state' }], + }, + ], + }, + ], + candidates: [ + { + id: 'scoped', + ref: 'scoped:v1', + createAdapter() { + const base = createScopedTestAdapter('failure') + return { + ...base, + async clear(scope) { + clears += 1 + await base.clear?.(scope) + throw new Error('provider clear failed') + }, + async close() { + closes += 1 + }, + } + }, + async disposeAdapter() { + disposals += 1 + }, + }, + ], + runDir: '/runs/failed-step-cleanup', + storage: inMemoryCampaignStorage(), + cleanupBranches: true, + executeStepRef: 'failing-worker/v1', + async executeStep() { + throw new Error('worker failed') + }, + }) + + await expect(run).rejects.toThrow('memory experiment cleanup failed after dispatch') + expect(clears).toBe(1) + expect(closes).toBe(1) + expect(disposals).toBe(1) + }) + + it('cleans an unjournaled provider side effect and ranks failed histories below complete ones', async () => { + let dirtyRows = 0 + let clears = 0 + const result = await runAgentMemoryExperiment({ + experimentId: 'provider-side-effect-cleanup', + sequences: [ + { + id: 'one', + family: 'first-party', + steps: [ + { + id: 'remember', + scope: { agentId: 'worker' }, + writes: [{ kind: 'fact', text: 'Launch is Friday.' }], + probes: [{ id: 'launch', query: 'launch', referenceAnswer: 'Launch is Friday.' }], + }, + ], + }, + ], + candidates: [ + { + id: 'side-effect-then-error', + ref: 'side-effect-then-error:v1', + createAdapter: () => ({ + id: 'side-effect-then-error', + branchIsolation: { mode: 'scoped' }, + async search() { + return [] + }, + async getContext(query) { + return { query, text: '', hits: [], sourceRecords: [] } + }, + async write() { + dirtyRows += 1 + throw new Error('provider disconnected after commit') + }, + async clear() { + clears += 1 + dirtyRows = 0 + }, + }), + }, + { + id: 'complete', + ref: 'complete:v1', + createAdapter: () => createScopedTestAdapter('complete'), + }, + ], + runDir: '/runs/provider-side-effect-cleanup', + storage: inMemoryCampaignStorage(), + cleanupBranches: true, + maxConcurrency: 2, + }) + + expect(dirtyRows).toBe(0) + expect(clears).toBe(1) + expect(result.rows[0]).toMatchObject({ + candidateId: 'complete', + cellsFailed: 0, + scoreMean: 1, + }) + expect(result.rows[1]).toMatchObject({ + candidateId: 'side-effect-then-error', + cellsFailed: 1, + scoreMean: 0, + passRate: 0, + }) + }) + + it('closes and disposes an adapter when branch validation fails', async () => { + let closes = 0 + let disposals = 0 + const result = await runAgentMemoryExperiment({ + experimentId: 'invalid-branch-cleanup', + sequences: [ + { + id: 'one', + family: 'first-party', + steps: [ + { + id: 'probe', + scope: { agentId: 'worker' }, + probes: [{ id: 'state', query: 'state', referenceAnswer: 'state' }], + }, + ], + }, + ], + candidates: [ + { + id: 'legacy', + ref: 'legacy:v1', + createAdapter() { + const { branchIsolation: _branchIsolation, ...legacy } = + createScopedTestAdapter('legacy') + return { + ...legacy, + async close() { + closes += 1 + }, + } + }, + async disposeAdapter() { + disposals += 1 + }, + }, + ], + runDir: '/runs/invalid-branch-cleanup', + storage: inMemoryCampaignStorage(), + cleanupBranches: false, + }) + + expect(result.rows[0]).toMatchObject({ cellsFailed: 1 }) + expect(result.campaign.cells[0]?.error).toContain('adapter must declare branchIsolation') + expect(closes).toBe(1) + expect(disposals).toBe(1) + }) + + it('fails closed when cleanup is requested for an adapter without scoped clear', async () => { + let disposals = 0 + const result = await runAgentMemoryExperiment({ + experimentId: 'unsupported-cleanup', + sequences: [ + { + id: 'one', + family: 'first-party', + steps: [ + { + id: 'probe', + scope: { agentId: 'worker' }, + probes: [{ id: 'state', query: 'state', referenceAnswer: 'state' }], + }, + ], + }, + ], + candidates: [ + { + id: 'no-clear', + ref: 'no-clear:v1', + createAdapter() { + const { clear: _clear, ...adapter } = createScopedTestAdapter('no-clear') + return adapter + }, + async disposeAdapter() { + disposals += 1 + }, + }, + ], + runDir: '/runs/unsupported-cleanup', + storage: inMemoryCampaignStorage(), + cleanupBranches: true, + }) + + expect(result.rows[0]).toMatchObject({ cellsFailed: 1 }) + expect(result.campaign.cells[0]?.error).toContain('requires an adapter with scoped clear') + expect(disposals).toBe(1) + }) + + it('rejects dynamic writes whose scope cannot be cleaned after a crash', async () => { + let providerWrites = 0 + const result = await runAgentMemoryExperiment({ + experimentId: 'undeclared-dynamic-scope', + sequences: [ + { + id: 'one', + family: 'first-party', + steps: [ + { + id: 'agent-step', + scope: { agentId: 'declared' }, + probes: [{ id: 'state', query: 'state', referenceAnswer: 'state' }], + }, + ], + }, + ], + candidates: [ + { + id: 'scoped', + ref: 'scoped:v1', + createAdapter: () => + createScopedTestAdapter('scoped', async () => { + providerWrites += 1 + }), + }, + ], + runDir: '/runs/undeclared-dynamic-scope', + storage: inMemoryCampaignStorage(), + cleanupBranches: true, + executeStepRef: 'dynamic-scope-test/v1', + executeStep: async ({ memory }) => { + await memory.write({ + kind: 'fact', + text: 'must not reach the provider', + scope: { agentId: 'undeclared' }, + }) + }, + }) + + expect(result.rows[0]).toMatchObject({ cellsFailed: 1 }) + expect(result.campaign.cells[0]?.error).toContain( + 'write scope was not declared in the experiment sequence cleanupScopes', + ) + expect(providerWrites).toBe(0) + }) + + it('requires an external-state disposer when scoped cleanup is disabled', async () => { + let creates = 0 + await expect( + runAgentMemoryExperiment({ + experimentId: 'missing-disposer', + sequences: [ + { + id: 'one', + family: 'first-party', + steps: [ + { + id: 'probe', + scope: { agentId: 'worker' }, + probes: [{ id: 'state', query: 'state', referenceAnswer: 'state' }], + }, + ], + }, + ], + candidates: [ + { + id: 'persistent', + ref: 'persistent:v1', + createAdapter() { + creates += 1 + return createScopedTestAdapter('persistent') + }, + }, + ], + runDir: '/runs/missing-disposer', + storage: inMemoryCampaignStorage(), + cleanupBranches: false, + }), + ).rejects.toThrow('requires disposeAdapter') + + expect(creates).toBe(0) + }) + + it('rejects probes that cannot distinguish a useful memory system', async () => { + await expect( + runAgentMemoryExperiment({ + experimentId: 'no-target', + sequences: [ + { + id: 'one', + family: 'first-party', + steps: [ + { + id: 'probe', + scope: { agentId: 'worker' }, + probes: [{ id: 'state', query: 'state' }], + }, + ], + }, + ], + candidates: [ + { + id: 'memory', + ref: 'memory:v1', + createAdapter: () => createScopedTestAdapter('memory'), + }, + ], + runDir: '/runs/no-target', + storage: inMemoryCampaignStorage(), + }), + ).rejects.toThrow('has no measurable target') + }) +}) diff --git a/tests/memory/graphiti.test.ts b/tests/memory/graphiti.test.ts new file mode 100644 index 0000000..bb6849f --- /dev/null +++ b/tests/memory/graphiti.test.ts @@ -0,0 +1,456 @@ +import { describe, expect, it } from 'vitest' +import { + type AgentMemoryScope, + createAgentMemoryBranch, + createGraphitiMemoryAdapter, + type GraphitiMcpClientLike, + graphitiMemoryAdapterIdentity, +} from '../../src/memory/index' + +describe('Graphiti adapter', () => { + it('uses the full default ingestion window before recovering an abandoned branch', () => { + const adapter = createGraphitiMemoryAdapter({ + client: { + async callTool() { + return { structuredContent: {} } + }, + }, + }) + + expect(adapter.branchIsolation).toEqual({ + mode: 'scoped', + processExitSafe: false, + recoveryDelayMs: 120_000, + }) + }) + + it('uses official MCP request shapes and rejoins fact provenance to episodes', async () => { + const requests: Array<{ name: string; arguments?: Record }> = [] + const episodes = new Map>() + const client: GraphitiMcpClientLike = { + async callTool(request) { + requests.push(request) + const args = request.arguments ?? {} + if (request.name === 'add_memory') { + episodes.set(String(args.uuid), { + uuid: args.uuid, + name: args.name, + content: args.episode_body, + source_description: args.source_description, + group_id: args.group_id, + }) + return { structuredContent: { message: 'queued' } } + } + if (request.name === 'get_episodes') { + return { + content: [{ type: 'text', text: JSON.stringify({ episodes: [...episodes.values()] }) }], + } + } + if (request.name === 'search_memory_facts') { + const episode = [...episodes.values()][0]! + return { + structuredContent: { + facts: [ + { + uuid: 'fact-1', + name: 'PREFERS', + fact: 'The user prefers concise status updates.', + group_id: episode.group_id, + episodes: [episode.uuid], + }, + ], + }, + } + } + if (request.name === 'search_nodes') { + return { structuredContent: { nodes: [] } } + } + throw new Error(`unexpected tool ${request.name}`) + }, + } + const adapter = createGraphitiMemoryAdapter({ + client, + search: ['facts'], + pollIntervalMs: 1, + ingestionTimeoutMs: 50, + }) + + const write = await adapter.write({ + id: 'event-1', + kind: 'message', + role: 'user', + text: 'I prefer concise status updates.', + scope: { userId: 'user-1', namespace: 'branch-1' }, + metadata: { actorId: 'user', timestamp: '2026-07-17T12:00:00Z' }, + }) + const hits = await adapter.search('status preference', { + scope: { userId: 'user-1', namespace: 'branch-1' }, + limit: 5, + }) + + expect(requests[0]).toMatchObject({ + name: 'add_memory', + arguments: { + episode_body: 'I prefer concise status updates.', + source: 'message', + reference_time: '2026-07-17T12:00:00Z', + }, + }) + expect(String(requests[0]?.arguments?.group_id)).toMatch(/^ak_[a-f0-9]{32}$/) + expect(String(requests[0]?.arguments?.uuid)).toMatch( + /^[a-f0-9]{8}-[a-f0-9]{4}-5[a-f0-9]{3}-a[a-f0-9]{3}-[a-f0-9]{12}$/, + ) + expect(write.metadata).toMatchObject({ consistency: 'visible', queued: false }) + expect(hits[0]).toMatchObject({ + id: 'fact-1', + kind: 'fact', + text: 'The user prefers concise status updates.', + metadata: { eventId: 'event-1', actorId: 'user' }, + }) + }) + + it('surfaces Graphiti tool errors', async () => { + const adapter = createGraphitiMemoryAdapter({ + client: { + async callTool() { + return { structuredContent: { error: 'database unavailable' } } + }, + }, + }) + + await expect(adapter.search('anything')).rejects.toThrow('database unavailable') + }) + + it('expands the episode scan when Graphiti truncates a long group', async () => { + const episodes = Array.from({ length: 150 }, (_, index) => ({ + uuid: `older-${index.toString().padStart(3, '0')}`, + })) + const limits: number[] = [] + const adapter = createGraphitiMemoryAdapter({ + pollIntervalMs: 1, + ingestionTimeoutMs: 50, + client: { + async callTool(request) { + if (request.name === 'add_memory') { + episodes.push({ uuid: String(request.arguments?.uuid) }) + return { structuredContent: { message: 'queued' } } + } + if (request.name === 'get_episodes') { + const limit = Number(request.arguments?.max_episodes) + limits.push(limit) + return { structuredContent: { episodes: episodes.slice(0, limit) } } + } + throw new Error(`unexpected tool ${request.name}`) + }, + }, + }) + + await adapter.write({ kind: 'fact', text: 'Visible beyond the first page.' }) + + expect(limits).toEqual([100, 200]) + }) + + it('keeps polling a full Graphiti group while the new episode becomes visible', async () => { + const older = Array.from({ length: 100 }, (_, index) => ({ uuid: `older-${index}` })) + let pendingUuid = '' + let visible = false + let polls = 0 + const adapter = createGraphitiMemoryAdapter({ + episodeScanLimit: 100, + pollIntervalMs: 1, + ingestionTimeoutMs: 50, + client: { + async callTool(request) { + if (request.name === 'add_memory') { + pendingUuid = String(request.arguments?.uuid) + setTimeout(() => { + visible = true + }, 5) + return { structuredContent: { message: 'queued' } } + } + if (request.name === 'get_episodes') { + polls += 1 + return { + structuredContent: { + episodes: visible ? [...older.slice(1), { uuid: pendingUuid }] : older, + }, + } + } + throw new Error(`unexpected tool ${request.name}`) + }, + }, + }) + + await expect( + adapter.write({ kind: 'fact', text: 'Eventually visible.' }), + ).resolves.toMatchObject({ accepted: true }) + expect(polls).toBeGreaterThan(1) + }) + + it('fails explicitly when visibility cannot be proved within the episode scan limit', async () => { + const episodes = Array.from({ length: 100 }, (_, index) => ({ uuid: `older-${index}` })) + const adapter = createGraphitiMemoryAdapter({ + episodeScanLimit: 100, + pollIntervalMs: 1, + ingestionTimeoutMs: 50, + client: { + async callTool(request) { + if (request.name === 'add_memory') { + episodes.push({ uuid: String(request.arguments?.uuid) }) + return { structuredContent: { message: 'queued' } } + } + if (request.name === 'get_episodes') { + return { structuredContent: { episodes: episodes.slice(0, 100) } } + } + throw new Error(`unexpected tool ${request.name}`) + }, + }, + }) + + await expect(adapter.write({ kind: 'fact', text: 'Outside the scan window.' })).rejects.toThrow( + 'episodeScanLimit=100', + ) + }) + + it('supports documented tool aliases and does not invent missing relevance scores', async () => { + const names: string[] = [] + const adapter = createGraphitiMemoryAdapter({ + search: ['facts'], + toolNames: { searchFacts: 'search_facts' }, + client: { + async callTool(request) { + names.push(request.name) + return { + structuredContent: { + facts: [{ uuid: 'fact-1', fact: 'An unscored fact', episodes: [] }], + }, + } + }, + }, + }) + + const hits = await adapter.search('fact', { minScore: 0.5 }) + + expect(names).toEqual(['search_facts']) + expect(hits).toEqual([]) + }) + + it('isolates run, session, and tag scopes into different Graphiti groups', async () => { + const groupIds: string[] = [] + const adapter = createGraphitiMemoryAdapter({ + search: ['facts'], + client: { + async callTool(request) { + if (request.name === 'search_memory_facts') { + groupIds.push(String((request.arguments?.group_ids as string[] | undefined)?.[0])) + return { structuredContent: { facts: [] } } + } + return { structuredContent: { episodes: [] } } + }, + }, + }) + + await adapter.search('query', { + scope: { namespace: 'same', runId: 'run-a', sessionId: 'session', tags: { arm: 'a' } }, + }) + await adapter.search('query', { + scope: { namespace: 'same', runId: 'run-b', sessionId: 'session', tags: { arm: 'b' } }, + }) + + expect(new Set(groupIds).size).toBe(2) + }) + + it('keeps caller-identified episode ids stable when groups execute in a different order', async () => { + const run = async (scopes: AgentMemoryScope[]) => { + const ids = new Map() + const adapter = createGraphitiMemoryAdapter({ + consistency: 'queued', + client: { + async callTool(request) { + if (request.name === 'add_memory') { + ids.set(String(request.arguments?.group_id), String(request.arguments?.uuid)) + } + return { structuredContent: { message: 'queued' } } + }, + }, + }) + for (const scope of scopes) { + await adapter.write({ id: 'source-1', kind: 'fact', text: 'same fact', scope }) + } + return ids + } + const first = await run([{ namespace: 'a' }, { namespace: 'b' }]) + const reversed = await run([{ namespace: 'b' }, { namespace: 'a' }]) + + expect(first).toEqual(reversed) + }) + + it('uses a new episode id when the content of one source event changes', async () => { + const episodeIds: string[] = [] + const adapter = createGraphitiMemoryAdapter({ + consistency: 'queued', + client: { + async callTool(request) { + if (request.name === 'add_memory') { + episodeIds.push(String(request.arguments?.uuid)) + } + return { structuredContent: { message: 'queued' } } + }, + }, + }) + + await adapter.write({ id: 'source-1', kind: 'fact', text: 'Launch is Friday.' }) + await adapter.write({ id: 'source-1', kind: 'fact', text: 'Launch is Monday.' }) + + expect(episodeIds).toHaveLength(2) + expect(episodeIds[0]).not.toBe(episodeIds[1]) + }) + + it('keeps repeated id-less Graphiti writes as distinct episodes', async () => { + const episodeIds: string[] = [] + const adapter = createGraphitiMemoryAdapter({ + consistency: 'queued', + client: { + async callTool(request) { + if (request.name === 'add_memory') { + episodeIds.push(String(request.arguments?.uuid)) + } + return { structuredContent: { message: 'queued' } } + }, + }, + }) + + await adapter.write({ kind: 'fact', text: 'The same observation.' }) + await adapter.write({ kind: 'fact', text: 'The same observation.' }) + + expect(episodeIds).toHaveLength(2) + expect(episodeIds[0]).not.toBe(episodeIds[1]) + }) + + it('keeps id-less Graphiti writes distinct across adapter restarts', async () => { + const episodeIds: string[] = [] + const client: GraphitiMcpClientLike = { + async callTool(request) { + if (request.name === 'add_memory') episodeIds.push(String(request.arguments?.uuid)) + return { structuredContent: { message: 'queued' } } + }, + } + + await createGraphitiMemoryAdapter({ client, consistency: 'queued' }).write({ + kind: 'fact', + text: 'The same observation.', + }) + await createGraphitiMemoryAdapter({ client, consistency: 'queued' }).write({ + kind: 'fact', + text: 'The same observation.', + }) + + expect(episodeIds).toHaveLength(2) + expect(episodeIds[0]).not.toBe(episodeIds[1]) + }) + + it('treats an empty Graphiti kind list as unfiltered search', async () => { + const names: string[] = [] + const adapter = createGraphitiMemoryAdapter({ + client: { + async callTool(request) { + names.push(request.name) + if (request.name === 'search_memory_facts') { + return { structuredContent: { facts: [] } } + } + if (request.name === 'search_nodes') return { structuredContent: { nodes: [] } } + throw new Error(`unexpected tool ${request.name}`) + }, + }, + }) + + await adapter.search('anything', { kinds: [] }) + + expect(names).toEqual(['search_memory_facts', 'search_nodes']) + }) + + it('fuses unscored Graphiti fact and node rankings before applying the limit', async () => { + const adapter = createGraphitiMemoryAdapter({ + client: { + async callTool(request) { + if (request.name === 'search_memory_facts') { + return { + structuredContent: { + facts: [ + { uuid: 'fact-1', fact: 'first fact' }, + { uuid: 'fact-2', fact: 'second fact' }, + ], + }, + } + } + if (request.name === 'search_nodes') { + return { + structuredContent: { + nodes: [ + { uuid: 'node-1', name: 'first node' }, + { uuid: 'node-2', name: 'second node' }, + ], + }, + } + } + throw new Error(`unexpected tool ${request.name}`) + }, + }, + }) + + const hits = await adapter.search('anything', { limit: 2 }) + + expect(hits.map((hit) => hit.id)).toEqual(['fact-1', 'node-1']) + }) + + it('uses Graphiti clear_graph for exact group cleanup', async () => { + const requests: Array<{ name: string; arguments?: Record }> = [] + const adapter = createGraphitiMemoryAdapter({ + client: { + async callTool(request) { + requests.push(request) + return { structuredContent: { message: 'Graph cleared successfully' } } + }, + }, + }) + + await adapter.clear?.({ userId: 'user-1', namespace: 'candidate-1' }) + + expect(requests).toHaveLength(1) + expect(requests[0]).toMatchObject({ + name: 'clear_graph', + arguments: { group_ids: [expect.stringMatching(/^ak_[a-f0-9]{32}$/)] }, + }) + }) + + it('rejects queued ingestion for resumable branch experiments', () => { + const adapter = createGraphitiMemoryAdapter({ + consistency: 'queued', + client: { + async callTool() { + return { structuredContent: { message: 'queued' } } + }, + }, + }) + + expect(() => createAgentMemoryBranch({ adapter, branchId: 'candidate-1' })).toThrow( + 'queued Graphiti writes can become visible after a process restart', + ) + }) + + it('includes the default scope in the Graphiti adapter identity', () => { + expect( + graphitiMemoryAdapterIdentity({ + id: 'graphiti', + backendRef: 'graphiti-cluster-a', + defaultScope: { namespace: 'one' }, + }), + ).not.toBe( + graphitiMemoryAdapterIdentity({ + id: 'graphiti', + backendRef: 'graphiti-cluster-a', + defaultScope: { namespace: 'two' }, + }), + ) + }) +}) diff --git a/tests/memory/improvement.test.ts b/tests/memory/improvement.test.ts new file mode 100644 index 0000000..af1d8cd --- /dev/null +++ b/tests/memory/improvement.test.ts @@ -0,0 +1,646 @@ +import { + createRunCostLedger, + inMemoryCampaignStorage, + type SurfaceProposer, +} from '@tangle-network/agent-eval/campaign' +import { describe, expect, it } from 'vitest' +import { runAgentMemoryImprovement as runAgentMemoryImprovementRaw } from '../../src/memory/index' +import { createScopedTestAdapter, runAgentMemoryImprovement } from '../support/memory' + +describe('agent memory improvement', () => { + it('fails fast when a JavaScript caller uses the removed onPromote option', async () => { + await expect( + runAgentMemoryImprovementRaw({ + onPromote() {}, + } as unknown as RunAgentMemoryImprovementOptions), + ).rejects.toThrow( + 'onPromote was removed; use activation.readCurrent and activation.compareAndSet', + ) + }) + + it('requires an explicit controller policy for custom improvement storage', async () => { + await expect( + runAgentMemoryImprovementRaw({ + experimentId: 'custom-improvement-storage', + trainSequences: [improvementSequence('train', 'train')], + holdoutSequences: [improvementSequence('holdout', 'holdout')], + seeds: [ + { + config: { mode: 'baseline' }, + track: 'baseline', + proposer: 'default', + }, + ], + createCandidate: () => ({ + ref: 'memory:v1', + createAdapter: () => createScopedTestAdapter('memory'), + }), + proposer: { kind: 'noop', propose: async () => [] }, + improvementRef: 'custom-improvement-storage:v1', + budget: { maxSteps: 1 }, + runDir: '/runs/custom-improvement-storage', + storage: inMemoryCampaignStorage(), + }), + ).rejects.toThrow("requires acquireRunLease or controllerMode='process-local'") + }) + + it('searches isolated configs and activates only a fresh holdout win', async () => { + type Config = { visibility: 'private' | 'team' | 'shared' } + const storage = inMemoryCampaignStorage() + const promoted: Config[] = [] + let activeConfig: Config = { visibility: 'private' } + const activationIds: string[] = [] + const contenderIds = new Set() + const activeContenderCalls = new Map() + let maxConcurrentConfigs = 0 + let reportContendersActive: (() => void) | undefined + const contendersActive = new Promise((resolve) => { + reportContendersActive = resolve + }) + let releaseContenders: (() => void) | undefined + const continueContenders = new Promise((resolve) => { + releaseContenders = resolve + }) + let proposalCalls = 0 + const proposer: SurfaceProposer = { + kind: 'team-sharing-proposer', + async propose() { + proposalCalls += 1 + return [ + { + surface: JSON.stringify({ visibility: 'team' }), + label: 'share within the team', + rationale: "the second agent needs the first agent's accepted fact", + }, + { + surface: JSON.stringify({ visibility: 'shared' }), + label: 'share globally', + rationale: 'compare a broader sharing policy under the same histories', + }, + ] + }, + } + + const options: RunAgentMemoryImprovementOptions = { + experimentId: 'improve-team-memory', + trainSequences: [ + improvementSequence('train-a', 'train'), + improvementSequence('train-b', 'train'), + ], + holdoutSequences: [ + improvementSequence('holdout-a', 'holdout'), + improvementSequence('holdout-b', 'holdout'), + ], + seeds: [ + { + config: { visibility: 'private' }, + track: 'baseline', + proposer: 'seed', + }, + ], + proposer, + improvementRef: 'team-memory-policy/v1', + budget: { maxSteps: 1 }, + populationSize: 2, + candidateConcurrency: 2, + sequenceConcurrency: 4, + runDir: '/runs/improve-team-memory', + storage, + significance: { minProductiveRuns: 2, resamples: 200, seed: 7 }, + createCandidate: ({ config, candidateId }) => { + if (config.visibility !== 'private') contenderIds.add(candidateId) + return { + ref: `visibility:${config.visibility}:v1`, + label: config.visibility, + policy: { read: [config.visibility], write: config.visibility }, + createAdapter: ({ branchId }) => createScopedTestAdapter(`${candidateId}:${branchId}`), + } + }, + executeStepRef: 'parallel-config-proof/v1', + executeStep: async ({ candidateId, step }) => { + if (step.id !== 'research' || !contenderIds.has(candidateId)) return + activeContenderCalls.set(candidateId, (activeContenderCalls.get(candidateId) ?? 0) + 1) + maxConcurrentConfigs = Math.max(maxConcurrentConfigs, activeContenderCalls.size) + if (activeContenderCalls.size === 2) reportContendersActive?.() + try { + await continueContenders + } finally { + const remaining = (activeContenderCalls.get(candidateId) ?? 1) - 1 + if (remaining === 0) activeContenderCalls.delete(candidateId) + else activeContenderCalls.set(candidateId, remaining) + } + }, + activation: { + ref: 'memory-policy/live:v1', + async readCurrent() { + return structuredClone(activeConfig) + }, + async compareAndSet({ activationId, expectedConfig, config }) { + expect(activeConfig).toEqual(expectedConfig) + activationIds.push(activationId) + activeConfig = structuredClone(config) + promoted.push(structuredClone(config)) + }, + }, + } + const firstRun = runAgentMemoryImprovement(options) + await contendersActive + await expect(runAgentMemoryImprovement(options)).rejects.toThrow('active controller') + releaseContenders?.() + const result = await firstRun + + expect(result.decision).toMatchObject({ + status: 'promote', + reasons: [], + baselineScore: 0.25, + winnerScore: 1, + lift: 0.75, + }) + expect(result.decision.significance).toMatchObject({ n: 2, significant: true }) + expect(result.winnerConfig).toEqual({ visibility: 'team' }) + expect(result.holdout?.campaign.cells).toHaveLength(4) + expect(maxConcurrentConfigs).toBe(2) + expect(promoted).toEqual([{ visibility: 'team' }]) + expect(result.activation).toMatchObject({ status: 'activated' }) + expect(storage.read(result.resultJsonPath)).toContain('"status": "promote"') + expect( + JSON.parse(storage.read('/runs/improve-team-memory/memory-improvement-manifest.json') ?? '{}') + .identity?.schema, + ).toBe(6) + + const resumed = await runAgentMemoryImprovement(options) + expect(proposalCalls).toBe(1) + expect(promoted).toEqual([{ visibility: 'team' }]) + expect(activationIds).toEqual([result.activation.id]) + expect(resumed.activation).toEqual({ + ...result.activation, + status: 'already-activated', + }) + expect(activeConfig).toEqual({ visibility: 'team' }) + await expect( + runAgentMemoryImprovement({ ...options, budget: { maxSteps: 2 } }), + ).rejects.toThrow('does not match its persisted inputs or implementationRef') + await expect(runAgentMemoryImprovement({ ...options, minHoldoutScore: 0.99 })).rejects.toThrow( + 'does not match its persisted inputs or implementationRef', + ) + await expect( + runAgentMemoryImprovement({ ...options, improvementRef: 'team-memory-policy/v2' }), + ).rejects.toThrow('does not match its persisted inputs or implementationRef') + }) + + it('recovers when the live config changes before the activation event is persisted', async () => { + type Config = { visibility: 'private' | 'team' } + const storage = inMemoryCampaignStorage() + const append = storage.append!.bind(storage) + let rejectActivatedEvent = true + storage.append = (path, value, expectedBytes) => { + if ( + rejectActivatedEvent && + path.includes('/activations/') && + value.includes('"status":"activated"') + ) { + rejectActivatedEvent = false + throw new Error('activation journal unavailable') + } + return append(path, value, expectedBytes) + } + let activeConfig: Config = { visibility: 'private' } + let compareAndSetCalls = 0 + const options: RunAgentMemoryImprovementOptions = { + experimentId: 'recover-memory-activation', + trainSequences: [ + improvementSequence('activation-train-a', 'train'), + improvementSequence('activation-train-b', 'train'), + ], + holdoutSequences: [ + improvementSequence('activation-holdout-a', 'holdout'), + improvementSequence('activation-holdout-b', 'holdout'), + ], + seeds: [ + { + config: { visibility: 'private' }, + track: 'baseline', + proposer: 'seed', + }, + ], + proposer: { + kind: 'team-sharing-proposer', + async propose() { + return [ + { + surface: JSON.stringify({ visibility: 'team' }), + label: 'share with team', + rationale: 'the second agent needs the accepted fact', + }, + ] + }, + }, + improvementRef: 'recover-memory-activation:v1', + budget: { maxSteps: 1 }, + runDir: '/runs/recover-memory-activation', + storage, + significance: { minProductiveRuns: 2, resamples: 200, seed: 11 }, + createCandidate: ({ config, candidateId }) => ({ + ref: `visibility:${config.visibility}:v1`, + policy: { read: [config.visibility], write: config.visibility }, + createAdapter: ({ branchId }) => createScopedTestAdapter(`${candidateId}:${branchId}`), + }), + activation: { + ref: 'memory-policy/live:v1', + async readCurrent() { + return structuredClone(activeConfig) + }, + async compareAndSet({ expectedConfig, config }) { + expect(activeConfig).toEqual(expectedConfig) + compareAndSetCalls += 1 + activeConfig = structuredClone(config) + }, + }, + } + + await expect(runAgentMemoryImprovement(options)).rejects.toThrow( + 'activation journal unavailable', + ) + expect(activeConfig).toEqual({ visibility: 'team' }) + expect(compareAndSetCalls).toBe(1) + + const recovered = await runAgentMemoryImprovement(options) + expect(recovered.activation.status).toBe('recovered') + expect(compareAndSetCalls).toBe(1) + const resumed = await runAgentMemoryImprovement(options) + expect(resumed.activation.status).toBe('already-activated') + expect(compareAndSetCalls).toBe(1) + }) + + it('routes independent tracks to their named proposers with track context', async () => { + type Config = { visibility: 'private' | 'team' } + const trackContexts: Array<{ + id?: string + operation?: string + vision?: string + generation: number + costPhase?: string + hasCostLedger: boolean + }> = [] + let governorCostPhase: string | undefined + let governorHasCostLedger = false + const trackProposer: SurfaceProposer = { + kind: 'team-memory-researcher', + async propose(context) { + trackContexts.push({ + id: context.track?.id, + operation: context.track?.operation, + vision: context.track?.vision, + generation: context.generation, + costPhase: context.costPhase, + hasCostLedger: context.costLedger !== undefined, + }) + return [JSON.stringify({ visibility: 'team' })] + }, + } + + await runAgentMemoryImprovement({ + experimentId: 'named-track-proposers', + trainSequences: [improvementSequence('track-train', 'train')], + holdoutSequences: [improvementSequence('track-holdout', 'holdout')], + seeds: [ + { config: { visibility: 'private' }, track: 'baseline', proposer: 'baseline' }, + { + config: { visibility: 'private' }, + track: 'sharing-research', + proposer: 'team-memory-researcher', + vision: 'test whether team memory transfers accepted facts', + }, + ], + proposer: { + kind: 'unexpected-fallback', + async propose() { + throw new Error('named track should not use the fallback proposer') + }, + }, + proposers: { 'team-memory-researcher': trackProposer }, + governor: { + decide(context) { + governorCostPhase = context.costPhase + governorHasCostLedger = context.costLedger !== undefined + return { op: 'extend', track: 'sharing-research' } + }, + }, + improvementRef: 'named-track-proposers/v1', + budget: { maxSteps: 1 }, + populationSize: 1, + runDir: '/runs/named-track-proposers', + storage: inMemoryCampaignStorage(), + createCandidate: ({ config, candidateId }) => ({ + ref: `visibility:${config.visibility}:v1`, + policy: { read: [config.visibility], write: config.visibility }, + createAdapter: ({ branchId }) => createScopedTestAdapter(`${candidateId}:${branchId}`), + }), + }) + + expect(trackContexts).toEqual([ + { + id: 'sharing-research', + operation: 'extend', + vision: 'test whether team memory transfers accepted facts', + generation: 1, + costPhase: 'memory.proposal.sharing-research', + hasCostLedger: true, + }, + ]) + expect(governorCostPhase).toBe('memory.governor') + expect(governorHasCostLedger).toBe(true) + }) + + it('holds a winner when holdout histories do not test a critical dimension', async () => { + type Config = { visibility: 'private' | 'team' } + const result = await runAgentMemoryImprovement({ + experimentId: 'missing-critical-dimension', + trainSequences: [improvementSequence('critical-train', 'train')], + holdoutSequences: [ + improvementSequence('critical-holdout-a', 'holdout', false), + improvementSequence('critical-holdout-b', 'holdout', false), + improvementSequence('critical-holdout-c', 'holdout', false), + ], + seeds: [{ config: { visibility: 'private' }, track: 'baseline', proposer: 'sharing' }], + proposer: { + kind: 'sharing', + async propose() { + return [JSON.stringify({ visibility: 'team' })] + }, + }, + improvementRef: 'missing-critical-dimension/v1', + budget: { maxSteps: 1 }, + populationSize: 1, + runDir: '/runs/missing-critical-dimension', + storage: inMemoryCampaignStorage(), + significance: { minProductiveRuns: 1, resamples: 100, seed: 9 }, + criticalDimensions: ['memory_stale_safe'], + createCandidate: ({ config, candidateId }) => ({ + ref: `visibility:${config.visibility}:v1`, + policy: { read: [config.visibility], write: config.visibility }, + createAdapter: ({ branchId }) => createScopedTestAdapter(`${candidateId}:${branchId}`), + }), + }) + + expect(result.decision.status).toBe('hold') + expect(result.decision.criticalDimensions).toEqual([ + expect.objectContaining({ + dimension: 'memory_stale_safe', + n: 0, + expectedN: 0, + measured: false, + }), + ]) + expect(result.decision.reasons).toContain( + 'critical dimension memory_stale_safe has no applicable holdout histories', + ) + }) + + it('stops before a proposer call would exceed the run-wide cost limit', async () => { + let proposerExecuted = false + + await expect( + runAgentMemoryImprovement({ + experimentId: 'proposer-cost-limit', + trainSequences: [improvementSequence('cost-train', 'train')], + holdoutSequences: [improvementSequence('cost-holdout', 'holdout')], + seeds: [ + { + config: { visibility: 'private' as const }, + track: 'baseline', + proposer: 'costed', + }, + ], + proposer: { + kind: 'costed', + async propose(context) { + if (!context.costLedger) throw new Error('missing run cost ledger') + const paid = await context.costLedger.runPaidCall({ + actor: 'memory-config-proposer', + channel: 'agent', + phase: context.costPhase, + model: 'fixture-model', + maximumCharge: { externallyEnforcedMaximumUsd: 0.06 }, + execute: async () => { + proposerExecuted = true + return JSON.stringify({ visibility: 'team' }) + }, + receipt: () => ({ + model: 'fixture-model', + inputTokens: 0, + outputTokens: 0, + usageUnknown: true, + actualCostUsd: 0.06, + }), + }) + if (!paid.succeeded) throw paid.error + return [paid.value] + }, + }, + improvementRef: 'proposer-cost-limit/v1', + budget: { maxSteps: 1 }, + maxTotalCostUsd: 0.05, + runDir: '/runs/proposer-cost-limit', + storage: inMemoryCampaignStorage(), + createCandidate: ({ config, candidateId }) => ({ + ref: `visibility:${config.visibility}:v1`, + policy: { read: [config.visibility], write: config.visibility }, + createAdapter: ({ branchId }) => createScopedTestAdapter(`${candidateId}:${branchId}`), + }), + }), + ).rejects.toThrow('would exceed ceiling 0.05') + expect(proposerExecuted).toBe(false) + }) + + it('charges an interrupted proposer reservation before resuming the search', async () => { + type Config = { visibility: 'private' | 'team' } + const storage = inMemoryCampaignStorage() + const runDir = '/runs/interrupted-proposer-recovery' + const append = storage.append!.bind(storage) + let failFirstProposerReceipt = true + storage.append = (path, value, expectedBytes) => { + if ( + failFirstProposerReceipt && + path.endsWith('/cost-ledger.jsonl') && + value.includes('"status":"settled"') && + value.includes('memory-config-proposer') + ) { + failFirstProposerReceipt = false + throw new Error('simulated process exit before proposer receipt') + } + return append(path, value, expectedBytes) + } + let proposerCalls = 0 + const options: RunAgentMemoryImprovementOptions = { + experimentId: 'interrupted-proposer-recovery', + trainSequences: [improvementSequence('proposer-train', 'train')], + holdoutSequences: [improvementSequence('proposer-holdout', 'holdout')], + seeds: [ + { + config: { visibility: 'private' }, + track: 'baseline', + proposer: 'costed', + }, + ], + proposer: { + kind: 'costed', + async propose(context) { + proposerCalls += 1 + if (!context.costLedger) throw new Error('missing run cost ledger') + const paid = await context.costLedger.runPaidCall({ + actor: 'memory-config-proposer', + channel: 'agent', + phase: context.costPhase, + model: 'fixture-model', + maximumCharge: { externallyEnforcedMaximumUsd: 0.1 }, + execute: async () => JSON.stringify({ visibility: 'team' }), + receipt: () => ({ + model: 'fixture-model', + inputTokens: 0, + outputTokens: 0, + actualCostUsd: 0.1, + }), + }) + if (!paid.succeeded) throw paid.error + return [paid.value] + }, + }, + improvementRef: 'interrupted-proposer-recovery/v1', + budget: { maxSteps: 1 }, + maxTotalCostUsd: 0.2, + runDir, + storage, + createCandidate: ({ config, candidateId }) => ({ + ref: `visibility:${config.visibility}:v1`, + policy: { read: [config.visibility], write: config.visibility }, + createAdapter: ({ branchId }) => createScopedTestAdapter(`${candidateId}:${branchId}`), + }), + } + + await expect(runAgentMemoryImprovement(options)).rejects.toThrow('failed to persist') + const interruptedLedger = createRunCostLedger({ + storage, + runDir, + costCeilingUsd: 0.2, + }) + expect(interruptedLedger.listPending()).toEqual([ + expect.objectContaining({ actor: 'memory-config-proposer', state: 'interrupted' }), + ]) + + const result = await runAgentMemoryImprovement(options) + const resumedLedger = createRunCostLedger({ storage, runDir, costCeilingUsd: 0.2 }) + + expect(proposerCalls).toBe(2) + expect(result.totalCostUsd).toBe(0.2) + expect(resumedLedger.summary()).toMatchObject({ + totalCalls: 2, + unresolvedCalls: 0, + totalCostUsd: 0.2, + accountingComplete: true, + }) + expect(resumedLedger.list()[0]).toMatchObject({ + actor: 'memory-config-proposer', + costUsd: 0.1, + error: expect.stringContaining('charged the reserved maximum'), + }) + }) + + it('rejects train and holdout histories with the same id', async () => { + const sequence = improvementSequence('duplicate', 'train') + await expect( + runAgentMemoryImprovement({ + experimentId: 'overlap', + trainSequences: [sequence], + holdoutSequences: [{ ...sequence, split: 'holdout' }], + seeds: [{ config: {}, track: 'baseline', proposer: 'seed' }], + proposer: { + kind: 'unused', + async propose() { + return [] + }, + }, + improvementRef: 'overlap-test/v1', + budget: { maxSteps: 1 }, + runDir: '/runs/overlap', + storage: inMemoryCampaignStorage(), + createCandidate: () => ({ + ref: 'unused:v1', + createAdapter: () => createScopedTestAdapter('unused'), + }), + }), + ).rejects.toThrow('train/holdout overlap: duplicate') + }) + + it('rejects a holdout history copied under a different id', async () => { + const train = improvementSequence('train-original', 'train') + await expect( + runAgentMemoryImprovement({ + experimentId: 'renamed-overlap', + trainSequences: [train], + holdoutSequences: [{ ...train, id: 'renamed-holdout', split: 'holdout' }], + seeds: [{ config: {}, track: 'baseline', proposer: 'seed' }], + proposer: { + kind: 'unused', + async propose() { + return [] + }, + }, + improvementRef: 'renamed-overlap/v1', + budget: { maxSteps: 0 }, + runDir: '/runs/renamed-overlap', + storage: inMemoryCampaignStorage(), + createCandidate: () => ({ + ref: 'unused:v1', + createAdapter: () => createScopedTestAdapter('unused'), + }), + }), + ).rejects.toThrow('histories duplicate content') + }) +}) + +function improvementSequence(id: string, split: 'train' | 'holdout', includeStaleTarget = true) { + return { + id, + family: 'first-party' as const, + split, + steps: [ + { + id: 'research', + scope: { agentId: 'researcher', teamId: 'team-1' }, + writes: [ + { + id: `${id}-event`, + kind: 'fact' as const, + text: `${id} launch date is Friday`, + metadata: { eventId: `${id}-event`, actorId: 'researcher' }, + }, + ], + }, + { + id: 'delivery', + scope: { agentId: 'builder', teamId: 'team-1' }, + probes: [ + { + id: 'launch-date', + query: `${id} launch date`, + requiredFacts: [{ id: 'current', anyOf: [`${id} launch date is Friday`] }], + ...(includeStaleTarget + ? { + forbiddenFacts: [ + { + id: 'stale', + anyOf: [`${id} launch date is Thursday`], + obsolete: true, + }, + ], + } + : {}), + expectedEventIds: [`${id}-event`], + expectedActorIds: ['researcher'], + }, + ], + }, + ], + } +} diff --git a/tests/memory/lifecycle.test.ts b/tests/memory/lifecycle.test.ts new file mode 100644 index 0000000..decd3a3 --- /dev/null +++ b/tests/memory/lifecycle.test.ts @@ -0,0 +1,70 @@ +import { describe, expect, it } from 'vitest' +import { type AgentMemoryHit, runBoundedMemoryLifecycle } from '../../src/memory/index' +import { mergeRankedMemoryHits } from '../../src/memory/rank' + +describe('memory lifecycle', () => { + it('blocks later operations until timed-out work on the same adapter settles', async () => { + const resource = {} + const abortController = new AbortController() + let release: (() => void) | undefined + const pending = new Promise((resolve) => { + release = resolve + }) + let laterRuns = 0 + + await expect( + runBoundedMemoryLifecycle({ + operation: 'slow clear', + timeoutMs: 5, + resource, + abortController, + run: () => pending, + }), + ).rejects.toThrow('slow clear did not finish within 5ms') + expect(abortController.signal.aborted).toBe(true) + await expect( + runBoundedMemoryLifecycle({ + operation: 'unsafe close', + timeoutMs: 5, + resource, + run: () => { + laterRuns += 1 + }, + }), + ).rejects.toThrow("unsafe close cannot start because 'slow clear' is still running") + expect(laterRuns).toBe(0) + + release?.() + await pending + await new Promise((resolve) => setTimeout(resolve, 0)) + await runBoundedMemoryLifecycle({ + operation: 'settled close', + timeoutMs: 5, + resource, + run: () => { + laterRuns += 1 + }, + }) + expect(laterRuns).toBe(1) + }) + + it('keeps URI and text pairs distinct when newline-delimited identities collide', () => { + const first: AgentMemoryHit = { + id: 'first', + uri: 'memory://provider/a\nb', + kind: 'fact', + text: 'c', + } + const second: AgentMemoryHit = { + id: 'second', + uri: 'memory://provider/a', + kind: 'fact', + text: 'b\nc', + } + + expect(mergeRankedMemoryHits([[first], [second]]).map((hit) => hit.id)).toEqual([ + 'first', + 'second', + ]) + }) +}) diff --git a/tests/memory/mem0.test.ts b/tests/memory/mem0.test.ts new file mode 100644 index 0000000..b65a70e --- /dev/null +++ b/tests/memory/mem0.test.ts @@ -0,0 +1,603 @@ +import type { MemoryClient } from 'mem0ai' +import type { Memory as OssMemory } from 'mem0ai/oss' +import { describe, expect, it } from 'vitest' +import { + createAgentMemoryBranch, + createMem0MemoryAdapter, + type Mem0ClientLike, + mem0MemoryAdapterIdentity, +} from '../../src/memory/index' + +describe('Mem0 adapter', () => { + it('is type-compatible with both current Mem0 clients', () => { + const hosted = null as unknown as MemoryClient + const oss = null as unknown as OssMemory + + expect(createMem0MemoryAdapter({ client: hosted, mode: 'hosted' }).id).toBe('mem0-hosted') + expect(createMem0MemoryAdapter({ client: oss, mode: 'oss' }).id).toBe('mem0-oss') + }) + + it('writes and searches with the same provider scope', async () => { + const calls: Array<{ method: string; options?: Record }> = [] + const client: Mem0ClientLike = { + async add(_messages, options) { + calls.push({ method: 'add', options }) + return [{ id: 'memory-1', event: 'ADD' }] + }, + async search(_query, options) { + calls.push({ method: 'search', options }) + return { + results: [ + { + id: 'memory-1', + memory: 'Use concise status updates.', + score: 0.91, + metadata: { + memoryKind: 'preference', + memoryTitle: 'Writing style', + eventId: 'event-1', + }, + }, + ], + } + }, + } + const adapter = createMem0MemoryAdapter({ client, mode: 'hosted', rerank: true }) + const scope = { + tenantId: 'tenant-1', + userId: 'user-1', + agentId: 'agent-1', + teamId: 'team-1', + namespace: 'branch-1/private', + sessionId: 'session-1', + tags: { task: 'support' }, + } + + await adapter.write({ + id: 'event-1', + kind: 'preference', + title: 'Writing style', + text: 'Use concise status updates.', + scope, + metadata: { run_id: 'attempted-override', memoryKind: 'entity' }, + }) + const hits = await adapter.search('status style', { scope, limit: 3 }) + + const add = calls[0]?.options + const search = calls[1]?.options + expect(add).toMatchObject({ + userId: 'user-1', + agentId: 'agent-1', + runId: 'branch-1/private', + infer: true, + metadata: { + memoryKind: 'preference', + tenant_id: 'tenant-1', + user_id: 'user-1', + agent_id: 'agent-1', + team_id: 'team-1', + run_id: 'branch-1/private', + session_id: 'session-1', + tag_task: 'support', + }, + }) + expect(search).toMatchObject({ + filters: { + user_id: 'user-1', + agent_id: 'agent-1', + run_id: 'branch-1/private', + tenant_id: 'tenant-1', + team_id: 'team-1', + session_id: 'session-1', + tag_task: 'support', + }, + topK: 3, + rerank: true, + }) + expect(hits[0]).toMatchObject({ + id: 'memory-1', + kind: 'preference', + text: 'Use concise status updates.', + score: 0.91, + normalizedScore: 0.91, + metadata: { eventId: 'event-1' }, + }) + }) + + it('pushes memory kinds into provider filters before top-k truncation', async () => { + const searchOptions: Record[] = [] + const adapter = createMem0MemoryAdapter({ + mode: 'hosted', + client: { + async add() { + return [] + }, + async search(_query, options) { + searchOptions.push(options ?? {}) + const filters = options?.filters as Record | undefined + if (filters?.memoryKind === 'preference') { + return { + results: [ + { + id: 'preference-1', + memory: 'Use concise updates.', + metadata: { memoryKind: 'preference' }, + }, + ], + } + } + return { + results: [ + { + id: 'fact-1', + memory: 'Higher-ranked unrelated fact.', + metadata: { memoryKind: 'fact' }, + }, + ], + } + }, + }, + }) + + const filtered = await adapter.search('updates', { + scope: { userId: 'user-1' }, + kinds: ['preference'], + limit: 1, + }) + const unfiltered = await adapter.search('updates', { + scope: { userId: 'user-1' }, + kinds: [], + limit: 1, + }) + + expect(filtered.map((hit) => hit.id)).toEqual(['preference-1']) + expect(unfiltered.map((hit) => hit.id)).toEqual(['fact-1']) + expect(searchOptions[0]).toMatchObject({ + filters: { user_id: 'user-1', memoryKind: 'preference' }, + topK: 1, + }) + expect(searchOptions[1]).toMatchObject({ filters: { user_id: 'user-1' }, topK: 1 }) + }) + + it('rejects legacy asynchronous hosted responses the current SDK cannot produce', async () => { + const adapter = createMem0MemoryAdapter({ + mode: 'hosted', + client: { + async add() { + return { status: 'PENDING', eventId: 'event-1' } + }, + async search() { + return { results: [] } + }, + }, + }) + + await expect( + adapter.write({ + kind: 'fact', + text: 'Remember this.', + scope: { userId: 'user-1' }, + }), + ).rejects.toThrow('mem0ai 3.x must return a memory array') + }) + + it('requires fresh attempt branches for hosted Mem0', async () => { + const client: Mem0ClientLike = { + async add() { + return [] + }, + async search() { + return { results: [] } + }, + } + const visible = createMem0MemoryAdapter({ mode: 'hosted', client }) + + expect(() => createAgentMemoryBranch({ adapter: visible, branchId: 'mem0-resumable' })).toThrow( + "must use lifetime='attempt'", + ) + const attempt = createAgentMemoryBranch({ + adapter: visible, + branchId: 'mem0-attempt', + lifetime: 'attempt', + }) + const snapshot = await attempt.snapshot() + await attempt.close() + expect(() => + createAgentMemoryBranch({ + adapter: visible, + branchId: 'mem0-attempt', + snapshot, + }), + ).toThrow('attempt snapshots cannot be resumed') + }) + + it('does not journal an empty provider result as an accepted write', async () => { + const adapter = createMem0MemoryAdapter({ + mode: 'hosted', + client: { + async add() { + return [] + }, + async search() { + return { results: [] } + }, + }, + }) + + await expect( + adapter.write({ + kind: 'fact', + text: 'No provider mutation', + scope: { userId: 'user-1' }, + }), + ).resolves.toMatchObject({ accepted: false }) + }) + + it('journals and replays Mem0 deletion mutations', async () => { + const parentCalls: string[] = [] + const childCalls: string[] = [] + const adapter = (calls: string[]) => + createMem0MemoryAdapter({ + mode: 'oss', + client: { + async add(messages) { + const text = messages[0]!.content + calls.push(text) + return [ + { + id: text.startsWith('Forget') ? 'memory-old' : 'memory-new', + metadata: { event: text.startsWith('Forget') ? 'DELETE' : 'ADD' }, + }, + ] + }, + async search() { + return { results: [] } + }, + }, + }) + const parent = createAgentMemoryBranch({ + adapter: adapter(parentCalls), + branchId: 'mem0-parent', + baseScope: { agentId: 'agent-1' }, + }) + + await parent.write({ kind: 'fact', text: 'Remember the old launch date.' }) + const deletion = await parent.write({ kind: 'fact', text: 'Forget the old launch date.' }) + const snapshot = await parent.snapshot() + await parent.fork({ branchId: 'mem0-child', adapter: adapter(childCalls) }) + + expect(deletion).toMatchObject({ accepted: true, metadata: { event: 'DELETE' } }) + expect(snapshot.journal.map((entry) => entry.input.text)).toEqual([ + 'Remember the old launch date.', + 'Forget the old launch date.', + ]) + expect(childCalls).toEqual(parentCalls) + }) + + it('includes the default scope in the Mem0 adapter identity', () => { + const base = { mode: 'hosted' as const, id: 'mem0', backendRef: 'mem0-account-a' } + expect(mem0MemoryAdapterIdentity({ ...base, defaultScope: { tenantId: 'tenant-a' } })).not.toBe( + mem0MemoryAdapterIdentity({ ...base, defaultScope: { tenantId: 'tenant-b' } }), + ) + expect(mem0MemoryAdapterIdentity({ ...base, ingestionTimeoutMs: 100 })).not.toBe( + mem0MemoryAdapterIdentity({ ...base, ingestionTimeoutMs: 200 }), + ) + }) + + it('passes OSS entity filters on add', async () => { + let addOptions: Record | undefined + const adapter = createMem0MemoryAdapter({ + mode: 'oss', + client: { + async add(_messages, options) { + addOptions = options + return { results: [{ id: 'oss-1', memory: 'Remembered' }] } + }, + async search() { + return { results: [] } + }, + }, + }) + + await adapter.write({ + kind: 'fact', + text: 'Remembered', + scope: { userId: 'user-1', agentId: 'agent-1', namespace: 'run-1' }, + }) + + expect(addOptions).toMatchObject({ + userId: 'user-1', + agentId: 'agent-1', + runId: 'run-1', + filters: { user_id: 'user-1', agent_id: 'agent-1', run_id: 'run-1' }, + }) + }) + + it('rejects direct OSS operations without a provider entity scope', async () => { + let providerCalls = 0 + const adapter = createMem0MemoryAdapter({ + mode: 'oss', + client: { + async add() { + providerCalls += 1 + return { results: [] } + }, + async search() { + providerCalls += 1 + return { results: [] } + }, + }, + }) + + await expect(adapter.write({ kind: 'fact', text: 'No owner.' })).rejects.toThrow( + 'requires scope.userId, scope.agentId, scope.runId, or scope.namespace', + ) + await expect(adapter.search('No owner.')).rejects.toThrow( + 'requires scope.userId, scope.agentId, scope.runId, or scope.namespace', + ) + expect(providerCalls).toBe(0) + }) + + it('rejects unscoped hosted operations before calling Mem0', async () => { + let providerCalls = 0 + const adapter = createMem0MemoryAdapter({ + mode: 'hosted', + appId: 'support-app', + client: { + async add() { + providerCalls += 1 + return [] + }, + async search() { + providerCalls += 1 + return { results: [] } + }, + async getAll() { + providerCalls += 1 + return { results: [] } + }, + async delete() { + providerCalls += 1 + return { message: 'deleted' } + }, + }, + }) + + await expect(adapter.write({ kind: 'fact', text: 'No owner.' })).rejects.toThrow( + 'Mem0 hosted write requires', + ) + await expect(adapter.search('No owner.')).rejects.toThrow('Mem0 hosted search requires') + await expect(adapter.clear?.()).rejects.toThrow('refusing an unscoped Mem0 clear') + expect(providerCalls).toBe(0) + }) + + it('deletes only memories matching a complete scoped filter', async () => { + const rows = new Set(['memory-1', 'memory-2']) + const getAllOptions: Record[] = [] + let deleteAllCalls = 0 + const adapter = createMem0MemoryAdapter({ + mode: 'hosted', + appId: 'support-app', + client: { + async add() { + return [] + }, + async search() { + return { results: [] } + }, + async getAll(options) { + getAllOptions.push(options ?? {}) + return { results: [...rows].map((id) => ({ id, memory: id })) } + }, + async delete(memoryId) { + rows.delete(memoryId) + return { message: 'deleted' } + }, + async deleteAll() { + deleteAllCalls += 1 + return { message: 'deleted all' } + }, + }, + }) + + await adapter.clear?.({ + tenantId: 'tenant-1', + userId: 'user-1', + namespace: 'physical-branch', + runId: 'logical-run', + sessionId: 'session-1', + tags: { visibility: 'private' }, + }) + + expect(rows.size).toBe(0) + expect(deleteAllCalls).toBe(0) + expect(getAllOptions[0]).toMatchObject({ + filters: { + app_id: 'support-app', + user_id: 'user-1', + run_id: 'physical-branch', + tenant_id: 'tenant-1', + logical_run_id: 'logical-run', + session_id: 'session-1', + tag_visibility: 'private', + }, + page: 1, + pageSize: 100, + showExpired: true, + }) + }) + + it('waits for hosted deletes to disappear without deleting the same memory twice', async () => { + let deletedAt: number | undefined + let getAllCalls = 0 + const deletedIds: string[] = [] + const adapter = createMem0MemoryAdapter({ + mode: 'hosted', + ingestionTimeoutMs: 50, + pollIntervalMs: 1, + client: { + async add() { + return [] + }, + async search() { + return { results: [] } + }, + async getAll() { + getAllCalls += 1 + const visible = deletedAt === undefined || Date.now() - deletedAt < 5 + return { results: visible ? [{ id: 'memory-1', memory: 'stale listing' }] : [] } + }, + async delete(memoryId) { + deletedIds.push(memoryId) + deletedAt = Date.now() + return { message: 'deletion accepted' } + }, + }, + }) + + await adapter.clear?.({ userId: 'user-1', namespace: 'branch-1' }) + + expect(deletedIds).toEqual(['memory-1']) + expect(getAllCalls).toBeGreaterThan(2) + }) + + it('waits for list and search indexes to drop deleted memories before reporting cleanup', async () => { + let deleted = false + let getAllCalls = 0 + let searchCalls = 0 + let deleteCalls = 0 + const adapter = createMem0MemoryAdapter({ + mode: 'hosted', + ingestionTimeoutMs: 50, + pollIntervalMs: 1, + client: { + async add() { + return [] + }, + async search() { + searchCalls += 1 + const visible = !deleted || searchCalls === 1 + return { results: visible ? [{ id: 'memory-1', memory: 'stale listing' }] : [] } + }, + async getAll() { + getAllCalls += 1 + const visible = !deleted || getAllCalls === 2 + return { results: visible ? [{ id: 'memory-1', memory: 'stale listing' }] : [] } + }, + async delete() { + deleteCalls += 1 + deleted = true + return { message: 'deletion accepted' } + }, + }, + }) + + await adapter.clear?.({ userId: 'user-1', namespace: 'branch-1' }) + + expect(deleteCalls).toBe(1) + expect(getAllCalls).toBeGreaterThan(1) + expect(searchCalls).toBeGreaterThan(1) + }) + + it('waits for a fresh narrower id-less write when clearing a broader Mem0 scope', async () => { + const writtenAt = Date.now() + let deleted = false + let getAllCalls = 0 + let deleteCalls = 0 + const visible = () => !deleted && Date.now() - writtenAt >= 5 + const adapter = createMem0MemoryAdapter({ + mode: 'hosted', + ingestionTimeoutMs: 15, + pollIntervalMs: 1, + client: { + async add() { + return [{ event: 'ADD' }] + }, + async search() { + return { results: visible() ? [{ id: 'late-memory', memory: 'late fact' }] : [] } + }, + async getAll() { + getAllCalls += 1 + return { results: visible() ? [{ id: 'late-memory', memory: 'late fact' }] : [] } + }, + async delete(memoryId) { + expect(memoryId).toBe('late-memory') + deleteCalls += 1 + deleted = true + return { message: 'deleted' } + }, + }, + }) + + await adapter.write({ + kind: 'fact', + text: 'late fact', + scope: { userId: 'user-1', agentId: 'agent-1', namespace: 'branch-1' }, + }) + await adapter.clear?.({ userId: 'user-1' }) + + expect(deleteCalls).toBe(1) + expect(getAllCalls).toBeGreaterThan(1) + }) + + it('expires pending write probes after the configured visibility window', async () => { + let searchCalls = 0 + const adapter = createMem0MemoryAdapter({ + mode: 'hosted', + ingestionTimeoutMs: 1, + pollIntervalMs: 1, + client: { + async add() { + return [{ event: 'ADD' }] + }, + async search() { + searchCalls += 1 + return { results: [] } + }, + async getAll() { + return { results: [] } + }, + async delete() { + return { message: 'deleted' } + }, + }, + }) + + await adapter.write({ + kind: 'fact', + text: 'bounded pending fact', + scope: { userId: 'user-1' }, + }) + await new Promise((resolve) => setTimeout(resolve, 5)) + await adapter.clear?.({ userId: 'user-1' }) + + expect(searchCalls).toBe(0) + }) + + it('refuses to clear Mem0 without an exact identity scope', async () => { + let providerCalls = 0 + const adapter = createMem0MemoryAdapter({ + mode: 'hosted', + client: { + async add() { + return [] + }, + async search() { + providerCalls += 1 + return { results: [] } + }, + async getAll() { + providerCalls += 1 + return { results: [] } + }, + async delete() { + providerCalls += 1 + return { message: 'deleted' } + }, + }, + }) + + await expect(adapter.clear?.()).rejects.toThrow('refusing an unscoped Mem0 clear') + expect(providerCalls).toBe(0) + }) +}) diff --git a/tests/memory/neo4j.test.ts b/tests/memory/neo4j.test.ts new file mode 100644 index 0000000..6f8cdf8 --- /dev/null +++ b/tests/memory/neo4j.test.ts @@ -0,0 +1,204 @@ +import type { MemoryClient as Neo4jMemoryClient } from '@neo4j-labs/agent-memory' +import { describe, expect, it } from 'vitest' +import { createNeo4jAgentMemoryAdapter } from '../../src/memory/index' + +describe('Neo4j Agent Memory adapter', () => { + it('is type-compatible with the current official client', () => { + const client = {} as unknown as Neo4jMemoryClient + expect( + createNeo4jAgentMemoryAdapter({ client, transport: 'rest', branchId: 'branch-1' }).id, + ).toBe('neo4j-agent-memory') + }) + + it('stores observations as searchable messages without retrying a successful void write', async () => { + let camelWrites = 0 + let snakeWrites = 0 + let role: unknown + let closes = 0 + const adapter = createNeo4jAgentMemoryAdapter({ + transport: 'rest', + branchId: 'branch-1', + client: { + shortTerm: { + async addMessage(_sessionId: unknown, inputRole: unknown) { + camelWrites += 1 + role = inputRole + return undefined + }, + async add_message() { + snakeWrites += 1 + return { id: 'duplicate' } + }, + async searchMessages() { + return [ + { + id: 'observation-1', + role: 'assistant', + content: 'The deployment completed.', + metadata: { agentKnowledgeKind: 'observation' }, + }, + ] + }, + }, + async close() { + closes += 1 + }, + }, + }) + + const write = await adapter.write({ + kind: 'observation', + role: 'tool', + text: 'The deployment completed.', + scope: { sessionId: 'session-1' }, + }) + const hits = await adapter.search('deployment', { + kinds: ['observation'], + scope: { sessionId: 'session-1' }, + }) + await adapter.close?.() + + expect(write.accepted).toBe(true) + expect(camelWrites).toBe(1) + expect(snakeWrites).toBe(0) + expect(role).toBe('assistant') + expect(hits).toMatchObject([{ id: 'observation-1', kind: 'observation' }]) + expect(closes).toBe(1) + }) + + it('fails clearly when REST is asked for bridge-only facts and preferences', async () => { + const adapter = createNeo4jAgentMemoryAdapter({ + transport: 'rest', + client: { longTerm: {} }, + }) + + await expect(adapter.write({ kind: 'fact', text: 'A fact' })).rejects.toThrow( + 'Neo4j REST cannot write fact', + ) + await expect(adapter.search('preference', { kinds: ['preference'] })).rejects.toThrow( + 'Neo4j rest cannot search memory kinds: preference', + ) + }) + + it('rejects an incompatible Neo4j client instead of reporting an empty search', async () => { + const adapter = createNeo4jAgentMemoryAdapter({ + client: { longTerm: {} }, + transport: 'rest', + }) + + await expect(adapter.search('company', { kinds: ['entity'] })).rejects.toThrow( + 'missing method searchEntities', + ) + }) + + it('rejects non-throwing Neo4j write errors', async () => { + const adapter = createNeo4jAgentMemoryAdapter({ + transport: 'rest', + client: { + shortTerm: { + async addMessage() { + return { success: false, error: { message: 'database unavailable' } } + }, + }, + }, + }) + + await expect( + adapter.write({ + kind: 'message', + text: 'Remember this.', + scope: { sessionId: 'session-1' }, + }), + ).rejects.toThrow('database unavailable') + }) + + it('maps hosted reasoning memories to recordStep', async () => { + let recorded: Record | undefined + const adapter = createNeo4jAgentMemoryAdapter({ + transport: 'rest', + client: { + reasoning: { + async recordStep(input: Record) { + recorded = input + return { id: 'step-1', ...input } + }, + }, + }, + }) + + const result = await adapter.write({ + kind: 'reasoning-trace', + text: 'Check the deployment status.', + title: 'status-check', + scope: { sessionId: 'conversation-1' }, + metadata: { action: 'query_status', result: 'healthy' }, + }) + + expect(recorded).toEqual({ + conversationId: 'conversation-1', + reasoning: 'Check the deployment status.', + actionTaken: 'query_status', + result: 'healthy', + }) + expect(result).toMatchObject({ accepted: true, id: 'step-1', kind: 'reasoning-trace' }) + }) + + it('uses bridge preference APIs only in bridge mode', async () => { + const adapter = createNeo4jAgentMemoryAdapter({ + transport: 'bridge', + client: { + longTerm: { + async addPreference(category: string, preference: string) { + return { id: 'preference-1', category, preference } + }, + async searchPreferences() { + return [{ id: 'preference-1', category: 'style', preference: 'Be concise.' }] + }, + }, + }, + }) + + await expect( + adapter.write({ kind: 'preference', category: 'style', text: 'Be concise.' }), + ).resolves.toMatchObject({ id: 'preference-1' }) + await expect(adapter.search('concise', { kinds: ['preference'] })).resolves.toMatchObject([ + { id: 'preference-1', kind: 'preference', text: 'Be concise.' }, + ]) + }) + + it('fuses unscored Neo4j memory-type rankings before applying the limit', async () => { + const adapter = createNeo4jAgentMemoryAdapter({ + transport: 'bridge', + client: { + shortTerm: { + async searchMessages() { + return [ + { id: 'message-1', role: 'user', content: 'first message' }, + { id: 'message-2', role: 'user', content: 'second message' }, + ] + }, + }, + longTerm: { + async searchEntities() { + return [ + { id: 'entity-1', name: 'first entity', type: 'custom' }, + { id: 'entity-2', name: 'second entity', type: 'custom' }, + ] + }, + async searchPreferences() { + return [] + }, + }, + reasoning: { + async getSimilarTraces() { + return [] + }, + }, + }, + }) + + const hits = await adapter.search('anything', { limit: 2 }) + + expect(hits.map((hit) => hit.id)).toEqual(['message-1', 'entity-1']) + }) +}) diff --git a/tests/support/benchmarks.ts b/tests/support/benchmarks.ts new file mode 100644 index 0000000..9b046fa --- /dev/null +++ b/tests/support/benchmarks.ts @@ -0,0 +1,13 @@ +import { + type RunMemoryAdapterBenchmarkOptions, + runMemoryAdapterBenchmark as runMemoryAdapterBenchmarkRaw, +} from '../../src/benchmarks/index' + +export function runMemoryAdapterBenchmark(options: RunMemoryAdapterBenchmarkOptions) { + return runMemoryAdapterBenchmarkRaw({ + ...options, + ...(options.storage && !options.controllerMode && !options.acquireRunLease + ? { controllerMode: 'process-local' as const } + : {}), + }) +} diff --git a/tests/support/memory.ts b/tests/support/memory.ts new file mode 100644 index 0000000..9381514 --- /dev/null +++ b/tests/support/memory.ts @@ -0,0 +1,91 @@ +import { + type AgentMemoryAdapter, + type AgentMemoryHit, + type AgentMemoryScope, + type RunAgentMemoryExperimentOptions, + type RunAgentMemoryImprovementOptions, + runAgentMemoryExperiment as runAgentMemoryExperimentRaw, + runAgentMemoryImprovement as runAgentMemoryImprovementRaw, +} from '../../src/memory/index' + +function withProcessLocalController< + T extends { + storage?: unknown + controllerMode?: 'process-local' + acquireRunLease?: unknown + }, +>(options: T): T { + if (!options.storage || options.controllerMode || options.acquireRunLease) return options + return { ...options, controllerMode: 'process-local' } +} + +export function runAgentMemoryExperiment(options: RunAgentMemoryExperimentOptions) { + return runAgentMemoryExperimentRaw(withProcessLocalController(options)) +} + +export function runAgentMemoryImprovement( + options: RunAgentMemoryImprovementOptions, +) { + return runAgentMemoryImprovementRaw(withProcessLocalController(options)) +} + +export function hitText(hit: AgentMemoryHit): string { + return hit.text +} + +export function createScopedTestAdapter( + id: string, + beforeWrite?: (scope: AgentMemoryScope, text: string) => Promise, +): AgentMemoryAdapter { + const rows: Array<{ scope: AgentMemoryScope; hit: AgentMemoryHit }> = [] + let sequence = 0 + return { + id, + branchIsolation: { mode: 'scoped' }, + async search(_query, options = {}) { + return rows.filter((row) => sameScope(row.scope, options.scope)).map((row) => row.hit) + }, + async getContext(query, options = {}) { + const hits = await this.search(query, options) + return { query, hits, text: hits.map(hitText).join('\n'), sourceRecords: [] } + }, + async write(input) { + const scope = input.scope ?? {} + await beforeWrite?.(scope, input.text) + sequence += 1 + const memoryId = input.id ?? `${id}-${sequence}` + const hit: AgentMemoryHit = { + id: memoryId, + uri: `memory://${id}/${memoryId}`, + kind: input.kind, + text: input.text, + metadata: input.metadata, + } + rows.push({ scope, hit }) + return { accepted: true, id: memoryId, uri: hit.uri, kind: input.kind } + }, + async clear(scope) { + for (let index = rows.length - 1; index >= 0; index -= 1) { + if (sameScope(rows[index]!.scope, scope)) rows.splice(index, 1) + } + }, + } +} + +function sameScope(left: AgentMemoryScope, right: AgentMemoryScope = {}): boolean { + for (const key of [ + 'tenantId', + 'userId', + 'agentId', + 'teamId', + 'runId', + 'sessionId', + 'namespace', + ] as const) { + if (right[key] !== undefined && left[key] !== right[key]) return false + } + for (const [key, value] of Object.entries(right.tags ?? {})) { + if (left.tags?.[key] !== value) return false + } + return true +}