From d5ece495193583462391dbf5f662c68bb122091d Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Fri, 17 Jul 2026 17:41:59 -0600 Subject: [PATCH 1/4] feat(memory): add branchable provider improvement system --- README.md | 196 ++- package.json | 1 + pnpm-lock.yaml | 2638 +++++++++++++++++++++++++++++++++- src/benchmarks/index.ts | 204 ++- src/memory/branch.ts | 707 +++++++++ src/memory/experiment.ts | 957 ++++++++++++ src/memory/graphiti.ts | 552 +++++++ src/memory/improvement.ts | 1012 +++++++++++++ src/memory/index.ts | 5 + src/memory/mem0.ts | 376 +++++ src/memory/neo4j.ts | 464 +++--- src/memory/schemas.ts | 3 + src/memory/types.ts | 17 + tests/benchmarks.test.ts | 28 + tests/memory-adapter.test.ts | 106 +- tests/memory-holdout.test.ts | 38 +- tests/memory-systems.test.ts | 2306 +++++++++++++++++++++++++++++ 17 files changed, 9229 insertions(+), 381 deletions(-) create mode 100644 src/memory/branch.ts create mode 100644 src/memory/experiment.ts create mode 100644 src/memory/graphiti.ts create mode 100644 src/memory/improvement.ts create mode 100644 src/memory/mem0.ts create mode 100644 tests/memory-systems.test.ts diff --git a/README.md b/README.md index 5a51a47..3952d25 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ This package turns raw sources and generated markdown knowledge into a versionab - [Design](#design): the invariants, including immutable sources, cited claims, and deterministic graph output - [Benchmark harness](#benchmark-harness): BEIR/MTEB/qrels, RAG answer, hallucination, KB-improvement cases - [Agent-Eval integration](#agent-eval-integration): retrieval eval, readiness bundles, and release reports -- [Memory adapters](#memory-adapters): generic memory contract plus Neo4j Agent Memory bridge +- [Memory adapters](#memory-adapters): Mem0, Graphiti, Neo4j, branching, and automatic improvement - [Research loop](#research-loop): `runKnowledgeResearchLoop` plus the control-loop adapter - [Runtime integration](#runtime-integration): how agent runners plug into the pure KB loop - [Pluggable knowledge sources](#pluggable-knowledge-sources): live authorities → eval re-runs @@ -51,10 +51,10 @@ Storage stays consumer-owned via `KbStore` (`MemoryKbStore`, `FileSystemKbStore` |---|---|---| | LLM wiki or docs KB | Source records, cited markdown pages, lint, readiness scoring, candidate promotion | `improveKnowledgeBase` or `runKnowledgeResearchLoop` | | RAG | Retrieval eval, source-span labels, answer quality checks, missing/stale/noisy-source diagnosis | `runRagKnowledgeImprovementLoop` | -| Memory DB | Adapter contract that turns memory hits into source-like evidence and benchmark rows | `/memory` plus `runMemoryAdapterBenchmark` | +| Memory DB | Provider adapters, isolated branches, sharing policies, ordered histories, and fresh comparison | `/memory` plus `runAgentMemoryExperiment` | | From scratch | Empty KB layout, source ingestion, write-block application, indexing, readiness checks | CLI `init` or `runKnowledgeResearchLoop` | | Existing KB | Candidate workspace, resume state, base-hash conflict check, exact approved promotion | `improveKnowledgeBase` | -| Parallel agents | Per-run locks, isolated candidates, frozen candidate handoff, retry with the same `runId` | `improveKnowledgeBase` with runtime workers | +| Parallel agents | Isolated KB candidates or memory branches, ordered per-agent writes, concurrent histories, and explicit activation | `improveKnowledgeBase` or `runAgentMemoryImprovement` | | One-call agent job | Runtime supervisor plus all KB mechanics above | `runKnowledgeImprovementJob` from `@tangle-network/agent-runtime/knowledge` | ## CLI @@ -171,7 +171,6 @@ import { runMemoryAdapterBenchmark, runKnowledgeBenchmarkSuite, } from '@tangle-network/agent-knowledge/benchmarks' -import { createNeo4jAgentMemoryAdapter } from '@tangle-network/agent-knowledge/memory' await runKnowledgeBenchmarkSuite({ cases: buildIndustryRagBenchmarkSmokeCases(), @@ -198,11 +197,6 @@ const memoryRanking = await runMemoryAdapterBenchmark({ createAdapter: () => createInMemoryBenchmarkAdapter(), searchLimit: 1, }, - { - id: 'neo4j-agent-memory', - createAdapter: () => createNeo4jAgentMemoryAdapter({ client: neo4jMemoryClient }), - searchLimit: 5, - }, ], }) @@ -230,6 +224,9 @@ const result = await runKnowledgeBenchmarkSuite({ console.log(result.report.score.mean) ``` +`runMemoryAdapterBenchmark()` is for simple adapters whose request scope fully isolates each case. +Use `runAgentMemoryExperiment()` below for external providers, retries, resume, parallel candidates, and branch cleanup. + Billable responders must execute paid work through `context.cost.runPaidCall()`; `costUsd` on an artifact is display-only. Use `buildRetrievalBenchmarkCasesFromQrels()` for qrels-backed retrieval datasets. The smoke pack proves that every declared benchmark family is wired through the runner; full BEIR, MTEB, MS MARCO, TREC DL, MIRACL, LoTTE, BRIGHT, CRAG, HotpotQA, KILT, RAGTruth, and FaithBench runs should pass real dataset rows through the same case shapes. @@ -239,7 +236,8 @@ Use `KnowledgeMemoryBenchmarkCase` for memory systems. Memory cases carry ordered events plus current required facts, stale forbidden facts, expected event IDs, and expected actor IDs. The built-in memory scorer reports current fact recall, stale-memory safety, event recall, and actor recall, which maps to LoCoMo, LongMemEval, Memora, MemoryAgentBench, MemoryBank-style personalization, GroupMemBench, and first-party memory lifecycle checks. Use `runMemoryAdapterBenchmark()` to rank real memory systems that implement `AgentMemoryAdapter`. -This is the path for Neo4j Agent Memory, Mem0, Zep, Letta, Graphiti, a vector store, or a product-owned memory backend. +Built-in adapters cover Neo4j Agent Memory, Mem0 hosted and open source, and the official Graphiti MCP server. +Zep, Letta, a vector store, or a product-owned backend can implement the same small contract. ## Agent-Eval Integration @@ -429,35 +427,179 @@ Pass `readiness.report` to `blockingKnowledgeEval()` from ## Memory Adapters -`agent-knowledge` does not store operational memory itself. It defines the -contract that lets a runtime read/write memory through any backend, then turn -memory hits into `SourceRecord` evidence for readiness, linting, and eval gates. +`agent-knowledge` does not replace a memory database. +It gives existing memory systems one contract, isolated branches, ordered multi-agent histories, comparable measurements, and an automatic improvement loop. + +Use the provider adapter that matches the system you already run: ```ts import { + createGraphitiMemoryAdapter, + createMem0MemoryAdapter, createNeo4jAgentMemoryAdapter, - memoryHitToSourceRecord, } from '@tangle-network/agent-knowledge/memory' -const memory = createNeo4jAgentMemoryAdapter({ client: neo4jMemoryClient }) +const mem0 = createMem0MemoryAdapter({ + client: mem0Client, + mode: 'hosted', // use 'oss' with mem0ai/oss Memory +}) + +const graphiti = createGraphitiMemoryAdapter({ + client: graphitiMcpClient, +}) + +const neo4j = createNeo4jAgentMemoryAdapter({ + client: neo4jMemoryClient, + transport: 'rest', + contextMode: 'search', +}) +``` + +The Graphiti defaults match the current official server tools: `add_memory`, `search_memory_facts`, `search_nodes`, `get_episodes`, and `clear_graph`. +Set `toolNames` explicitly when your server lists different names. +The adapter never retries a write under another name because that could enqueue it twice. +Visible writes expand the episode scan when Graphiti truncates a long group and fail explicitly at `episodeScanLimit` instead of reporting a false success. + +Each adapter supports the same `search`, `getContext`, and `write` calls. +Mem0 and Graphiti apply tenant, user, agent, team, run, session, namespace, and tag scopes per request. +Neo4j Agent Memory experiments require disposable external state per branch because the SDK cannot clear every memory kind atomically. +Pass `branchId` only after the caller has provisioned that isolation. +Provider-specific identifiers remain internal to the adapter. +Custom adapters must declare `branchIsolation: { mode: 'scoped' }` only when every operation enforces the supplied scope; undeclared adapters are rejected by branch execution. + +```ts +const neo4jInstances = new WeakMap() + +const neo4jCandidate = { + id: 'neo4j', + policy: { read: ['shared'], write: 'shared' }, + async createAdapter({ branchId }: { branchId: string }) { + const client = await provisionDedicatedNeo4jMemory({ branchId, transport: 'rest' }) + const adapter = createNeo4jAgentMemoryAdapter({ client, transport: 'rest', branchId }) + neo4jInstances.set(adapter, branchId) + return adapter + }, + async disposeAdapter(adapter: object) { + const branchId = neo4jInstances.get(adapter) + if (!branchId) throw new Error('unknown Neo4j branch') + await destroyDedicatedNeo4jMemory({ branchId }) + }, +} +``` + +Hosted REST supports messages, entities, and conversation reasoning steps. +The bridge transport also supports facts, preferences, and reusable reasoning traces. +The adapter requires `transport: 'rest' | 'bridge'` and rejects unsupported memory kinds before making a provider call. +`contextMode: 'search'` is the default and preserves query-based behavior across providers. +Set `contextMode: 'native'` with hosted REST to use Neo4j's whole-conversation context for message and observation memory. + +Use a branch when agents or candidate configurations must not share state accidentally: -const context = await memory.getContext('What does this user prefer?', { - scope: { userId: 'user-123', sessionId: 'session-456' }, +```ts +import { createAgentMemoryBranch } from '@tangle-network/agent-knowledge/memory' + +const memory = createAgentMemoryBranch({ + adapter: mem0, + branchId: 'candidate-17', + policy: { read: ['private', 'team'], write: 'team' }, + baseScope: { tenantId: 'acme', teamId: 'research' }, +}) + +await memory.write({ + kind: 'fact', + text: 'The launch date is Friday.', + scope: { agentId: 'researcher' }, +}) + +const context = await memory.getContext('When is launch?', { + scope: { agentId: 'builder' }, limit: 5, }) -const sourceRecords = context.hits.map((hit) => - memoryHitToSourceRecord(hit, { scope: { userId: 'user-123' } }), -) +const snapshot = await memory.snapshot() +const resumed = createAgentMemoryBranch({ + adapter: mem0, + branchId: 'candidate-17', + snapshot, +}) +``` + +Private writes stay visible only to one agent. +Team writes are shared inside one team. +Shared writes are visible to every agent in the branch. +`fork()` replays accepted writes into a child branch, including a branch backed by a different provider. + +Use `runAgentMemoryExperiment()` to compare known providers or configurations across ordered histories. +`buildAgentMemorySequencesFromBenchmarkCases()` feeds existing LoCoMo, LongMemEval, GroupMemBench, or first-party memory cases into the same runner without changing their event order. +Use `runAgentMemoryImprovement()` when a proposer should search configurations, measure them on training histories, compare the winner with the baseline on fresh histories, and activate it only after a statistically supported win. + +```ts +const result = await runAgentMemoryImprovement({ + experimentId: 'support-memory-v3', + trainSequences, + holdoutSequences, + seeds: [ + { + config: currentConfig, + track: 'conservative', + proposer: 'conservative', + vision: 'Improve recall without increasing stale-memory reuse.', + }, + { + config: graphConfig, + track: 'graph', + proposer: 'graph', + vision: 'Improve temporal and multi-actor reasoning.', + }, + ], + proposer: configProposer, + proposers: { + conservative: conservativeConfigProposer, + graph: graphConfigProposer, + }, + improvementRef: 'memory-stack@8f2c1a7', + budget: { maxSteps: 12 }, + maxTotalCostUsd: 25, + candidateConcurrency: 4, + sequenceConcurrency: 8, + createCandidate: ({ config }) => buildMemoryCandidate(config), + executeStep: ({ memory, step, context }) => + runProfileStep({ memory, step, context }), + executeStepRef: 'support-profile@8f2c1a7', + runDir: '.agent-knowledge/memory-runs/support-v3', + onPromote: ({ config }) => activateMemoryConfig(config), +}) ``` -The Neo4j adapter is runtime dependency-free: pass the real -`@neo4j-labs/agent-memory` client in products, or a fake client in tests. CI -typechecks against `@neo4j-labs/agent-memory@0.4.0` and covers the published -TypeScript SDK surface: `shortTerm.addMessage/searchMessages/getContext`, -`longTerm.addEntity/addPreference/addFact/searchEntities/searchPreferences`, -and `reasoning.getSimilarTraces`. Generic `search` / `getContext` and -snake_case bridge-style methods remain supported for non-hosted clients. +`candidateConcurrency` controls how many configurations are measured at once. +`sequenceConcurrency` controls how many independent histories run at once. +The first seed is always the deployed baseline. +Each seed starts an independent track with its own objective and optional proposer, and a branch inherits its parent track's proposer unless the governor chooses another one. +Set each candidate's `externalCostUsdPerSequence` to a conservative memory-provider charge; model calls inside `executeStep` should use `context.cost.runPaidCall()` so `maxTotalCostUsd` counts both sources before starting more work. +The same budget is supplied to named proposers as `context.costLedger` and to the governor as `context.costLedger`; the standard `agent-eval` proposers charge their model calls there. +Steps inside one history remain ordered, and writes from one agent remain ordered while independent agents can write concurrently. +Runtime callbacks may write only to scopes declared on a step, write, probe, or `sequence.cleanupScopes`; this lets an interrupted retry clear every actor and session it could have touched. +Search never changes the live configuration. +Only `onPromote` can activate the fresh-history winner. +The callback receives a stable `activationId`; use it as the idempotency key when activation updates an external service. +The default filesystem path permits one controller and many parallel workers, persists candidate history, and resumes without adding more than `maxSteps` candidate nodes. +Distributed controllers using custom storage provide `acquireRunLease`; independent workers still use `candidateConcurrency` and `sequenceConcurrency`. +Workers that execute one experiment from different local paths must share an explicit `experimentRunId`; otherwise the resolved `runDir` deliberately creates a different provider branch namespace. +Timed-out histories finish branch cleanup before `runAgentMemoryExperiment()` returns, and a cleanup failure fails the run. +The persisted run identity includes `budget.maxSteps`; start a new `runDir` to enlarge a completed search instead of mutating its history. +Promotion is blocked when a configured critical dimension is missing or incomplete on either fresh-history arm. +Improvement runs clear each scoped branch before and after a measured history by default, including recovery after an interrupted cell. +Set `cleanupBranches: false` only when the candidate creates and disposes a physically isolated provider instance per cell. +The current Neo4j SDK has no complete all-memory clear operation, so a Neo4j improvement run must use disposable per-cell service instances with `cleanupBranches: false` and `disposeAdapter`; a persistent shared workspace is rejected because failed retries could read partial state. + +The package stays independent of `agent-runtime`. +The `executeStep` callback is where a runtime profile, coding agent, research agent, or deterministic worker uses the branch. +This keeps provider and measurement code reusable while the profile owns what agents do and what they choose to remember. + +The Neo4j adapter accepts the real `@neo4j-labs/agent-memory` client and rejects unsafe branch reuse unless `branchId` identifies caller-provisioned isolated state. +The Mem0 adapter typechecks against `mem0ai` hosted and open-source clients. +The Graphiti adapter calls the official MCP tools and waits for queued episodes to become visible before evaluating them by default. +`consistency: 'queued'` is available for direct ingestion, but branch experiments reject it because a queued write can appear after a crashed process has already cleaned the branch. ## Research Loop diff --git a/package.json b/package.json index 8be44de..626e0c8 100644 --- a/package.json +++ b/package.json @@ -80,6 +80,7 @@ "@tangle-network/sandbox": "^0.9.7", "@types/node": "^25.6.0", "@types/proper-lockfile": "4.1.4", + "mem0ai": "3.1.0", "tsup": "^8.0.0", "tsx": "^4.22.4", "typescript": "^5.7.0", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 4a0c904..3aa1d7d 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -40,6 +40,9 @@ importers: '@types/proper-lockfile': specifier: 4.1.4 version: 4.1.4 + mem0ai: + specifier: 3.1.0 + version: 3.1.0(@anthropic-ai/sdk@0.40.1)(@azure/identity@4.13.1)(@azure/search-documents@12.2.0)(@cloudflare/workers-types@4.20260702.1)(@google/genai@1.52.0)(@langchain/core@1.2.3(@opentelemetry/api@1.9.1)(ws@8.21.0))(@mistralai/mistralai@1.15.1)(@qdrant/js-client-rest@1.18.0(typescript@5.9.3))(@supabase/supabase-js@2.110.7)(@types/jest@29.5.14)(@types/pg@8.11.0)(better-sqlite3@12.11.1)(cloudflare@4.5.0)(compromise@14.16.0)(groq-sdk@0.3.0)(mongodb@7.2.0)(natural@8.1.1(@opentelemetry/api@1.9.1))(ollama@0.5.18)(pg@8.11.3)(redis@4.7.1)(ws@8.21.0) tsup: specifier: ^8.0.0 version: 8.5.1(postcss@8.5.13)(tsx@4.22.4)(typescript@5.9.3)(yaml@2.8.4) @@ -58,6 +61,9 @@ packages: '@adraffy/ens-normalize@1.11.1': resolution: {integrity: sha512-nhCBV3quEgesuf7c7KYfperqSS14T8bYuvJ8PcLJp6znkZpFc0AuW4qBtr8eKVyPPe/8RSr7sglCWPU5eaxwKQ==} + '@anthropic-ai/sdk@0.40.1': + resolution: {integrity: sha512-DJMWm8lTEM9Lk/MSFL+V+ugF7jKOn0M2Ujvb5fN8r2nY14aHbGPZ1k6sgjL+tpJ3VuOGJNG+4R83jEpOuYPv8w==} + '@asteasolutions/zod-to-openapi@8.5.0': resolution: {integrity: sha512-SABbKiObg5dLRiTFnqiW1WWwGcg1BJfmHtT2asIBnBHg6Smy/Ms2KHc650+JI4Hw7lSkdiNebEGXpwoxfben8Q==} peerDependencies: @@ -72,6 +78,73 @@ packages: zod: optional: true + '@azure/abort-controller@2.2.0': + resolution: {integrity: sha512-fNAjWnA/nZ2jz31kxR/AqRaUT8ewHBw/WuBIosK0moMy1C9e5ValbDfFdIxJzVOOYaYkV/b2F1S4H/aHiqfVQg==} + engines: {node: '>=22.0.0'} + + '@azure/core-auth@1.11.0': + resolution: {integrity: sha512-IUZydyTUkDnYdstOW9pFOOUQlBjAepK5teihDE3x6yxsPJs/hsAaaYpeGxdxrgtOiJbBKSjKW7MDk7AEhb4LRg==} + engines: {node: '>=22.0.0'} + + '@azure/core-client@1.11.0': + resolution: {integrity: sha512-JjQWO6akOck45PH/XBrxzsQGAiKrfFl4m5iggJ0ItMIz5omRufOXWpqCPpdjKN3vKDzlSUvFjaMb7Zwf0gvAdA==} + engines: {node: '>=22.0.0'} + + '@azure/core-http-compat@2.5.0': + resolution: {integrity: sha512-BoSmXPx2er1Ai+wKlDvj29jIQespCNBwEmKyZVHO2kEFsWbGjAjwMCGzug3DJM5/QYIV3vej0S1zcU5bq9fa8w==} + engines: {node: '>=22.0.0'} + peerDependencies: + '@azure/core-client': ^1.10.0 + '@azure/core-rest-pipeline': ^1.22.0 + + '@azure/core-paging@1.7.0': + resolution: {integrity: sha512-7GEAoIsaoBr6KELNRb8nypowCqvk8dnCHFCYg4XD4lOQGY2GqjQg5IhkRjyBFRO18CGSMq05PaNqSOE9GQro3g==} + engines: {node: '>=22.0.0'} + + '@azure/core-rest-pipeline@1.25.0': + resolution: {integrity: sha512-bMs8ekJLjX8wPV+9IPBges1SLPyuDtE9g5gLDWOpxzKcoOFQnpLGkbcT1tdw3FaAmDS1gnPmMmJ6y/T5B96kIA==} + engines: {node: '>=22.0.0'} + + '@azure/core-tracing@1.4.0': + resolution: {integrity: sha512-eGwxD0AtncrxeBM4tG8R55Pc3rdX1hNW2WibJAgYpCVA6E93mvvVH+LcssoVjOBrSKWS55yEIHsk0X8ctHmfOQ==} + engines: {node: '>=22.0.0'} + + '@azure/core-util@1.14.0': + resolution: {integrity: sha512-9n2pWK61veAuN0V20t9lOuoV4CFMdyAZ1ygZzvBGk/pBBJRib/PjL9PLXa/aI2CcPpyHfqVsxxqLCYl6uZlfDw==} + engines: {node: '>=22.0.0'} + + '@azure/identity@4.13.1': + resolution: {integrity: sha512-5C/2WD5Vb1lHnZS16dNQRPMjN6oV/Upba+C9nBIs15PmOi6A3ZGs4Lr2u60zw4S04gi+u3cEXiqTVP7M4Pz3kw==} + engines: {node: '>=20.0.0'} + + '@azure/logger@1.4.0': + resolution: {integrity: sha512-rbAE25KUfjU/s3XHUdJgceoCP5dEOpMx85J04kF+QMdta73XkuG9JGHHinch+XIoKpBdqljin+KqURpJriSzLA==} + engines: {node: '>=22.0.0'} + + '@azure/msal-browser@5.17.1': + resolution: {integrity: sha512-zBhRGzABKSI7hfWh5EaZmril5ybZ7imBN1qEZl5sDTaelr+l8SnPjZO50Q4dnKnm347YPIlBMSnXKZyh3Yu5DQ==} + engines: {node: '>=0.8.0'} + + '@azure/msal-common@16.11.2': + resolution: {integrity: sha512-yDhtBOGDCdK9ipQ9g3+wmlMEPnZx2pXaDicDd9jYyR1L+7lEbvEohTDmF5qejZDutZY3m9pWPxeYxzNC701A2w==} + engines: {node: '>=0.8.0'} + + '@azure/msal-node@5.4.1': + resolution: {integrity: sha512-yqgoyOIMCH7TNaSLMBTP+4LUlbMMf1zgC8nzOFG95lmW82CmsAEtUT0J93e4BdqDcnX5qle/9X+yb7A8Mw9M0g==} + engines: {node: '>=20'} + + '@azure/search-documents@12.2.0': + resolution: {integrity: sha512-4+Qw+qaGqnkdUCq/vEFzk/bkROogTvdbPb1fmI8poxNfDDN1q2WHxBmhI7CYwesrBj1yXC4i5E0aISBxZqZi0g==} + engines: {node: '>=20.0.0'} + + '@babel/code-frame@7.29.7': + resolution: {integrity: sha512-Aup7aUOfpbAUg2ROOJN6Iw5f9DMBlzu0mIkm/malLQFN/YQgO48wCj0Kxa3sEHJvPVFg7siR+qRInwXd2qhQKw==} + engines: {node: '>=6.9.0'} + + '@babel/helper-validator-identifier@7.29.7': + resolution: {integrity: sha512-qehxGkRj55h/ff8EMaJ+cYhyaKlHIxqYDn682wQD7RNp9UujOQsHog2uS0r2vzr4pW+sXf90NeeayjcNaX3fFg==} + engines: {node: '>=6.9.0'} + '@biomejs/biome@2.4.15': resolution: {integrity: sha512-j5VH3a/h/HXTKBM50MDMxRCzkeLv9S2XJcW2WgnZT1+xyisi+0bISrXR82gCX+8S9lvK0skEvHJRN+3Ktr2hlw==} engines: {node: '>=14.21.3'} @@ -125,6 +198,12 @@ packages: cpu: [x64] os: [win32] + '@cfworker/json-schema@4.1.1': + resolution: {integrity: sha512-gAmrUZSGtKc3AiBL71iNWxDsyUC5uMaKKGdvzYsBoTW/xi42JQHl7eKV2OYzCUqvc+D2RCcf7EXY2iCyFIk6og==} + + '@cloudflare/workers-types@4.20260702.1': + resolution: {integrity: sha512-mOhf5TUEB1m2vPrxtqoIGfz0fUC9xyxRDx5gWHy5s+OCo6dcV+g7wI1R7gYCMFohhqF/2y2xeKVwMwCJjfn/WA==} + '@esbuild/aix-ppc64@0.27.7': resolution: {integrity: sha512-EKX3Qwmhz1eMdEJokhALr0YiD0lhQNwDqkPYyPhiSwKrh7/4KRjQc04sZ8db+5DVVnZ1LmbNDI1uAMPEUBnQPg==} engines: {node: '>=18'} @@ -437,12 +516,33 @@ packages: cpu: [x64] os: [win32] + '@google/genai@1.52.0': + resolution: {integrity: sha512-gwSvbpiN/17O9TbsqSsE/OzZcpv5Fo4RQjdngGgogtuB9RsyJ8ZHhX5KjHj1bp5N9snN2eK8LDGXSaWW2hof8Q==} + engines: {node: '>=20.0.0'} + peerDependencies: + '@modelcontextprotocol/sdk': ^1.25.2 + peerDependenciesMeta: + '@modelcontextprotocol/sdk': + optional: true + '@hono/node-server@2.0.1': resolution: {integrity: sha512-jI9yMDyFpqBeSighf/zlXnQG/nl9AyBc6aAgy4XtxJMyt/CNyJpvPfzDD+bCc2zAOmhhqtF6TnmIaY+xV4mIrw==} engines: {node: '>=20'} peerDependencies: hono: 4.12.30 + '@jest/expect-utils@29.7.0': + resolution: {integrity: sha512-GlsNBWiFQFCVi9QVSx7f5AgMeLxe9YCCs5PuP2O2LdjDAA8Jh9eX7lA1Jq/xdXw3Wb3hyvlFNfZIfcRetSzYcA==} + engines: {node: ^14.15.0 || ^16.10.0 || >=18.0.0} + + '@jest/schemas@29.6.3': + resolution: {integrity: sha512-mo5j5X+jIZmJQveBKeS/clAueipV7KgiX1vMgCxam1RNYiqE1w62n0/tJJnHtjW8ZHcQco5gY85jA3mi0L+nSA==} + engines: {node: ^14.15.0 || ^16.10.0 || >=18.0.0} + + '@jest/types@29.6.3': + resolution: {integrity: sha512-u3UPsIilWKOM3F9CXtrG8LEJmNxwoCQC/XVj4IKYXvvpx7QIi/Kg1LI5uDmDpKlac62NUtX7eLjRh+jVZcLOzw==} + engines: {node: ^14.15.0 || ^16.10.0 || >=18.0.0} + '@jridgewell/gen-mapping@0.3.13': resolution: {integrity: sha512-2kkt/7niJ6MgEPxF0bYdQ6etZaA+fQvDcLKckhy1yIQOzaoKjBBjSj63/aLVjYE3qhRt5dvM+uUyfCg6UKCBbA==} @@ -456,6 +556,16 @@ packages: '@jridgewell/trace-mapping@0.3.31': resolution: {integrity: sha512-zzNR+SdQSDJzc8joaeP8QQoCQr8NuYx2dIIytl1QeBEZHJ9uW6hebsrYgbz8hJwUQao3TWCMtmfV8Nu1twOLAw==} + '@langchain/core@1.2.3': + resolution: {integrity: sha512-F+L5SsciykwDl7eDxacnhDTcWe1IF6jetzfkvI5PPfq6ogWHO7xcjU90SGh/3lqbbS0tgun+qF01KIqxawrCsA==} + engines: {node: '>=20'} + + '@mistralai/mistralai@1.15.1': + resolution: {integrity: sha512-fb995eiz3r0KsBGtRjFV+/iLbX+UpfalxpF+YitT3R6ukrPD4PN+FGwwmYcRFhNAzVzDUtTVxQYnjQWEnwV5nw==} + + '@mongodb-js/saslprep@1.4.12': + resolution: {integrity: sha512-QAfAMwNgnYxZ2C6D1HgeP7Gc4i/uvJRim415PCIL9ptRxWMNbWeLBYb2/9R4pGKny/s1FVu2JA2cxCUBUOggrA==} + '@neo4j-labs/agent-memory@0.4.0': resolution: {integrity: sha512-iKUi+STQm0DQqDUDx0cR486RTMCHlPfmmHBqjyN/gaVXSBlaTzYaLp63D5O/qP5tDc4E4iqiMfMITLUabllTaw==} engines: {node: '>=20.0.0'} @@ -484,6 +594,108 @@ packages: resolution: {integrity: sha512-gLyJlPHPZYdAk1JENA9LeHejZe1Ti77/pTeFm/nMXmQH/HFZlcS/O2XJB+L8fkbrNSqhdtlvjBVjxwUYanNH5Q==} engines: {node: '>=8.0.0'} + '@protobufjs/aspromise@1.1.2': + resolution: {integrity: sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ==} + + '@protobufjs/base64@1.1.2': + resolution: {integrity: sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg==} + + '@protobufjs/codegen@2.0.5': + resolution: {integrity: sha512-zgXFLzW3Ap33e6d0Wlj4MGIm6Ce8O89n/apUaGNB/jx+hw+ruWEp7EwGUshdLKVRCxZW12fp9r40E1mQrf/34g==} + + '@protobufjs/eventemitter@1.1.1': + resolution: {integrity: sha512-vW1GmwMZNnL+gMRaovlh9yZX74kc+TTU3FObkkurpMaRtBfLP3ldjS9KQWlwZgraRE0+dheEEoAxdzcJQ8eXZg==} + + '@protobufjs/fetch@1.1.1': + resolution: {integrity: sha512-GpptLrs57adMSuHi3VNj0mAF8dwh36LMaYF6XyJ6JMWlVsc+t42tm1HSEDmOs3A8fC9yyeisgLhsTVQokOZ0zw==} + + '@protobufjs/float@1.0.2': + resolution: {integrity: sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ==} + + '@protobufjs/path@1.1.2': + resolution: {integrity: sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA==} + + '@protobufjs/pool@1.1.0': + resolution: {integrity: sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw==} + + '@protobufjs/utf8@1.1.2': + resolution: {integrity: sha512-b1UQwcEZ4yCnMCD8DAL1VlbvBJE9/IX4FTIp7BG1xYpf29SLazLSrqUkj4w7Y5y7cCVP6E5tcqqcI0xemPkHug==} + + '@qdrant/js-client-rest@1.18.0': + resolution: {integrity: sha512-/0dqX5uV9chC1DnYSnU4gNMrDqse/pt6hHg3Rqqpl5isH7xl1xSNvffjzBoxycDD79luWn7Ho6Rh/61sOs5DNw==} + engines: {node: '>=18.17.0', pnpm: '>=8'} + peerDependencies: + typescript: '>=4.7' + + '@qdrant/openapi-typescript-fetch@1.2.6': + resolution: {integrity: sha512-oQG/FejNpItrxRHoyctYvT3rwGZOnK4jr3JdppO/c78ktDvkWiPXPHNsrDf33K9sZdRb6PR7gi4noIapu5q4HA==} + engines: {node: '>=18.0.0', pnpm: '>=8'} + + '@redis/bloom@1.2.0': + resolution: {integrity: sha512-HG2DFjYKbpNmVXsa0keLHp/3leGJz1mjh09f2RLGGLQZzSHpkmZWuwJbAvo3QcRY8p80m5+ZdXZdYOSBLlp7Cg==} + peerDependencies: + '@redis/client': ^1.0.0 + + '@redis/bloom@5.12.1': + resolution: {integrity: sha512-PUUfv+ms7jgPSBVoo/DN4AkPHj4D5TZSd6SbJX7egzBplkYUcKmHRE8RKia7UtZ8bSQbLguLvxVO+asKtQfZWA==} + engines: {node: '>= 18.19.0'} + peerDependencies: + '@redis/client': ^5.12.1 + + '@redis/client@1.6.1': + resolution: {integrity: sha512-/KCsg3xSlR+nCK8/8ZYSknYxvXHwubJrU82F3Lm1Fp6789VQ0/3RJKfsmRXjqfaTA++23CvC3hqmqe/2GEt6Kw==} + engines: {node: '>=14'} + + '@redis/client@5.12.1': + resolution: {integrity: sha512-7aPGWeqA3uFm43o19umzdl16CEjK/JQGtSXVPevplTaOU3VJA/rseBC1QvYUz9lLDIMBimc4SW/zrW4S89BaCA==} + engines: {node: '>= 18.19.0'} + peerDependencies: + '@node-rs/xxhash': ^1.1.0 + '@opentelemetry/api': '>=1 <2' + peerDependenciesMeta: + '@node-rs/xxhash': + optional: true + '@opentelemetry/api': + optional: true + + '@redis/graph@1.1.1': + resolution: {integrity: sha512-FEMTcTHZozZciLRl6GiiIB4zGm5z5F3F6a6FZCyrfxdKOhFlGkiAqlexWMBzCi4DcRoyiOsuLfW+cjlGWyExOw==} + peerDependencies: + '@redis/client': ^1.0.0 + + '@redis/json@1.0.7': + resolution: {integrity: sha512-6UyXfjVaTBTJtKNG4/9Z8PSpKE6XgSyEb8iwaqDcy+uKrd/DGYHTWkUdnQDyzm727V7p21WUMhsqz5oy65kPcQ==} + peerDependencies: + '@redis/client': ^1.0.0 + + '@redis/json@5.12.1': + resolution: {integrity: sha512-eOze75esLve4vfqDel7aMX08CNaiLLQS2fV8mpRN9NxPe1rVR4vQyYiW/OgtGUysF6QOr9ANhfxABKNOJfXdKg==} + engines: {node: '>= 18.19.0'} + peerDependencies: + '@redis/client': ^5.12.1 + + '@redis/search@1.2.0': + resolution: {integrity: sha512-tYoDBbtqOVigEDMAcTGsRlMycIIjwMCgD8eR2t0NANeQmgK/lvxNAvYyb6bZDD4frHRhIHkJu2TBRvB0ERkOmw==} + peerDependencies: + '@redis/client': ^1.0.0 + + '@redis/search@5.12.1': + resolution: {integrity: sha512-ItlxbxC9cKI6IU1TLWoczwJCRb6TdmkEpWv05UrPawqaAnWGRu3rcIqsc5vN483T2fSociuyV1UkWIL5I4//2w==} + engines: {node: '>= 18.19.0'} + peerDependencies: + '@redis/client': ^5.12.1 + + '@redis/time-series@1.1.0': + resolution: {integrity: sha512-c1Q99M5ljsIuc4YdaCwfUEXsofakb9c8+Zse2qxTadu8TalLXuAESzLvFAvNVbkmSlvlzIQOLpBCmWI9wTOt+g==} + peerDependencies: + '@redis/client': ^1.0.0 + + '@redis/time-series@5.12.1': + resolution: {integrity: sha512-c6JL6E3EcZJuNqKFz+KM+l9l5mpcQiKvTwgA3blt5glWJ8hjDk0yeHN3beE/MpqYIQ8UEX44ItQzgkE/gCBELQ==} + engines: {node: '>= 18.19.0'} + peerDependencies: + '@redis/client': ^5.12.1 + '@rollup/rollup-android-arm-eabi@4.60.2': resolution: {integrity: sha512-dnlp69efPPg6Uaw2dVqzWRfAWRnYVb1XJ8CyyhIbZeaq4CA5/mLeZ1IEt9QqQxmbdvagjLIm2ZL8BxXv5lH4Yw==} cpu: [arm] @@ -627,6 +839,39 @@ packages: '@scure/bip39@2.2.0': resolution: {integrity: sha512-T/Bj/YvYMNkIPq6EENO6/rcs2e7qTNuyoUXf0KBFDmp0ZDu0H2X4Lq6yC3i0c8PcWkov5EbW+yQZZbdMmk154A==} + '@sinclair/typebox@0.27.12': + resolution: {integrity: sha512-hhyNJ+nbR6ZR7pToHvllEFun9TL0sbL+tk/ON75lo+Xas054uez98qRbsuNt7MBCyZKK4+8Yli/OAGZhmfBZ/g==} + + '@standard-schema/spec@1.1.0': + resolution: {integrity: sha512-l2aFy5jALhniG5HgqrD6jXLi/rUWrKvqN/qJx6yoJsgKhblVd+iqqU4RCXavm/jPityDo5TCvKMnpjKnOriy0w==} + + '@supabase/auth-js@2.110.7': + resolution: {integrity: sha512-M5Bpl4hCv6kHcOO/xM06Dyfg1mYLHljMkp1plhzG9IRZPc3czvyMsSN1XpL5+GKisOKM3lSN59zhpcm6sMVXfA==} + engines: {node: '>=22.0.0'} + + '@supabase/functions-js@2.110.7': + resolution: {integrity: sha512-megYmexlYEoR/0qlsr4Snh9wtzAodO7MAri3NMevZrXzNvQRKlvmTcSBoKGLQEPDakgDZMqbMdf9DwoZz6qfoA==} + engines: {node: '>=22.0.0'} + + '@supabase/phoenix@0.4.5': + resolution: {integrity: sha512-aAn9H9ovVyeApKy11OWOrrOGq8DV68yWeH4ud2lN9fzn4aO8Zb5GLL9m1pUg9nLqIcT+ZDfAcsZe0E/nqdv2lw==} + + '@supabase/postgrest-js@2.110.7': + resolution: {integrity: sha512-ban6YV0djhVaqVYezlOARKLIuOBSvLLhyQVZjA2nxPrtswhxHCl1+gI4giFgI9ATQAaMNbUZb4JXiuL5lEA/5g==} + engines: {node: '>=22.0.0'} + + '@supabase/realtime-js@2.110.7': + resolution: {integrity: sha512-AMtZjyFA2gsmjuxopPNS/sRznLQHG0Ht5x+ytTPTOh3vAcOTUlVRLx7gW4/CONNnbb3PKOkE+HmM35HOSbmomQ==} + engines: {node: '>=22.0.0'} + + '@supabase/storage-js@2.110.7': + resolution: {integrity: sha512-2tcDE8cjEDy1uKxKavBpKQod1JdMV1jDXQag48TCa+kycmJOltc0yVabC0BUlhOwAl6WykXU2aOsH3ELMtZrmQ==} + engines: {node: '>=22.0.0'} + + '@supabase/supabase-js@2.110.7': + resolution: {integrity: sha512-AnfO3A230Shy6RMO7cya3Wl1OcXnABJrzH8vP+fY7/RFjhzcchB7DjKkkTIAntlwekD+GkSFzEvt2tC+D4Fp8w==} + engines: {node: '>=22.0.0'} + '@tangle-network/agent-core@0.3.8': resolution: {integrity: sha512-bZfVpdiFjXbcQwSxSQABdtXEmUuapWChCcXrHkP6fAnwqEy9hWEfeZLlMZUy+1XaOfpTaMLUOEAwYhXfN018wQ==} @@ -688,15 +933,58 @@ packages: '@types/estree@1.0.8': resolution: {integrity: sha512-dWHzHa2WqEXI/O1E9OjrocMTKJl2mSrEolh1Iomrv6U+JuNwaHXsXx9bLu5gG7BUWFIN0skIQJQ/L1rIex4X6w==} + '@types/istanbul-lib-coverage@2.0.6': + resolution: {integrity: sha512-2QF/t/auWm0lsy8XtKVPG19v3sSOQlJe/YHZgfjb/KBBHOGSV+J2q/S671rcq9uTBrLAXmZpqJiaQbMT+zNU1w==} + + '@types/istanbul-lib-report@3.0.3': + resolution: {integrity: sha512-NQn7AHQnk/RSLOxrBbGyJM/aVQ+pjj5HCgasFxc0K/KhoATfQ/47AyUl15I2yBUpihjmas+a+VJBOqecrFH+uA==} + + '@types/istanbul-reports@3.0.4': + resolution: {integrity: sha512-pk2B1NWalF9toCRu6gjBzR69syFjP4Od8WRAX+0mmf9lAjCRicLOWc+ZrxZHx/0XRjotgkF9t6iaMJ+aXcOdZQ==} + + '@types/jest@29.5.14': + resolution: {integrity: sha512-ZN+4sdnLUbo8EVvVc2ao0GFW6oVrQRPn4K2lglySj7APvSrgzxHiNNK99us4WDMi57xxA2yggblIAMNhXOotLQ==} + + '@types/node-fetch@2.6.13': + resolution: {integrity: sha512-QGpRVpzSaUs30JBSGPjOg4Uveu384erbHBoT1zeONvyCfwQxIkUshLAOqN/k9EjGviPRmWTTe6aH2qySWKTVSw==} + + '@types/node@18.19.130': + resolution: {integrity: sha512-GRaXQx6jGfL8sKfaIDD6OupbIHBr9jv7Jnaml9tB7l4v068PAOXqfcujMMo5PhbIs6ggR1XODELqahT2R8v0fg==} + '@types/node@25.6.0': resolution: {integrity: sha512-+qIYRKdNYJwY3vRCZMdJbPLJAtGjQBudzZzdzwQYkEPQd+PJGixUL5QfvCLDaULoLv+RhT3LDkwEfKaAkgSmNQ==} + '@types/pg@8.11.0': + resolution: {integrity: sha512-sDAlRiBNthGjNFfvt0k6mtotoVYVQ63pA8R4EMWka7crawSR60waVYR0HAgmPRs/e2YaeJTD/43OoZ3PFw80pw==} + '@types/proper-lockfile@4.1.4': resolution: {integrity: sha512-uo2ABllncSqg9F1D4nugVl9v93RmjxF6LJzQLMLDdPaXCUIDPeOJ21Gbqi43xNKzBi/WQ0Q0dICqufzQbMjipQ==} + '@types/retry@0.12.0': + resolution: {integrity: sha512-wWKOClTTiizcZhXnPY4wikVAwmdYHp8q6DmC+EJUzAMsycb7HB32Kh9RN4+0gExjmPmZSAQjgURXIGATPegAvA==} + '@types/retry@0.12.5': resolution: {integrity: sha512-3xSjTp3v03X/lSQLkczaN9UIEwJMoMCA1+Nb5HfbJEQWogdeQIyVtTvxPXDQjZ5zws8rFQfVfRdz03ARihPJgw==} + '@types/stack-utils@2.0.3': + resolution: {integrity: sha512-9aEbYZ3TbYMznPdcdr3SmIrLXwC/AKZXQeCf9Pgao5CKb8CyHuEX5jzWPTkvregvhRJHcpRO6BFoGW9ycaOkYw==} + + '@types/webidl-conversions@7.0.3': + resolution: {integrity: sha512-CiJJvcRtIgzadHCYXw7dqEnMNRjhGZlYK05Mj9OyktqV8uVT8fD2BFOB7S1uwBE3Kj2Z+4UyPmFw/Ixgw/LAlA==} + + '@types/whatwg-url@13.0.0': + resolution: {integrity: sha512-N8WXpbE6Wgri7KUSvrmQcqrMllKZ9uxkYWMt+mCSGwNc0Hsw9VQTW7ApqI4XNrx6/SaM2QQJCzMPDEXE058s+Q==} + + '@types/yargs-parser@21.0.3': + resolution: {integrity: sha512-I4q9QU9MQv4oEOz4tAHJtNz1cwuLxn2F3xcc2iV5WdqLPpUnj30aUuxt1mAxYTG+oe8CZMV/+6rU4S4gRDzqtQ==} + + '@types/yargs@17.0.35': + resolution: {integrity: sha512-qUHkeCyQFxMXg79wQfTtfndEC+N9ZZg76HJftDJp+qH2tV7Gj4OJi7l+PiWwJ+pWtW8GwSmqsDj/oymhrTWXjg==} + + '@typespec/ts-http-runtime@0.3.7': + resolution: {integrity: sha512-JVUD8X2tfDMWjcjLs4yVxxVrS8yR5vnh386GAXT9Qj79nBxxXSaHFQZg5FweLmT8HlPQ3kii6noUB+Z9RN7DvQ==} + engines: {node: '>=22.0.0'} + '@vitest/expect@3.2.4': resolution: {integrity: sha512-Io0yyORnB6sikFlt8QW5K7slY4OjqNX9jmJQ02QDda8lyM6B5oNgVWoSoKPac8/kgnCUzuHQKrSLtu/uOqqrig==} @@ -737,18 +1025,99 @@ packages: zod: optional: true + abort-controller@3.0.0: + resolution: {integrity: sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==} + engines: {node: '>=6.5'} + acorn@8.16.0: resolution: {integrity: sha512-UVJyE9MttOsBQIDKw1skb9nAwQuR5wuGD3+82K6JgJlm/Y+KI92oNsMNGZCYdDsVtRHSak0pcV5Dno5+4jh9sw==} engines: {node: '>=0.4.0'} hasBin: true + afinn-165-financialmarketnews@3.0.0: + resolution: {integrity: sha512-0g9A1S3ZomFIGDTzZ0t6xmv4AuokBvBmpes8htiyHpH7N4xDmvSQL6UxL/Zcs2ypRb3VwgCscaD8Q3zEawKYhw==} + + afinn-165@2.0.2: + resolution: {integrity: sha512-mJ/RLUfpXfQA6bzugv+bBsc/QYkVrKaLYeS8fWBpKbTCsonv4iuV9ET0fgReEunm9vKLkaNgnekuSNlTC3WQ1Q==} + + agent-base@6.0.2: + resolution: {integrity: sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==} + engines: {node: '>= 6.0.0'} + + agent-base@7.1.4: + resolution: {integrity: sha512-MnA+YT8fwfJPgBx3m60MNqakm30XOkyIoH1y6huTQvC0PwZG7ki8NacLBcrPbNoo8vEZy7Jpuk7+jMO+CUovTQ==} + engines: {node: '>= 14'} + + agentkeepalive@4.6.0: + resolution: {integrity: sha512-kja8j7PjmncONqaTsB8fQ+wE2mSU2DJ9D4XKoJ5PFWIdRMa6SLSN1ff4mOr4jCbfRSsxR4keIiySJU0N9T5hIQ==} + engines: {node: '>= 8.0.0'} + + ansi-styles@4.3.0: + resolution: {integrity: sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==} + engines: {node: '>=8'} + + ansi-styles@5.2.0: + resolution: {integrity: sha512-Cxwpt2SfTzTtXcfOlzGEee8O+c+MmUgGrNiBcXnuWxuFJHe6a5Hz7qwhwe5OgaSYI0IJvkLqWX1ASG+cJOkEiA==} + engines: {node: '>=10'} + any-promise@1.3.0: resolution: {integrity: sha512-7UvmKalWRt1wgjL1RrGxoSJW/0QZFIegpeGvZG9kjp8vrRu55XTHbwnqq2GpXm9uLbcuhxm3IqX9OB4MZR1b2A==} + apparatus@0.0.10: + resolution: {integrity: sha512-KLy/ugo33KZA7nugtQ7O0E1c8kQ52N3IvD/XgIh4w/Nr28ypfkwDfA67F1ev4N1m5D+BOk1+b2dEJDfpj/VvZg==} + engines: {node: '>=0.2.6'} + assertion-error@2.0.1: resolution: {integrity: sha512-Izi8RQcffqCeNVgFigKli1ssklIbpHnCYc6AknXGYoB6grJqyeby7jv12JUQgmTAnIDnbck1uxksT4dzN3PWBA==} engines: {node: '>=12'} + asynckit@0.4.0: + resolution: {integrity: sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==} + + axios@1.18.1: + resolution: {integrity: sha512-3nTvFlvpn9Zu/RkHUqtc7/+al4UpRW5az71ap5zccp6e8RAYEzhMTecX8Dz1wWDYrPpUoB1HAQEGEAEvUr7S9g==} + + base-64@0.1.0: + resolution: {integrity: sha512-Y5gU45svrR5tI2Vt/X9GPd3L0HNIKzGu202EjxrXMpuc2V2CiKgemAbUUsqYmZJvPtCXoUKjNZwBJzsNScUbXA==} + + base64-js@1.5.1: + resolution: {integrity: sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==} + + better-sqlite3@12.11.1: + resolution: {integrity: sha512-dq9AtApgg5PGFtBzPFSBl3HZQjHok5gaQCM6zh2Yk0aSmDCs1CbnVI8/HgASQkNKsWFpseIO9beg5xxpYhbIfA==} + engines: {node: 20.x || 22.x || 23.x || 24.x || 25.x || 26.x} + + bignumber.js@9.3.1: + resolution: {integrity: sha512-Ko0uX15oIUS7wJ3Rb30Fs6SkVbLmPBAKdlm7q9+ak9bbIeFf0MwuBsQV6z7+X768/cHsfg+WlysDWJcmthjsjQ==} + + bindings@1.5.0: + resolution: {integrity: sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ==} + + bl@4.1.0: + resolution: {integrity: sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==} + + braces@3.0.3: + resolution: {integrity: sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==} + engines: {node: '>=8'} + + bson@7.3.1: + resolution: {integrity: sha512-h/C0qe6857pQhcSJHLfsR1uYGj98Ge3wKAD3Ed9KqH3wcVh+BM4Jq4xISD7vs9OPuT07n+q3QQVjslJ286j6ag==} + engines: {node: '>=20.19.0'} + + buffer-equal-constant-time@1.0.1: + resolution: {integrity: sha512-zRpUiDwd/xk6ADqPMATG8vc9VPrkck7T07OIx0gnjmJAnHnTVXNQG3vfvWNuiZIkwu9KrKdA1iJKfsfTVxE6NA==} + + buffer-writer@2.0.0: + resolution: {integrity: sha512-a7ZpuTZU1TRtnwyCNW3I5dc0wWNC3VR9S++Ewyk2HHZdrO3CQJqSpd+95Us590V6AL7JqUAH2IwZ/398PmNFgw==} + engines: {node: '>=4'} + + buffer@5.7.1: + resolution: {integrity: sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==} + + bundle-name@4.1.0: + resolution: {integrity: sha512-tjwM5exMg6BGRI+kNmTntNsvdZS1X8BFYS6tnJ2hdH0kVxM6/eVZ2xy+FqStSWvYmtfFMDLIxurorHwDKfDz5Q==} + engines: {node: '>=18'} + bundle-require@5.1.0: resolution: {integrity: sha512-3WrrOuZiyaaZPWiEt4G3+IffISVC9HYlWueJEBWED4ZH4aIAC2PnkdnuRrR94M+w6yGWn4AglWtJtBI8YqvgoA==} engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} @@ -759,10 +1128,21 @@ packages: resolution: {integrity: sha512-b6Ilus+c3RrdDk+JhLKUAQfzzgLEPy6wcXqS7f/xe1EETvsDP6GORG7SFuOs6cID5YkqchW/LXZbX5bc8j7ZcQ==} engines: {node: '>=8'} + call-bind-apply-helpers@1.0.2: + resolution: {integrity: sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==} + engines: {node: '>= 0.4'} + chai@5.3.3: resolution: {integrity: sha512-4zNhdJD/iOjSH0A05ea+Ke6MU5mmpQcbQsSOkgdaUMJ9zTlDTD/GYlwohmIE2u0gaxHYiVHEn1Fw9mZ/ktJWgw==} engines: {node: '>=18'} + chalk@4.1.2: + resolution: {integrity: sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==} + engines: {node: '>=10'} + + charenc@0.0.2: + resolution: {integrity: sha512-yrLQ/yVUFXkzg7EDQsPieE/53+0RlaWTs+wBrvW36cyilJ2SaDWfl4Yj7MtLTXleV9uEKefbAGUPv2/iWSooRA==} + check-error@2.1.3: resolution: {integrity: sha512-PAJdDJusoxnwm1VwW07VWwUN1sl7smmC3OKggvndJFadxxDRyFJBX/ggnu/KE4kQAB7a3Dp8f/YXC1FlUprWmA==} engines: {node: '>= 16'} @@ -771,6 +1151,31 @@ packages: resolution: {integrity: sha512-Qgzu8kfBvo+cA4962jnP1KkS6Dop5NS6g7R5LFYJr4b8Ub94PPQXUksCw9PvXoeXPRRddRNC5C1JQUR2SMGtnA==} engines: {node: '>= 14.16.0'} + chownr@1.1.4: + resolution: {integrity: sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==} + + ci-info@3.9.0: + resolution: {integrity: sha512-NIxF55hv4nSqQswkAeiOi1r83xy8JldOFDTWiug55KBu9Jnblncd2U6ViHmYgHf01TPZS77NJBhBMKdWj9HQMQ==} + engines: {node: '>=8'} + + cloudflare@4.5.0: + resolution: {integrity: sha512-fPcbPKx4zF45jBvQ0z7PCdgejVAPBBCZxwqk1k7krQNfpM07Cfj97/Q6wBzvYqlWXx/zt1S9+m8vnfCe06umbQ==} + + cluster-key-slot@1.1.2: + resolution: {integrity: sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA==} + engines: {node: '>=0.10.0'} + + color-convert@2.0.1: + resolution: {integrity: sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==} + engines: {node: '>=7.0.0'} + + color-name@1.1.4: + resolution: {integrity: sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==} + + combined-stream@1.0.8: + resolution: {integrity: sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==} + engines: {node: '>= 0.8'} + commander@14.0.3: resolution: {integrity: sha512-H+y0Jo/T1RZ9qPP4Eh1pkcQcLRglraJaSLoyOtHxu6AapkjWVCy2Sit1QQ4x3Dng8qDlSsZEet7g5Pq06MvTgw==} engines: {node: '>=20'} @@ -779,6 +1184,10 @@ packages: resolution: {integrity: sha512-NOKm8xhkzAjzFx8B2v5OAHT+u5pRQc2UCa2Vq9jYL/31o2wi9mxBA7LIFs3sV5VSC49z6pEhfbMULvShKj26WA==} engines: {node: '>= 6'} + compromise@14.16.0: + resolution: {integrity: sha512-4DFYl/Hl7sW4XWUDfx9S5vxqyYKpZDwwqrpXsQv5acdbVP+joKceIcIaLb0lhVWUpDBV0OnExk/o/dnYUwXnhQ==} + engines: {node: '>=12.0.0'} + confbox@0.1.8: resolution: {integrity: sha512-RMtmw0iFkeR4YV+fUOSucriAQNb9g8zFR52MWCtl+cCZOFRNL6zeB395vPzFhEjjn4fMxXudmELnl/KF/WrK6w==} @@ -786,6 +1195,13 @@ packages: resolution: {integrity: sha512-5IKcdX0nnYavi6G7TtOhwkYzyjfJlatbjMjuLSfE2kYT5pMDOilZ4OvMhi637CcDICTmz3wARPoyhqyX1Y+XvA==} engines: {node: ^14.18.0 || >=16.10.0} + crypt@0.0.2: + resolution: {integrity: sha512-mCxBlsHFYh9C+HVpiEacem8FEBnMXgU9gy4zmNC+SXAZNB/1idgp/aulFJ4FgCi7GPEVbfyng092GqL2k2rmow==} + + data-uri-to-buffer@4.0.1: + resolution: {integrity: sha512-0R9ikRb668HB7QDxT1vkpuUBtqc53YyAwMwGeUFKRojY/NWKvdZ+9UYtRfGmhqNbRkTSVpMbmyhXipFFv2cb/A==} + engines: {node: '>= 12'} + debug@4.4.3: resolution: {integrity: sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==} engines: {node: '>=6.0'} @@ -795,13 +1211,82 @@ packages: supports-color: optional: true + decompress-response@6.0.0: + resolution: {integrity: sha512-aW35yZM6Bb/4oJlZncMH2LCoZtJXTRxES17vE3hoRiowU2kWHaJKFkSBDnDR+cm9J+9QhXmREyIfv0pji9ejCQ==} + engines: {node: '>=10'} + deep-eql@5.0.2: resolution: {integrity: sha512-h5k/5U50IJJFpzfL6nO9jaaumfjO/f2NjK/oYB2Djzm4p9L+3T9qWpZqZ2hAbLPuuYq9wrU08WQyBTL5GbPk5Q==} engines: {node: '>=6'} + deep-extend@0.6.0: + resolution: {integrity: sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==} + engines: {node: '>=4.0.0'} + + default-browser-id@5.0.1: + resolution: {integrity: sha512-x1VCxdX4t+8wVfd1so/9w+vQ4vx7lKd2Qp5tDRutErwmR85OgmfX7RlLRMWafRMY7hbEiXIbudNrjOAPa/hL8Q==} + engines: {node: '>=18'} + + default-browser@5.5.0: + resolution: {integrity: sha512-H9LMLr5zwIbSxrmvikGuI/5KGhZ8E2zH3stkMgM5LpOWDutGM2JZaj460Udnf1a+946zc7YBgrqEWwbk7zHvGw==} + engines: {node: '>=18'} + + define-lazy-prop@3.0.0: + resolution: {integrity: sha512-N+MeXYoqr3pOgn8xfyRPREN7gHakLYjhsHhWGT3fWAiL4IkAt0iDw14QiiEm2bE30c5XX5q0FtAA3CK5f9/BUg==} + engines: {node: '>=12'} + + delayed-stream@1.0.0: + resolution: {integrity: sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==} + engines: {node: '>=0.4.0'} + + detect-libc@2.1.2: + resolution: {integrity: sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==} + engines: {node: '>=8'} + + diff-sequences@29.6.3: + resolution: {integrity: sha512-EjePK1srD3P08o2j4f0ExnylqRs5B9tJjcp9t1krH2qRi8CCdsYfwe9JgSLurFBWwq4uOlipzfk5fHNvwFKr8Q==} + engines: {node: ^14.15.0 || ^16.10.0 || >=18.0.0} + + digest-fetch@1.3.0: + resolution: {integrity: sha512-CGJuv6iKNM7QyZlM2T3sPAdZWd/p9zQiRNS9G+9COUCwzWFTs0Xp8NF5iePx7wtvhDykReiRRrSeNb4oMmB8lA==} + + dotenv@17.4.2: + resolution: {integrity: sha512-nI4U3TottKAcAD9LLud4Cb7b2QztQMUEfHbvhTH09bqXTxnSie8WnjPALV/WMCrJZ6UV/qHJ6L03OqO3LcdYZw==} + engines: {node: '>=12'} + + dunder-proto@1.0.1: + resolution: {integrity: sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==} + engines: {node: '>= 0.4'} + + ecdsa-sig-formatter@1.0.11: + resolution: {integrity: sha512-nagl3RYrbNv6kQkeJIpt6NJZy8twLB/2vtz6yN9Z4vRKHN4/QZJIEbqohALSgwKdnksuY3k5Addp5lg8sVoVcQ==} + + efrt@2.7.0: + resolution: {integrity: sha512-/RInbCy1d4P6Zdfa+TMVsf/ufZVotat5hCw3QXmWtjU+3pFEOvOQ7ibo3aIxyCJw2leIeAMjmPj+1SLJiCpdrQ==} + engines: {node: '>=12.0.0'} + + end-of-stream@1.4.5: + resolution: {integrity: sha512-ooEGc6HP26xXq/N+GCGOT0JKCLDGrq2bQUZrQ7gyrJiZANJ/8YDTxTpQBXGMn+WbIQXNVpyWymm7KYVICQnyOg==} + + es-define-property@1.0.1: + resolution: {integrity: sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==} + engines: {node: '>= 0.4'} + + es-errors@1.3.0: + resolution: {integrity: sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==} + engines: {node: '>= 0.4'} + es-module-lexer@1.7.0: resolution: {integrity: sha512-jEQoCwk8hyb2AZziIOLhDqpm5+2ww5uIE6lkO/6jcOCusfk6LhMHpXXfBLXTZ7Ydyt0j4VoUQv6uGNYbdW+kBA==} + es-object-atoms@1.1.2: + resolution: {integrity: sha512-HWcBoN6NileqtSydK2FqHbS/LoDd2pqrnQHLyJzBj4kOp/ky2MWMN694xOfkK8/SnUsW2DH7EfyVlydKCsm1Zw==} + engines: {node: '>= 0.4'} + + es-set-tostringtag@2.1.0: + resolution: {integrity: sha512-j6vWzfrGVfyXxge+O0x5sh6cvxAog0a/4Rdd2K36zCMV5eJ+/+tOAngRO8cODMNWbVRdVlmGZQL2YS3yR8bIUA==} + engines: {node: '>= 0.4'} + esbuild@0.27.7: resolution: {integrity: sha512-IxpibTjyVnmrIQo5aqNpCgoACA/dTKLTlhMHihVHhdkxKyPO1uBBthumT0rdHmcsk9uMonIWS0m4FljWzILh3w==} engines: {node: '>=18'} @@ -812,16 +1297,42 @@ packages: engines: {node: '>=18'} hasBin: true + escape-string-regexp@2.0.0: + resolution: {integrity: sha512-UpzcLCXolUWcNu5HtVMHYdXJjArjsF9C0aNnquZYY4uW/Vu0miy5YoWvbV345HauVvcAUnpRuhMMcqTcGOY2+w==} + engines: {node: '>=8'} + estree-walker@3.0.3: resolution: {integrity: sha512-7RUKfXgSMMkzt6ZuXmqapOurLGPPfgj6l9uRZ7lRGolvk0y2yocc35LdcxKC5PQZdn2DMqioAQ2NoWcrTKmm6g==} + event-target-shim@5.0.1: + resolution: {integrity: sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==} + engines: {node: '>=6'} + + eventemitter3@4.0.7: + resolution: {integrity: sha512-8guHBZCwKnFhYdHr2ysuRWErTwhoN2X8XELRlrRwpmfeY2jjuUN4taQMsULKUVo1K4DvZl+0pgfyoysHxvmvEw==} + eventemitter3@5.0.1: resolution: {integrity: sha512-GWkBvjiSZK87ELrYOSESUYeVIc9mvLLf/nXalMOS5dYrgZq9o5OVkbZAVM06CVxYsCwH9BDZFPlQTlPA1j4ahA==} + events@3.3.0: + resolution: {integrity: sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q==} + engines: {node: '>=0.8.x'} + + expand-template@2.0.3: + resolution: {integrity: sha512-XYfuKMvj4O35f/pOXLObndIRvyQ+/+6AhODh+OKWj9S9498pHHn/IMszH+gt0fBCRWMNfk1ZSp5x3AifmnI2vg==} + engines: {node: '>=6'} + expect-type@1.3.0: resolution: {integrity: sha512-knvyeauYhqjOYvQ66MznSMs83wmHrCycNEN6Ao+2AeYEfxUIkuiVxdEa1qlGEPK+We3n0THiDciYSsCcgW/DoA==} engines: {node: '>=12.0.0'} + expect@29.7.0: + resolution: {integrity: sha512-2Zks0hf1VLFYI1kbh0I5jP3KHHyCHpkfyHBzsSXRFgl/Bg9mWYfMW8oD+PdMPlEwy5HNsR9JutYy6pMeOh61nw==} + engines: {node: ^14.15.0 || ^16.10.0 || >=18.0.0} + + extend@3.0.2: + resolution: {integrity: sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==} + fdir@6.5.0: resolution: {integrity: sha512-tIbYtZbucOs0BRGqPJkshJUYdL+SDH7dVM8gjy+ERp3WAUjLEFJE+02kanyHtwjWOnwrKYBiwAmM0p4kLJAnXg==} engines: {node: '>=12.0.0'} @@ -831,33 +1342,244 @@ packages: picomatch: optional: true + fetch-blob@3.2.0: + resolution: {integrity: sha512-7yAQpD2UMJzLi1Dqv7qFYnPbaPx7ZfFK6PiIxQ4PfkGPyNyl2Ugx+a/umUonmKqjhM4DnfbMvdX6otXq83soQQ==} + engines: {node: ^12.20 || >= 14.13} + + file-uri-to-path@1.0.0: + resolution: {integrity: sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw==} + + fill-range@7.1.1: + resolution: {integrity: sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==} + engines: {node: '>=8'} + fix-dts-default-cjs-exports@1.0.1: resolution: {integrity: sha512-pVIECanWFC61Hzl2+oOCtoJ3F17kglZC/6N94eRWycFgBH35hHx0Li604ZIzhseh97mf2p0cv7vVrOZGoqhlEg==} + follow-redirects@1.16.0: + resolution: {integrity: sha512-y5rN/uOsadFT/JfYwhxRS5R7Qce+g3zG97+JrtFZlC9klX/W5hD7iiLzScI4nZqUS7DNUdhPgw4xI8W2LuXlUw==} + engines: {node: '>=4.0'} + peerDependencies: + debug: '*' + peerDependenciesMeta: + debug: + optional: true + + form-data-encoder@1.7.2: + resolution: {integrity: sha512-qfqtYan3rxrnCk1VYaA4H+Ms9xdpPqvLZa6xmMgFvhO32x7/3J/ExcTd6qpxM0vH2GdMI+poehyBZvqfMTto8A==} + + form-data@4.0.6: + resolution: {integrity: sha512-vKatAh4SlVfgbv+YtmhiRjhEMJsYpsG1Y2rMQtR+SVSbytsSD1YGzDIcrAJmdFec88u/+VoGmxnl+80gL1tRCQ==} + engines: {node: '>= 6'} + + formdata-node@4.4.1: + resolution: {integrity: sha512-0iirZp3uVDjVGt9p49aTaqjk84TrglENEDuqfdlZQ1roC9CWlPk6Avf8EEnZNcAqPonwkG35x4n3ww/1THYAeQ==} + engines: {node: '>= 12.20'} + + formdata-polyfill@4.0.10: + resolution: {integrity: sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g==} + engines: {node: '>=12.20.0'} + + fs-constants@1.0.0: + resolution: {integrity: sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==} + fsevents@2.3.3: resolution: {integrity: sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==} engines: {node: ^8.16.0 || ^10.6.0 || >=11.0.0} os: [darwin] + function-bind@1.1.2: + resolution: {integrity: sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==} + + gaxios@7.2.0: + resolution: {integrity: sha512-CUVb4wcYe+771XevyH6HtGmXFAGGKkIC3kswAP8Z1JCe0j80JMaTPZH930DWFrvo0atjh18Arc0pEyUCWa5bfg==} + engines: {node: '>=18'} + + gcp-metadata@8.1.2: + resolution: {integrity: sha512-zV/5HKTfCeKWnxG0Dmrw51hEWFGfcF2xiXqcA3+J90WDuP0SvoiSO5ORvcBsifmx/FoIjgQN3oNOGaQ5PhLFkg==} + engines: {node: '>=18'} + + generic-pool@3.9.0: + resolution: {integrity: sha512-hymDOu5B53XvN4QT9dBmZxPX4CWhBPPLguTZ9MMFeFa/Kg0xWVfylOVNlJji/E7yTZWFd/q9GO5TxDLq156D7g==} + engines: {node: '>= 4'} + + get-intrinsic@1.3.0: + resolution: {integrity: sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==} + engines: {node: '>= 0.4'} + + get-proto@1.0.1: + resolution: {integrity: sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==} + engines: {node: '>= 0.4'} + + github-from-package@0.0.0: + resolution: {integrity: sha512-SyHy3T1v2NUXn29OsWdxmK6RwHD+vkj3v8en8AOBZ1wBQ/hCAQ5bAQTD02kW4W9tUp/3Qh6J8r9EvntiyCmOOw==} + + google-auth-library@10.9.0: + resolution: {integrity: sha512-xtvUqvINPhTaBm7nXqlYPcrMHJPm1lCNdSovxnKKhTm+4JsvQ+KGVYJViLoH9Yxu8w+T0Qv5HubzYT9BLrppJg==} + engines: {node: '>=18'} + + google-logging-utils@1.1.3: + resolution: {integrity: sha512-eAmLkjDjAFCVXg7A1unxHsLf961m6y17QFqXqAXGj/gVkKFrEICfStRfwUlGNfeCEjNRa32JEWOUTlYXPyyKvA==} + engines: {node: '>=14'} + + gopd@1.2.0: + resolution: {integrity: sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==} + engines: {node: '>= 0.4'} + graceful-fs@4.2.11: resolution: {integrity: sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==} + grad-school@0.0.5: + resolution: {integrity: sha512-rXunEHF9M9EkMydTBux7+IryYXEZinRk6g8OBOGDBzo/qWJjhTxy86i5q7lQYpCLHN8Sqv1XX3OIOc7ka2gtvQ==} + engines: {node: '>=8.0.0'} + + groq-sdk@0.3.0: + resolution: {integrity: sha512-Cdgjh4YoSBE2X4S9sxPGXaAy1dlN4bRtAaDZ3cnq+XsxhhN9WSBeHF64l7LWwuD5ntmw7YC5Vf4Ff1oHCg1LOg==} + + has-flag@4.0.0: + resolution: {integrity: sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==} + engines: {node: '>=8'} + + has-symbols@1.1.0: + resolution: {integrity: sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==} + engines: {node: '>= 0.4'} + + has-tostringtag@1.0.2: + resolution: {integrity: sha512-NqADB8VjPFLM2V0VvHUewwwsw0ZWBaIdgo+ieHtK3hasLz4qeCRjYcqfB6AQrBggRKppKF8L52/VqdVsO47Dlw==} + engines: {node: '>= 0.4'} + + hasown@2.0.4: + resolution: {integrity: sha512-T2UbfbBEF32wiepXIsMlTW9+dDYC6wMh/t/vYA4tuOMKqWz/n3vr1NFSxQiyP+zk2mXsoMA/i/7qV6LKut1t1A==} + engines: {node: '>= 0.4'} + hono@4.12.30: resolution: {integrity: sha512-emn+JoJjrN9YTpRDS5it/UI2SO9BAE37T6I3d963RxcZ81G9A4pr2SZTEiiaiKbzx+NKRg5BZ89fCL7gCJCUog==} engines: {node: '>=16.9.0'} + http-proxy-agent@7.0.2: + resolution: {integrity: sha512-T1gkAiYYDWYx3V5Bmyu7HcfcvL7mUrTWiM6yOfa3PIphViJ/gFPbvidQ+veqSOHci/PxBcDabeUNCzpOODJZig==} + engines: {node: '>= 14'} + + https-proxy-agent@5.0.1: + resolution: {integrity: sha512-dFcAjpTQFgoLMzC2VwU+C/CbS7uRL0lWmxDITmqm7C+7F0Odmj6s9l6alZc6AELXhrnggM2CeWSXHGOdX2YtwA==} + engines: {node: '>= 6'} + + https-proxy-agent@7.0.6: + resolution: {integrity: sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw==} + engines: {node: '>= 14'} + + humanize-ms@1.2.1: + resolution: {integrity: sha512-Fl70vYtsAFb/C06PTS9dZBo7ihau+Tu/DNCk/OyHhea07S+aeMWpFFkUaXRa8fI+ScZbEI8dfSxwY7gxZ9SAVQ==} + + iceberg-js@0.8.1: + resolution: {integrity: sha512-1dhVQZXhcHje7798IVM+xoo/1ZdVfzOMIc8/rgVSijRK38EDqOJoGula9N/8ZI5RD8QTxNQtK/Gozpr+qUqRRA==} + engines: {node: '>=20.0.0'} + + ieee754@1.2.1: + resolution: {integrity: sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==} + + inherits@2.0.4: + resolution: {integrity: sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==} + + ini@1.3.8: + resolution: {integrity: sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==} + + is-buffer@1.1.6: + resolution: {integrity: sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w==} + + is-docker@3.0.0: + resolution: {integrity: sha512-eljcgEDlEns/7AXFosB5K/2nCM4P7FQPkGc/DWLy5rmFEWvZayGrik1d9/QIY5nJ4f9YsVvBkA6kJpHn9rISdQ==} + engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} + hasBin: true + + is-inside-container@1.0.0: + resolution: {integrity: sha512-KIYLCCJghfHZxqjYBE7rEy0OBuTd5xCHS7tHVgvCLkx7StIoaxwNW3hCALgEUjFfeRk+MG/Qxmp/vtETEF3tRA==} + engines: {node: '>=14.16'} + hasBin: true + + is-number@7.0.0: + resolution: {integrity: sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==} + engines: {node: '>=0.12.0'} + + is-wsl@3.1.1: + resolution: {integrity: sha512-e6rvdUCiQCAuumZslxRJWR/Doq4VpPR82kqclvcS0efgt430SlGIk05vdCN58+VrzgtIcfNODjozVielycD4Sw==} + engines: {node: '>=16'} + isows@1.0.7: resolution: {integrity: sha512-I1fSfDCZL5P0v33sVqeTDSpcstAg/N+wF5HS033mogOVIp4B+oHC7oOCsA3axAbBSGTJ8QubbNmnIRN/h8U7hg==} peerDependencies: ws: 8.21.0 + jest-diff@29.7.0: + resolution: {integrity: sha512-LMIgiIrhigmPrs03JHpxUh2yISK3vLFPkAodPeo0+BuF7wA2FoQbkEg1u8gBYBThncu7e1oEDUfIXVuTqLRUjw==} + engines: {node: ^14.15.0 || ^16.10.0 || >=18.0.0} + + jest-get-type@29.6.3: + resolution: {integrity: sha512-zrteXnqYxfQh7l5FHyL38jL39di8H8rHoecLH3JNxH3BwOrBsNeabdap5e0I23lD4HHI8W5VFBZqG4Eaq5LNcw==} + engines: {node: ^14.15.0 || ^16.10.0 || >=18.0.0} + + jest-matcher-utils@29.7.0: + resolution: {integrity: sha512-sBkD+Xi9DtcChsI3L3u0+N0opgPYnCRPtGcQYrgXmR+hmt/fYfWAL0xRXYU8eWOdfuLgBe0YCW3AFtnRLagq/g==} + engines: {node: ^14.15.0 || ^16.10.0 || >=18.0.0} + + jest-message-util@29.7.0: + resolution: {integrity: sha512-GBEV4GRADeP+qtB2+6u61stea8mGcOT4mCtrYISZwfu9/ISHFJ/5zOMXYbpBE9RsS5+Gb63DW4FgmnKJ79Kf6w==} + engines: {node: ^14.15.0 || ^16.10.0 || >=18.0.0} + + jest-util@29.7.0: + resolution: {integrity: sha512-z6EbKajIpqGKU56y5KBUgy1dt1ihhQJgWzUlZHArA/+X2ad7Cb5iF+AK1EWVL/Bo7Rz9uurpqw6SiBCefUbCGA==} + engines: {node: ^14.15.0 || ^16.10.0 || >=18.0.0} + joycon@3.1.1: resolution: {integrity: sha512-34wB/Y7MW7bzjKRjUKTa46I2Z7eV62Rkhva+KkopW7Qvv/OSWBqvkSY7vusOPrNuZcUG3tApvdVgNB8POj3SPw==} engines: {node: '>=10'} + js-tiktoken@1.0.21: + resolution: {integrity: sha512-biOj/6M5qdgx5TKjDnFT1ymSpM5tbd3ylwDtrQvFQSu0Z7bBYko2dF+W/aUkXUPuk6IVpRxk/3Q2sHOzGlS36g==} + + js-tokens@4.0.0: + resolution: {integrity: sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==} + js-tokens@9.0.1: resolution: {integrity: sha512-mxa9E9ITFOt0ban3j6L5MpjwegGz6lBQmM1IJkWeBZGcMxto50+eWdjC/52xDbS2vy0k7vIMK0Fe2wfL9OQSpQ==} + json-bigint@1.0.0: + resolution: {integrity: sha512-SiPv/8VpZuWbvLSMtTDU8hEfrZWg/mH/nV/b4o0CYbSxu1UIQPLdwKOCIyLQX+VIPO5vrLX3i8qtqFyhdPSUSQ==} + + jsonwebtoken@9.0.3: + resolution: {integrity: sha512-MT/xP0CrubFRNLNKvxJ2BYfy53Zkm++5bX9dtuPbqAeQpTVe0MQTFhao8+Cp//EmJp244xt6Drw/GVEGCUj40g==} + engines: {node: '>=12', npm: '>=6'} + + jwa@2.0.1: + resolution: {integrity: sha512-hRF04fqJIP8Abbkq5NKGN0Bbr3JxlQ+qhZufXVr0DvujKy93ZCbXZMHDL4EOtodSbCWxOqR8MS1tXA5hwqCXDg==} + + jws@4.0.1: + resolution: {integrity: sha512-EKI/M/yqPncGUUh44xz0PxSidXFr/+r0pA70+gIYhjv+et7yxM+s29Y+VGDkovRofQem0fs7Uvf4+YmAdyRduA==} + + kareem@3.3.0: + resolution: {integrity: sha512-kpSuLD3/7RenBnjnJdOHXCKC8dTd1JzeOiJhN0necWWci6cC+qX+VuwPnMVgb+a4+KNJSfgqahpnfWaeDXCimw==} + engines: {node: '>=18.0.0'} + + langsmith@0.8.3: + resolution: {integrity: sha512-3y1PRTmY24wiN8NiMb0CcNI2LwgovsKAz+ymVmBSmUx4nXL+GlmPY6Ju7FGeaPV1DMfQs9SjceNZHIfY2XYMKQ==} + peerDependencies: + '@opentelemetry/api': '*' + '@opentelemetry/exporter-trace-otlp-proto': '*' + '@opentelemetry/sdk-trace-base': '*' + openai: '*' + ws: 8.21.0 + peerDependenciesMeta: + '@opentelemetry/api': + optional: true + '@opentelemetry/exporter-trace-otlp-proto': + optional: true + '@opentelemetry/sdk-trace-base': + optional: true + openai: + optional: true + ws: + optional: true + lilconfig@3.1.3: resolution: {integrity: sha512-/vlFKAoH5Cgt3Ie+JLhRbwOsCQePABiU3tJ1egGvyQ+33R/vcwM2Zl2QR/LzjsBeItPt3oSVXapn+m4nQDvpzw==} engines: {node: '>=14'} @@ -869,18 +1591,216 @@ packages: resolution: {integrity: sha512-IXO6OCs9yg8tMKzfPZ1YmheJbZCiEsnBdcB03l0OcfK9prKnJb96siuHCr5Fl37/yo9DnKU+TLpxzTUspw9shg==} engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} - loupe@3.2.1: + lodash.includes@4.3.0: + resolution: {integrity: sha512-W3Bx6mdkRTGtlJISOvVD/lbqjTlPPUDTMnlXZFnVwi9NKJ6tiAk6LVdlhZMm17VZisqhKcgzpO5Wz91PCt5b0w==} + + lodash.isboolean@3.0.3: + resolution: {integrity: sha512-Bz5mupy2SVbPHURB98VAcw+aHh4vRV5IPNhILUCsOzRmsTmSQ17jIuqopAentWoehktxGd9e/hbIXq980/1QJg==} + + lodash.isinteger@4.0.4: + resolution: {integrity: sha512-DBwtEWN2caHQ9/imiNeEA5ys1JoRtRfY3d7V9wkqtbycnAmTvRRmbHKDV4a0EYc678/dia0jrte4tjYwVBaZUA==} + + lodash.isnumber@3.0.3: + resolution: {integrity: sha512-QYqzpfwO3/CWf3XP+Z+tkQsfaLL/EnUlXWVkIk5FUPc4sBdTehEqZONuyRt2P67PXAk+NXmTBcc97zw9t1FQrw==} + + lodash.isplainobject@4.0.6: + resolution: {integrity: sha512-oSXzaWypCMHkPC3NvBEaPHf0KsA5mvPrOPgQWDsbg8n7orZ290M0BmC/jgRZ4vcJ6DTAhjrsSYgdsW/F+MFOBA==} + + lodash.isstring@4.0.1: + resolution: {integrity: sha512-0wJxfxH1wgO3GrbuP+dTTk7op+6L41QCXbGINEmD+ny/G/eCqGzxyCsh7159S+mgDDcoarnBw6PC1PS5+wUGgw==} + + lodash.once@4.1.1: + resolution: {integrity: sha512-Sb487aTOCr9drQVL8pIxOzVhafOjZN9UU54hiN8PU3uAiSV7lx1yYNpbNmex2PK6dSJoNTSJUUswT651yww3Mg==} + + long@5.3.2: + resolution: {integrity: sha512-mNAgZ1GmyNhD7AuqnTG3/VQ26o760+ZYBPKjPvugO8+nLbYfX6TVpJPseBvopbdY+qpZ/lKUnmEc1LeZYS3QAA==} + + loupe@3.2.1: resolution: {integrity: sha512-CdzqowRJCeLU72bHvWqwRBBlLcMEtIvGrlvef74kMnV2AolS9Y8xUv1I0U/MNAWMhBlKIoyuEgoJ0t/bbwHbLQ==} magic-string@0.30.21: resolution: {integrity: sha512-vd2F4YUyEXKGcLHoq+TEyCjxueSeHnFxyyjNp80yg0XV4vUhnDer/lvvlqM/arB5bXQN5K2/3oinyCRyx8T2CQ==} + math-intrinsics@1.1.0: + resolution: {integrity: sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==} + engines: {node: '>= 0.4'} + + md5@2.3.0: + resolution: {integrity: sha512-T1GITYmFaKuO91vxyoQMFETst+O71VUPEU3ze5GNzDm0OWdP8v1ziTaAEPUr/3kLsY3Sftgz242A1SetQiDL7g==} + + mem0ai@3.1.0: + resolution: {integrity: sha512-N/ESxO2wWlQMvWdjNgTK9kuPYVdMYJglSR14BsDfuNMvK9YrigNdZ0nDtuOO4EtJ+Yge6YQnGJ8uuJDbKbJaXg==} + engines: {node: '>=18'} + peerDependencies: + '@anthropic-ai/sdk': ^0.40.1 + '@aws-sdk/client-bedrock-runtime': '>=3.0.0 <3.968.0' + '@aws-sdk/client-neptune-graph': '>=3.0.0 <3.968.0' + '@aws-sdk/client-s3vectors': 3.967.0 + '@azure/identity': ^4.0.0 + '@azure/search-documents': ^12.0.0 + '@cloudflare/workers-types': ^4.20250504.0 + '@databricks/sql': ^1.16.0 + '@elastic/elasticsearch': ^9.0.0 + '@google-cloud/aiplatform': ^6.8.0 + '@google/genai': ^1.40.0 + '@huggingface/transformers': ^3.0.0 || ^4.0.0 + '@langchain/core': ^1.1.47 + '@mistralai/mistralai': ^1.5.2 + '@mochow/mochow-sdk-node': ^2.1.5 + '@opensearch-project/opensearch': ^3.5.1 + '@pinecone-database/pinecone': ^8.0.0 + '@qdrant/js-client-rest': ^1.18.0 + '@supabase/supabase-js': ^2.49.1 + '@turbopuffer/turbopuffer': ^2.0.0 + '@types/jest': 29.5.14 + '@types/pg': 8.11.0 + '@upstash/vector': ^1.2.3 + '@zilliz/milvus2-sdk-node': ^2.4.0 || ^3.0.0 + better-sqlite3: ^12.6.2 + cassandra-driver: 4.8.0 + chromadb: ^3.5.0 + cloudflare: ^4.2.0 + cohere-ai: ^7.17.0 || ^8.0.0 + compromise: ^14.0.0 + fastembed: ^2.1.0 + groq-sdk: 0.3.0 + iovalkey: ^0.3.3 + mongodb: ^7.0.0 + mysql2: ^3.0.0 + natural: ^8.0.1 + ollama: ^0.5.14 + pg: 8.11.3 + redis: ^4.6.13 + weaviate-client: ^3.0.0 + zeroentropy: ^0.1.0-alpha.10 + peerDependenciesMeta: + '@aws-sdk/client-bedrock-runtime': + optional: true + '@aws-sdk/client-neptune-graph': + optional: true + '@aws-sdk/client-s3vectors': + optional: true + '@databricks/sql': + optional: true + '@elastic/elasticsearch': + optional: true + '@google-cloud/aiplatform': + optional: true + '@huggingface/transformers': + optional: true + '@mochow/mochow-sdk-node': + optional: true + '@opensearch-project/opensearch': + optional: true + '@pinecone-database/pinecone': + optional: true + '@turbopuffer/turbopuffer': + optional: true + '@upstash/vector': + optional: true + '@zilliz/milvus2-sdk-node': + optional: true + cassandra-driver: + optional: true + chromadb: + optional: true + cohere-ai: + optional: true + fastembed: + optional: true + iovalkey: + optional: true + mongodb: + optional: true + mysql2: + optional: true + weaviate-client: + optional: true + zeroentropy: + optional: true + + memjs@1.3.2: + resolution: {integrity: sha512-qUEg2g8vxPe+zPn09KidjIStHPtoBO8Cttm8bgJFWWabbsjQ9Av9Ky+6UcvKx6ue0LLb/LEhtcyQpRyKfzeXcg==} + engines: {node: '>=0.10.0'} + + memory-pager@1.5.0: + resolution: {integrity: sha512-ZS4Bp4r/Zoeq6+NLJpP+0Zzm0pR8whtGPf1XExKLJBAczGMnSi3It14OiNCStjQjM6NU1okjQGSxgEZN8eBYKg==} + + micromatch@4.0.8: + resolution: {integrity: sha512-PXwfBhYu0hBCPw8Dn0E+WDYb7af3dSLVWKi3HGv84IdF4TyFoC0ysxFd0Goxw7nSv4T/PzEJQxsYsEiFCKo2BA==} + engines: {node: '>=8.6'} + + mime-db@1.52.0: + resolution: {integrity: sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==} + engines: {node: '>= 0.6'} + + mime-types@2.1.35: + resolution: {integrity: sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==} + engines: {node: '>= 0.6'} + + mimic-response@3.1.0: + resolution: {integrity: sha512-z0yWI+4FDrrweS8Zmt4Ej5HdJmky15+L2e6Wgn3+iK5fWzb6T3fhNFq2+MeTRb064c6Wr4N/wv0DzQTjNzHNGQ==} + engines: {node: '>=10'} + + minimist@1.2.8: + resolution: {integrity: sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==} + + mkdirp-classic@0.5.3: + resolution: {integrity: sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==} + mlly@1.8.2: resolution: {integrity: sha512-d+ObxMQFmbt10sretNDytwt85VrbkhhUA/JBGm1MPaWJ65Cl4wOgLaB1NYvJSZ0Ef03MMEU/0xpPMXUIQ29UfA==} + mongodb-connection-string-url@7.0.2: + resolution: {integrity: sha512-ZoS07RoFqpKYQwAk59qmrx8+jJHNHU30UjlU96QktiGn1ltvDr+vCznLX5DiUBLEpMAHatHNWV1nM/74ul66kA==} + engines: {node: '>=20.19.0'} + + mongodb@7.2.0: + resolution: {integrity: sha512-F/2+BMZtLVhY30ioZp0dAmZ+IRZMBqI+nrv6t5+9/1AIwCa8sMRC3jBf81lpxMhnZgqq8CoUD503Z1oZWq1/sw==} + engines: {node: '>=20.19.0'} + peerDependencies: + '@aws-sdk/credential-providers': ^3.806.0 + '@mongodb-js/zstd': ^7.0.0 + gcp-metadata: ^7.0.1 + kerberos: ^7.0.0 + mongodb-client-encryption: '>=7.0.0 <7.1.0' + snappy: ^7.3.2 + socks: ^2.8.6 + peerDependenciesMeta: + '@aws-sdk/credential-providers': + optional: true + '@mongodb-js/zstd': + optional: true + gcp-metadata: + optional: true + kerberos: + optional: true + mongodb-client-encryption: + optional: true + snappy: + optional: true + socks: + optional: true + + mongoose@9.7.4: + resolution: {integrity: sha512-nuSYGUWWzNd4EAbGYxE469wPTL+kmxb5+91YvCvMkJ08rvNRht/usZUU3LuFuk7rDutF2QWBZHPHuzM8TxXApA==} + engines: {node: '>=20.19.0'} + + mpath@0.9.0: + resolution: {integrity: sha512-ikJRQTk8hw5DEoFVxHG1Gn9T/xcjtdnOKIU1JTmGjZZlg9LST2mBLmcX3/ICIbgJydT2GOc15RnNy5mHmzfSew==} + engines: {node: '>=4.0.0'} + + mquery@6.0.0: + resolution: {integrity: sha512-b2KQNsmgtkscfeDgkYMcWGn9vZI9YoXh802VDEwE6qc50zxBFQ0Oo8ROkawbPAsXCY1/Z1yp0MagqsZStPWJjw==} + engines: {node: '>=20.19.0'} + ms@2.1.3: resolution: {integrity: sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==} + mustache@4.2.0: + resolution: {integrity: sha512-71ippSywq5Yb7/tVYyGbkBggbU8H3u5Rz56fH60jGFgr8uHwxs+aSKeqmluIVzM0m0kB7xQjKS6qPfd0b2ZoqQ==} + hasBin: true + mz@2.7.0: resolution: {integrity: sha512-z81GNO7nnYMEhrGh9LeymoE4+Yr0Wn5McHIZMK5cfQCl+NDX08sCZgUc9/6MHni9IWuFLm1Z3HTCXu2z9fN62Q==} @@ -889,10 +1809,64 @@ packages: engines: {node: ^10 || ^12 || ^13.7 || ^14 || >=15.0.1} hasBin: true + napi-build-utils@2.0.0: + resolution: {integrity: sha512-GEbrYkbfF7MoNaoh2iGG84Mnf/WZfB0GdGEsM8wz7Expx/LlWf5U8t9nvJKXSp3qr5IsEbK04cBGhol/KwOsWA==} + + natural@8.1.1: + resolution: {integrity: sha512-Ucb+lsUcGxUqu3rn8cwHjT6gJQosO63nIX/aBQXB3+IDkNbFV7PuviysO+Rzz3aKn7PZhPj3bNF4PS9gDVjYCQ==} + engines: {node: '>=0.4.10'} + + node-abi@3.94.0: + resolution: {integrity: sha512-W5ZNO5KRPB5TkYmGVD9F6YqhsglXJzE6etpbmT+f6EQElhiX/UTG551cnsRGvLG3fyZEg9HwaDmNmj5nwJ4z9g==} + engines: {node: '>=10'} + + node-domexception@1.0.0: + resolution: {integrity: sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==} + engines: {node: '>=10.5.0'} + deprecated: Use your platform's native DOMException instead + + node-fetch@2.7.0: + resolution: {integrity: sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==} + engines: {node: 4.x || >=6.0.0} + peerDependencies: + encoding: ^0.1.0 + peerDependenciesMeta: + encoding: + optional: true + + node-fetch@3.3.2: + resolution: {integrity: sha512-dRB78srN/l6gqWulah9SrxeYnxeddIG30+GOqK/9OlLVyLg3HPnr6SqOWTWOXKRwC2eGYCkZ59NNuSgvSrpgOA==} + engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} + object-assign@4.1.1: resolution: {integrity: sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg==} engines: {node: '>=0.10.0'} + obuf@1.1.2: + resolution: {integrity: sha512-PX1wu0AmAdPqOL1mWhqmlOd8kOIZQwGZw6rh7uby9fTc5lhaOWFLX3I6R1hrF9k3zUY40e6igsLGkDXK92LJNg==} + + ollama@0.5.18: + resolution: {integrity: sha512-lTFqTf9bo7Cd3hpF6CviBe/DEhewjoZYd9N/uCe7O20qYTvGqrNOFOBDj3lbZgFWHUgDv5EeyusYxsZSLS8nvg==} + + once@1.4.0: + resolution: {integrity: sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==} + + open@10.2.0: + resolution: {integrity: sha512-YgBpdJHPyQ2UE5x+hlSXcnejzAvD0b22U2OuAP+8OnlJT+PjWPxtgmGqKKc+RgTM63U9gN0YzrYc71R2WT/hTA==} + engines: {node: '>=18'} + + openai@4.104.0: + resolution: {integrity: sha512-p99EFNsA/yX6UhVO93f5kJsDRLAg+CTA2RBqdHK4RtK8u5IJw32Hyb2dTGKbnnFmnuoBv5r7Z2CURI9sGZpSuA==} + hasBin: true + peerDependencies: + ws: 8.21.0 + zod: ^3.23.8 + peerDependenciesMeta: + ws: + optional: true + zod: + optional: true + openapi3-ts@4.5.0: resolution: {integrity: sha512-jaL+HgTq2Gj5jRcfdutgRGLosCy/hT8sQf6VOy+P+g36cZOjI1iukdPnijC+4CmeRzg/jEllJUboEic2FhxhtQ==} @@ -904,6 +1878,25 @@ packages: typescript: optional: true + p-finally@1.0.0: + resolution: {integrity: sha512-LICb2p9CB7FS+0eR1oqWnHhp0FljGLZCWBE9aix0Uye9W8LTQPwMTYVGWQWIw9RdQiDg4+epXQODwIYJtSJaow==} + engines: {node: '>=4'} + + p-queue@6.6.2: + resolution: {integrity: sha512-RwFpb72c/BhQLEXIZ5K2e+AhgNVmIejGlTgiB9MzZ0e93GRvqZ7uSi0dvRF7/XIXDeNkra2fNHBxTyPDGySpjQ==} + engines: {node: '>=8'} + + p-retry@4.6.2: + resolution: {integrity: sha512-312Id396EbJdvRONlngUx0NydfrIQ5lsYu0znKVUzVvArzEIt08V1qhtyESbGVd1FGX7UKtiFp5uwKZdM8wIuQ==} + engines: {node: '>=8'} + + p-timeout@3.2.0: + resolution: {integrity: sha512-rhIwUycgwwKcP9yTOOFK/AKsAopjjCakVqLHePO3CC6Mir1Z99xT+R63jZxAT5lFZLa2inS5h+ZS2GvR99/FBg==} + engines: {node: '>=8'} + + packet-reader@1.0.0: + resolution: {integrity: sha512-HAKu/fG3HpHFO0AA8WE8q2g+gBJaZ9MG7fcKk+IJPLTGAD6Psw4443l+9DGRbOIh3/aXr7Phy0TjilYivJo5XQ==} + pathe@2.0.3: resolution: {integrity: sha512-WUjGcAqP1gQacoQe+OBJsFA7Ld4DyXuUIjZ5cc75cLHvJ7dtNsTugphxIADwspS+AraAUePCKrSVtPLFj/F88w==} @@ -911,9 +1904,67 @@ packages: resolution: {integrity: sha512-//nshmD55c46FuFw26xV/xFAaB5HF9Xdap7HJBBnrKdAd6/GxDBaNA1870O79+9ueg61cZLSVc+OaFlfmObYVQ==} engines: {node: '>= 14.16'} + pg-cloudflare@1.4.0: + resolution: {integrity: sha512-Vo7z/6rrQYxpNRylp4Tlob2elzbh+N/MOQbxFVWCxS7oEx6jF53GTJFxK2WWpKuBRkmiin4Mt+xofFDjx09R0A==} + + pg-connection-string@2.14.0: + resolution: {integrity: sha512-XwWDGcLRGCXAR8F/AM5bG7Q+A3Wm2s6QeEjlOKZLlH3UYcguiqCWKyWXVag5TLTIjR7oOJUY8kcADaZgWPyLeg==} + + pg-int8@1.0.1: + resolution: {integrity: sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw==} + engines: {node: '>=4.0.0'} + + pg-numeric@1.0.2: + resolution: {integrity: sha512-BM/Thnrw5jm2kKLE5uJkXqqExRUY/toLHda65XgFTBTFYZyopbKjBe29Ii3RbkvlsMoFwD+tHeGaCjjv0gHlyw==} + engines: {node: '>=4'} + + pg-pool@3.14.0: + resolution: {integrity: sha512-gKtPkFdQPU3DksooVLi9LsjZxrsBUZIpa+7aVx+LV5pNh0KzP4Zleud2po+ConrxbuXGBJ6Hfer6hdgpIBpBaw==} + peerDependencies: + pg: '>=8.0' + + pg-protocol@1.15.0: + resolution: {integrity: sha512-cq9sECI5s0+uPUXjbz8ioyPJni6RzsRib0US67i5IoTZKw8fNeYlVE7u8F4dG7vEJJtc5wdD1K189lCCUwqWTQ==} + + pg-types@2.2.0: + resolution: {integrity: sha512-qTAAlrEsl8s4OiEQY69wDvcMIdQN6wdz5ojQiOy6YRMuynxenON0O5oCpJI6lshc6scgAY8qvJ2On/p+CXY0GA==} + engines: {node: '>=4'} + + pg-types@4.1.0: + resolution: {integrity: sha512-o2XFanIMy/3+mThw69O8d4n1E5zsLhdO+OPqswezu7Z5ekP4hYDqlDjlmOpYMbzY2Br0ufCwJLdDIXeNVwcWFg==} + engines: {node: '>=10'} + + pg@8.11.3: + resolution: {integrity: sha512-+9iuvG8QfaaUrrph+kpF24cXkH1YOOUeArRNYIxq1viYHZagBxrTno7cecY1Fa44tJeZvaoG+Djpkc3JwehN5g==} + engines: {node: '>= 8.0.0'} + peerDependencies: + pg-native: '>=3.0.1' + peerDependenciesMeta: + pg-native: + optional: true + + pg@8.22.0: + resolution: {integrity: sha512-8wih1vVIBMxoUM2oB4soJsD9tDnDpLv4OXBJ+EJzFsvycD+lfyIreC2gGHq78f8jbLLt+bvlPTFdFZfJkOuzAA==} + engines: {node: '>= 16.0.0'} + peerDependencies: + pg-native: '>=3.0.1' + peerDependenciesMeta: + pg-native: + optional: true + + pgpass@1.0.5: + resolution: {integrity: sha512-FdW9r/jQZhSeohs1Z3sI1yxFQNFvMcnmfuj4WBMUTxOrAyLMaTcE1aAMBiTlbMNaXvBCQuVi0R7hd8udDSP7ug==} + + pgpass@1.0.6: + resolution: {integrity: sha512-lqIfH7bdgsxHAY/ZnUOwm+aCFKrsHBDhSFuk9O0B9uCqJAIkrKTo/+LQqLPLUS4e04+jCmQVikxE3QipH5chPw==} + picocolors@1.1.1: resolution: {integrity: sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==} + picomatch@2.3.2: + resolution: {integrity: sha512-V7+vQEJ06Z+c5tSye8S+nHUfI51xoXIXjHQ99cQtKUkQqqO1kO/KCJUfZXuB47h/YBlDhah2H3hdUGXn8ie0oA==} + engines: {node: '>=8.6'} + picomatch@4.0.4: resolution: {integrity: sha512-QP88BAKvMam/3NxH6vj2o21R6MjxZUAd6nlwAS/pnGvN9IVLocLHxGYIzFhg6fUQ+5th6P4dv4eW9jX3DSIj7A==} engines: {node: '>=12'} @@ -947,13 +1998,91 @@ packages: resolution: {integrity: sha512-qif0+jGGZoLWdHey3UFHHWP0H7Gbmsk8T5VEqyYFbWqPr1XqvLGBbk/sl8V5exGmcYJklJOhOQq1pV9IcsiFag==} engines: {node: ^10 || ^12 || >=14} + postgres-array@2.0.0: + resolution: {integrity: sha512-VpZrUqU5A69eQyW2c5CA1jtLecCsN2U/bD6VilrFDWq5+5UIEVO7nazS3TEcHf1zuPYO/sqGvUvW62g86RXZuA==} + engines: {node: '>=4'} + + postgres-array@3.0.4: + resolution: {integrity: sha512-nAUSGfSDGOaOAEGwqsRY27GPOea7CNipJPOA7lPbdEpx5Kg3qzdP0AaWC5MlhTWV9s4hFX39nomVZ+C4tnGOJQ==} + engines: {node: '>=12'} + + postgres-bytea@1.0.1: + resolution: {integrity: sha512-5+5HqXnsZPE65IJZSMkZtURARZelel2oXUEO8rH83VS/hxH5vv1uHquPg5wZs8yMAfdv971IU+kcPUczi7NVBQ==} + engines: {node: '>=0.10.0'} + + postgres-bytea@3.0.0: + resolution: {integrity: sha512-CNd4jim9RFPkObHSjVHlVrxoVQXz7quwNFpz7RY1okNNme49+sVyiTvTRobiLV548Hx/hb1BG+iE7h9493WzFw==} + engines: {node: '>= 6'} + + postgres-date@1.0.7: + resolution: {integrity: sha512-suDmjLVQg78nMK2UZ454hAG+OAW+HQPZ6n++TNDUX+L0+uUlLywnoxJKDou51Zm+zTCjrCl0Nq6J9C5hP9vK/Q==} + engines: {node: '>=0.10.0'} + + postgres-date@2.1.0: + resolution: {integrity: sha512-K7Juri8gtgXVcDfZttFKVmhglp7epKb1K4pgrkLxehjqkrgPhfG6OO8LHLkfaqkbpjNRnra018XwAr1yQFWGcA==} + engines: {node: '>=12'} + + postgres-interval@1.2.0: + resolution: {integrity: sha512-9ZhXKM/rw350N1ovuWHbGxnGh/SNJ4cnxHiM0rxE4VN41wsg8P8zWn9hv/buK00RP4WvlOyr/RBDiptyxVbkZQ==} + engines: {node: '>=0.10.0'} + + postgres-interval@3.0.0: + resolution: {integrity: sha512-BSNDnbyZCXSxgA+1f5UU2GmwhoI0aU5yMxRGO8CdFEcY2BQF9xm/7MqKnYoM1nJDk8nONNWDk9WeSmePFhQdlw==} + engines: {node: '>=12'} + + postgres-range@1.1.4: + resolution: {integrity: sha512-i/hbxIE9803Alj/6ytL7UHQxRvZkI9O4Sy+J3HGc4F4oo/2eQAjTSNJ0bfxyse3bH0nuVesCk+3IRLaMtG3H6w==} + + prebuild-install@7.1.3: + resolution: {integrity: sha512-8Mf2cbV7x1cXPUILADGI3wuhfqWvtiLA1iclTDbFRZkgRQS0NqsPZphna9V+HyTEadheuPmjaJMsbzKQFOzLug==} + engines: {node: '>=10'} + deprecated: No longer maintained. Please contact the author of the relevant native addon; alternatives are available. + hasBin: true + + pretty-format@29.7.0: + resolution: {integrity: sha512-Pdlw/oPxN+aXdmM9R00JVC9WVFoCLTKJvDVLgmJ+qAffBMxsV85l/Lu7sNx4zSzPyoL2euImuEwHhOXdEgNFZQ==} + engines: {node: ^14.15.0 || ^16.10.0 || >=18.0.0} + proper-lockfile@4.1.2: resolution: {integrity: sha512-TjNPblN4BwAWMXU8s9AEz4JmQxnD1NNL7bNOY/AKUzyamc379FWASUhc/K1pL2noVb+XmZKLL68cjzLsiOAMaA==} + protobufjs@7.6.5: + resolution: {integrity: sha512-/FPD0nUc9jH6rfFjji9IBqOz4pcSE3CsT1m7Ep6Mdb0LxSUMj8hgl6GomOvZzpNpAqqGaXA0P3VSrZLFzIhQrw==} + engines: {node: '>=12.0.0'} + + proxy-from-env@2.1.0: + resolution: {integrity: sha512-cJ+oHTW1VAEa8cJslgmUZrc+sjRKgAKl3Zyse6+PV38hZe/V6Z14TbCuXcan9F9ghlz4QrFr2c92TNF82UkYHA==} + engines: {node: '>=10'} + + pump@3.0.4: + resolution: {integrity: sha512-VS7sjc6KR7e1ukRFhQSY5LM2uBWAUPiOPa/A3mkKmiMwSmRFUITt0xuj+/lesgnCv+dPIEYlkzrcyXgquIHMcA==} + + punycode@2.3.1: + resolution: {integrity: sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==} + engines: {node: '>=6'} + + rc@1.2.8: + resolution: {integrity: sha512-y3bGgqKj3QBdxLbLkomlohkvsA8gdAiUQlSBJnBhfn+BPxg4bc62d8TcBW15wavDfgexCgccckhcZvywyQYPOw==} + hasBin: true + + react-is@18.3.1: + resolution: {integrity: sha512-/LLMVyas0ljjAtoYiPqYiL8VWXzUUdThrmU5+n20DZv+a+ClRoevUzw5JxU+Ieh5/c87ytoTBV9G1FiKfNJdmg==} + + readable-stream@3.6.2: + resolution: {integrity: sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==} + engines: {node: '>= 6'} + readdirp@4.1.2: resolution: {integrity: sha512-GDhwkLfywWL2s6vEjyhri+eXmfH6j1L7JE27WhqLeYzoh/A3DBaYGEj2H/HFZCn/kMfim73FXxEJTw06WtxQwg==} engines: {node: '>= 14.18.0'} + redis@4.7.1: + resolution: {integrity: sha512-S1bJDnqLftzHXHP8JsT5II/CtHWQrASX5K96REjWjlmWKrviSOLWmM7QnRLstAWsu1VBBV1ffV6DzCvxNP0UJQ==} + + redis@5.12.1: + resolution: {integrity: sha512-LDsoVvb/CpoV9EN3FXvgvSHNJWuCIzl9MiO3ppOevuGLpSGJhwfQjpEwfFJcQvNSddHADDdZaWx0HnmMxRXG7g==} + engines: {node: '>= 18.19.0'} + resolve-from@5.0.0: resolution: {integrity: sha512-qYg9KP24dD5qka9J47d0aVky0N+b4fTU89LN9iDnjB5waksiC49rvMB0PrUJQGoTmH50XPiqOvAjDfaijGxYZw==} engines: {node: '>=8'} @@ -962,17 +2091,50 @@ packages: resolution: {integrity: sha512-9LkiTwjUh6rT555DtE9rTX+BKByPfrMzEAtnlEtdEwr3Nkffwiihqe2bWADg+OQRjt9gl6ICdmB/ZFDCGAtSow==} engines: {node: '>= 4'} + retry@0.13.1: + resolution: {integrity: sha512-XQBQ3I8W1Cge0Seh+6gjj03LbmRFWuoszgK9ooCpwYIrhhoO80pfq4cUkU5DkknwfOfFteRwlZ56PYOGYyFWdg==} + engines: {node: '>= 4'} + rollup@4.60.2: resolution: {integrity: sha512-J9qZyW++QK/09NyN/zeO0dG/1GdGfyp9lV8ajHnRVLfo/uFsbji5mHnDgn/qYdUHyCkM2N+8VyspgZclfAh0eQ==} engines: {node: '>=18.0.0', npm: '>=8.0.0'} hasBin: true + run-applescript@7.1.0: + resolution: {integrity: sha512-DPe5pVFaAsinSaV6QjQ6gdiedWDcRCbUuiQfQa2wmWV7+xC9bGulGI8+TdRmoFkAPaBXk8CrAbnlY2ISniJ47Q==} + engines: {node: '>=18'} + + safe-buffer@5.2.1: + resolution: {integrity: sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==} + + safe-stable-stringify@2.5.0: + resolution: {integrity: sha512-b3rppTKm9T+PsVCBEOUR46GWI7fdOs00VKZ1+9c1EWDaDMvjQc6tUwuFyIprgGgTcWoVHSKrU8H31ZHA2e0RHA==} + engines: {node: '>=10'} + + semver@7.8.5: + resolution: {integrity: sha512-Y7/KDsb8LjooZpwaqGyulO6DQlksgCncchHGk+sZIY4SBvUocMBEFH5Ur1fI4dV+Jvl0w6cjvucaIi40puRioA==} + engines: {node: '>=10'} + hasBin: true + + sift@17.1.3: + resolution: {integrity: sha512-Rtlj66/b0ICeFzYTuNvX/EF1igRbbnGSvEyT79McoZa/DeGhMyC5pWKOEsZKnpkqtSeovd5FL/bjHWC3CIIvCQ==} + siginfo@2.0.0: resolution: {integrity: sha512-ybx0WO1/8bSBLEWXZvEd7gMW3Sn3JFlW3TvX1nREbDLRNQNaeNN8WK0meBwPdAaOI7TtRRRJn/Es1zhrrCHu7g==} signal-exit@3.0.7: resolution: {integrity: sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ==} + simple-concat@1.0.1: + resolution: {integrity: sha512-cSFtAPtRhljv69IK0hTVZQ+OfE9nePi/rtJmw5UjHeVyVroEqJXP1sFztKUy1qU+xvz3u/sfYJLa947b7nAN2Q==} + + simple-get@4.0.1: + resolution: {integrity: sha512-brv7p5WgH0jmQJr1ZDDfKDOSeWWg+OVypG99A/5vYGPqJ6pxiaHLy8nxtFjBA7oMa01ebA9gfh1uMCFqOuXxvA==} + + slash@3.0.0: + resolution: {integrity: sha512-g9Q1haeby36OSStwb4ntCGGGaKsaVSjQ68fBxoQcutl5fS1vuY18H3wSt3jFyFtrkx+Kz0V1G85A4MyAdDMi2Q==} + engines: {node: '>=8'} + source-map-js@1.2.1: resolution: {integrity: sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==} engines: {node: '>=0.10.0'} @@ -981,12 +2143,34 @@ packages: resolution: {integrity: sha512-i5uvt8C3ikiWeNZSVZNWcfZPItFQOsYTUAOkcUPGd8DqDy1uOUikjt5dG+uRlwyvR108Fb9DOd4GvXfT0N2/uQ==} engines: {node: '>= 12'} + sparse-bitfield@3.0.3: + resolution: {integrity: sha512-kvzhi7vqKTfkh0PZU+2D2PIllw2ymqJKujUcyPMd9Y75Nv4nPbGJZXNhxsgdQab2BmlDct1YnfQCguEvHr7VsQ==} + + split2@4.2.0: + resolution: {integrity: sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg==} + engines: {node: '>= 10.x'} + + stack-utils@2.0.6: + resolution: {integrity: sha512-XlkWvfIm6RmsWtNJx+uqtKLS8eqFbxUg0ZzLXqY0caEy9l7hruX8IpiDnjsLavoBgqCCR71TqWO8MaXYheJ3RQ==} + engines: {node: '>=10'} + stackback@0.0.2: resolution: {integrity: sha512-1XMJE5fQo1jGH6Y/7ebnwPOBEkIEnT4QF32d5R1+VXdXveM0IBMJt8zfaxX1P3QhVwrYe+576+jkANtSS2mBbw==} std-env@3.10.0: resolution: {integrity: sha512-5GS12FdOZNliM5mAOxFRg7Ir0pWz8MdpYm6AY6VPkGpbA7ZzmbzNcBJQ0GPvvyWgcY7QAhCgf9Uy89I03faLkg==} + stopwords-iso@1.1.0: + resolution: {integrity: sha512-I6GPS/E0zyieHehMRPQcqkiBMJKGgLta+1hREixhoLPqEA0AlVFiC43dl8uPpmkkeRdDMzYRWFWk5/l9x7nmNg==} + engines: {node: '>=0.10.0'} + + string_decoder@1.3.0: + resolution: {integrity: sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==} + + strip-json-comments@2.0.1: + resolution: {integrity: sha512-4gB8na07fecVVkOI6Rs4e7T6NOTki5EmL7TUduTs6bu3EdnSycntVJ4re8kgZA+wx9IueI2Y11bfbgwtzuE0KQ==} + engines: {node: '>=0.10.0'} + strip-literal@3.1.0: resolution: {integrity: sha512-8r3mkIM/2+PpjHoOtiAW8Rg3jJLHaV7xPwG+YRGrv6FP0wwk/toTpATxWYOW0BKdWwl82VT2tFYi5DlROa0Mxg==} @@ -995,6 +2179,24 @@ packages: engines: {node: '>=16 || 14 >=14.17'} hasBin: true + suffix-thumb@5.0.2: + resolution: {integrity: sha512-I5PWXAFKx3FYnI9a+dQMWNqTxoRt6vdBdb0O+BJ1sxXCWtSoQCusc13E58f+9p4MYx/qCnEMkD5jac6K2j3dgA==} + + supports-color@7.2.0: + resolution: {integrity: sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==} + engines: {node: '>=8'} + + sylvester@0.0.21: + resolution: {integrity: sha512-yUT0ukFkFEt4nb+NY+n2ag51aS/u9UHXoZw+A4jgD77/jzZsBoSDHuqysrVCBC4CYR4TYvUJq54ONpXgDBH8tA==} + engines: {node: '>=0.2.6'} + + tar-fs@2.1.5: + resolution: {integrity: sha512-OboTd8mmMhZDNPV+UjQcK9yKAatXu2aJ+r1w4im1Otd4M4fl2hwvdoXUxIYHFTHWK/3y3FarBP70v3vwmGlOxw==} + + tar-stream@2.2.0: + resolution: {integrity: sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ==} + engines: {node: '>=6'} + thenify-all@1.6.0: resolution: {integrity: sha512-RNxQH/qI8/t3thXJDwcstUO4zeqo64+Uy/+sNVRBx4Xn2OX+OZ9oP+iJnNFqplFra2ZUVeKCSa2oVWi3T4uVmA==} engines: {node: '>=0.8'} @@ -1024,6 +2226,17 @@ packages: resolution: {integrity: sha512-azl+t0z7pw/z958Gy9svOTuzqIk6xq+NSheJzn5MMWtWTFywIacg2wUlzKFGtt3cthx0r2SxMK0yzJOR0IES7Q==} engines: {node: '>=14.0.0'} + to-regex-range@5.0.1: + resolution: {integrity: sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==} + engines: {node: '>=8.0'} + + tr46@0.0.3: + resolution: {integrity: sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==} + + tr46@5.1.1: + resolution: {integrity: sha512-hdF5ZgjTqgAntKkklYw0R03MG2x/bSzTtkxmIRw/sTNV8YXsCJ1tfLAX23lhxhHJlEf3CRCOCGGWw3vI3GaSPw==} + engines: {node: '>=18'} + tree-kill@1.2.2: resolution: {integrity: sha512-L0Orpi8qGpRG//Nd+H90vFB+3iHnue1zSSGmNOOCh1GLJ7rUKVwV2HvijphGQS2UmhUZewS9VgvxYIdgr+fG1A==} hasBin: true @@ -1031,6 +2244,9 @@ packages: ts-interface-checker@0.1.13: resolution: {integrity: sha512-Y/arvbn+rrz3JCKl9C4kVNfTfSm2/mEp5FSz5EsZSANGPSlQrpRI5M4PKF+mJnE52jOO90PnPSc3Ur3bTQw0gA==} + tslib@2.8.1: + resolution: {integrity: sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==} + tsup@8.5.1: resolution: {integrity: sha512-xtgkqwdhpKWr3tKPmCkvYmS9xnQK3m3XgxZHwSUjvfTjp7YfXe5tT3GgWi0F2N+ZSMsOeWeZFh7ZZFg5iPhing==} engines: {node: '>=18'} @@ -1055,6 +2271,9 @@ packages: engines: {node: '>=18.0.0'} hasBin: true + tunnel-agent@0.6.0: + resolution: {integrity: sha512-McnNiV1l8RYeY8tBgEpuodCC1mLUdbSN+CYBL7kJsJNInOP8UjDDEwdk6Mw60vdLLrr5NHKZhMAOSrR2NZuQ+w==} + typescript@5.9.3: resolution: {integrity: sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==} engines: {node: '>=14.17'} @@ -1063,9 +2282,30 @@ packages: ufo@1.6.4: resolution: {integrity: sha512-JFNbkD1Svwe0KvGi8GOeLcP4kAWQ609twvCdcHxq1oSL8svv39ZuSvajcD8B+5D0eL4+s1Is2D/O6KN3qcTeRA==} + underscore@1.13.8: + resolution: {integrity: sha512-DXtD3ZtEQzc7M8m4cXotyHR+FAS18C64asBYY5vqZexfYryNNnDc02W4hKg3rdQuqOYas1jkseX0+nZXjTXnvQ==} + + undici-types@5.26.5: + resolution: {integrity: sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==} + undici-types@7.19.2: resolution: {integrity: sha512-qYVnV5OEm2AW8cJMCpdV20CDyaN3g0AjDlOGf1OW4iaDEx8MwdtChUp4zu4H0VP3nDRF/8RKWH+IPp9uW0YGZg==} + undici@6.27.0: + resolution: {integrity: sha512-YmfV3YnEDzXRC5lZ2jWtWWHKGUm1zIt8AhesR1tens+HTNv+YZlN/dp6G727LOvMJ8xjP9Be7Y2Sdr96LDm+pg==} + engines: {node: '>=18.17'} + + util-deprecate@1.0.2: + resolution: {integrity: sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==} + + uuid@11.1.1: + resolution: {integrity: sha512-vIYxrBCC/N/K+Js3qSN88go7kIfNPssr/hHCesKCQNAjmgvYS2oqr69kIufEG+O4+PfezOH4EbIeHCfFov8ZgQ==} + hasBin: true + + uuid@13.0.2: + resolution: {integrity: sha512-vzi9uRZ926x4XV73S/4qQaTwPXM2JBj6/6lI/byHH1jOpCzb0zDbfytgA9LcN/hzb2l7WQSQnxITOVx5un/wGw==} + hasBin: true + viem@2.48.8: resolution: {integrity: sha512-Xj3Nrt66SKtn06kczU91ELn9Difr84ZM5A62BTlaisT5lpgt058i2mBkfMZCXHGb1ocOLjzC2ztPhD0Lvky7uQ==} peerDependencies: @@ -1147,11 +2387,43 @@ packages: jsdom: optional: true + web-streams-polyfill@3.3.3: + resolution: {integrity: sha512-d2JWLCivmZYTSIoge9MsgFCZrt571BikcWGYkjC1khllbTeDlGqZ2D8vD8E/lJa8WGWbb7Plm8/XJYV7IJHZZw==} + engines: {node: '>= 8'} + + web-streams-polyfill@4.0.0-beta.3: + resolution: {integrity: sha512-QW95TCTaHmsYfHDybGMwO5IJIM93I/6vTRk+daHTWFPhwh+C8Cg7j7XyKrwrj8Ib6vYXe0ocYNrmzY4xAAN6ug==} + engines: {node: '>= 14'} + + webidl-conversions@3.0.1: + resolution: {integrity: sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==} + + webidl-conversions@7.0.0: + resolution: {integrity: sha512-VwddBukDzu71offAQR975unBIGqfKZpM+8ZX6ySk8nYhVoo5CYaZyzt3YBvYtRtO+aoGlqxPg/B87NGVZ/fu6g==} + engines: {node: '>=12'} + + whatwg-fetch@3.6.20: + resolution: {integrity: sha512-EqhiFU6daOA8kpjOWTL0olhVOF3i7OrFzSYiGsEMB8GcXS+RrzauAERX65xMeNWVqxA6HXH2m69Z9LaKKdisfg==} + + whatwg-url@14.2.0: + resolution: {integrity: sha512-De72GdQZzNTUBBChsXueQUnPKDkg/5A5zp7pFDuQAj5UFoENpiACU0wlCvzpAGnTkj++ihpKwKyYewn/XNUbKw==} + engines: {node: '>=18'} + + whatwg-url@5.0.0: + resolution: {integrity: sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==} + why-is-node-running@2.3.0: resolution: {integrity: sha512-hUrmaWBdVDcxvYqnyh09zunKzROWjbZTiNy8dBEjkS7ehEDQibXJ7XvlmtbwuTclUiIyN+CyXQD4Vmko8fNm8w==} engines: {node: '>=8'} hasBin: true + wordnet-db@3.1.14: + resolution: {integrity: sha512-zVyFsvE+mq9MCmwXUWHIcpfbrHHClZWZiVOzKSxNJruIcFn2RbY55zkhiAMMxM8zCVSmtNiViq8FsAZSFpMYag==} + engines: {node: '>=0.6.0'} + + wrappy@1.0.2: + resolution: {integrity: sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==} + ws@8.21.0: resolution: {integrity: sha512-Vsp28b7DRcimFQvrqu2Wek3z1iYxDCWqHYB8Qsnk/S4RfaCQzPGPyBNuVjJV3cd6UiKtUtp6sNM77gWvzcCH+g==} engines: {node: '>=10.0.0'} @@ -1164,11 +2436,30 @@ packages: utf-8-validate: optional: true + wsl-utils@0.1.0: + resolution: {integrity: sha512-h3Fbisa2nKGPxCpm89Hk33lBLsnaGBvctQopaBSOW/uIs6FTe1ATyAnKFJrzVs9vpGdsTe73WF3V4lIsk4Gacw==} + engines: {node: '>=18'} + + xtend@4.0.2: + resolution: {integrity: sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==} + engines: {node: '>=0.4'} + + yallist@4.0.0: + resolution: {integrity: sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==} + yaml@2.8.4: resolution: {integrity: sha512-ml/JPOj9fOQK8RNnWojA67GbZ0ApXAUlN2UQclwv2eVgTgn7O9gg9o7paZWKMp4g0H3nTLtS9LVzhkpOFIKzog==} engines: {node: '>= 14.6'} hasBin: true + zod-to-json-schema@3.25.2: + resolution: {integrity: sha512-O/PgfnpT1xKSDeQYSCfRI5Gy3hPf91mKVDuYLUHZJMiDFptvP41MSnWofm8dnCm0256ZNfZIM7DSzuSMAFnjHA==} + peerDependencies: + zod: ^3.25.28 || ^4 + + zod@3.25.76: + resolution: {integrity: sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ==} + zod@4.4.2: resolution: {integrity: sha512-IynmDyxsEsb9RKzO3J9+4SxXnl2FTFSzNBaKKaMV6tsSk0rw9gYw9gs+JFCq/qk2LCZ78KDwyj+Z289TijSkUw==} @@ -1179,6 +2470,18 @@ snapshots: '@adraffy/ens-normalize@1.11.1': {} + '@anthropic-ai/sdk@0.40.1': + dependencies: + '@types/node': 18.19.130 + '@types/node-fetch': 2.6.13 + abort-controller: 3.0.0 + agentkeepalive: 4.6.0 + form-data-encoder: 1.7.2 + formdata-node: 4.4.1 + node-fetch: 2.7.0 + transitivePeerDependencies: + - encoding + '@asteasolutions/zod-to-openapi@8.5.0(zod@4.4.3)': dependencies: openapi3-ts: 4.5.0 @@ -1190,6 +2493,121 @@ snapshots: optionalDependencies: zod: 4.4.3 + '@azure/abort-controller@2.2.0': + dependencies: + tslib: 2.8.1 + + '@azure/core-auth@1.11.0': + dependencies: + '@azure/abort-controller': 2.2.0 + '@azure/core-util': 1.14.0 + tslib: 2.8.1 + transitivePeerDependencies: + - supports-color + + '@azure/core-client@1.11.0': + dependencies: + '@azure/abort-controller': 2.2.0 + '@azure/core-auth': 1.11.0 + '@azure/core-rest-pipeline': 1.25.0 + '@azure/core-tracing': 1.4.0 + '@azure/core-util': 1.14.0 + '@azure/logger': 1.4.0 + tslib: 2.8.1 + transitivePeerDependencies: + - supports-color + + '@azure/core-http-compat@2.5.0(@azure/core-client@1.11.0)(@azure/core-rest-pipeline@1.25.0)': + dependencies: + '@azure/abort-controller': 2.2.0 + '@azure/core-client': 1.11.0 + '@azure/core-rest-pipeline': 1.25.0 + + '@azure/core-paging@1.7.0': + dependencies: + tslib: 2.8.1 + + '@azure/core-rest-pipeline@1.25.0': + dependencies: + '@azure/abort-controller': 2.2.0 + '@azure/core-auth': 1.11.0 + '@azure/core-tracing': 1.4.0 + '@azure/core-util': 1.14.0 + '@azure/logger': 1.4.0 + '@typespec/ts-http-runtime': 0.3.7 + tslib: 2.8.1 + transitivePeerDependencies: + - supports-color + + '@azure/core-tracing@1.4.0': + dependencies: + tslib: 2.8.1 + + '@azure/core-util@1.14.0': + dependencies: + '@azure/abort-controller': 2.2.0 + '@typespec/ts-http-runtime': 0.3.7 + tslib: 2.8.1 + transitivePeerDependencies: + - supports-color + + '@azure/identity@4.13.1': + dependencies: + '@azure/abort-controller': 2.2.0 + '@azure/core-auth': 1.11.0 + '@azure/core-client': 1.11.0 + '@azure/core-rest-pipeline': 1.25.0 + '@azure/core-tracing': 1.4.0 + '@azure/core-util': 1.14.0 + '@azure/logger': 1.4.0 + '@azure/msal-browser': 5.17.1 + '@azure/msal-node': 5.4.1 + open: 10.2.0 + tslib: 2.8.1 + transitivePeerDependencies: + - supports-color + + '@azure/logger@1.4.0': + dependencies: + '@typespec/ts-http-runtime': 0.3.7 + tslib: 2.8.1 + transitivePeerDependencies: + - supports-color + + '@azure/msal-browser@5.17.1': + dependencies: + '@azure/msal-common': 16.11.2 + + '@azure/msal-common@16.11.2': {} + + '@azure/msal-node@5.4.1': + dependencies: + '@azure/msal-common': 16.11.2 + jsonwebtoken: 9.0.3 + + '@azure/search-documents@12.2.0': + dependencies: + '@azure/core-auth': 1.11.0 + '@azure/core-client': 1.11.0 + '@azure/core-http-compat': 2.5.0(@azure/core-client@1.11.0)(@azure/core-rest-pipeline@1.25.0) + '@azure/core-paging': 1.7.0 + '@azure/core-rest-pipeline': 1.25.0 + '@azure/core-tracing': 1.4.0 + '@azure/core-util': 1.14.0 + '@azure/logger': 1.4.0 + events: 3.3.0 + tslib: 2.8.1 + transitivePeerDependencies: + - supports-color + + '@babel/code-frame@7.29.7': + dependencies: + '@babel/helper-validator-identifier': 7.29.7 + js-tokens: 4.0.0 + picocolors: 1.1.1 + + '@babel/helper-validator-identifier@7.29.7': {} + '@biomejs/biome@2.4.15': optionalDependencies: '@biomejs/cli-darwin-arm64': 2.4.15 @@ -1225,6 +2643,10 @@ snapshots: '@biomejs/cli-win32-x64@2.4.15': optional: true + '@cfworker/json-schema@4.1.1': {} + + '@cloudflare/workers-types@4.20260702.1': {} + '@esbuild/aix-ppc64@0.27.7': optional: true @@ -1381,10 +2803,38 @@ snapshots: '@esbuild/win32-x64@0.28.1': optional: true + '@google/genai@1.52.0': + dependencies: + google-auth-library: 10.9.0 + p-retry: 4.6.2 + protobufjs: 7.6.5 + ws: 8.21.0 + transitivePeerDependencies: + - bufferutil + - supports-color + - utf-8-validate + '@hono/node-server@2.0.1(hono@4.12.30)': dependencies: hono: 4.12.30 + '@jest/expect-utils@29.7.0': + dependencies: + jest-get-type: 29.6.3 + + '@jest/schemas@29.6.3': + dependencies: + '@sinclair/typebox': 0.27.12 + + '@jest/types@29.6.3': + dependencies: + '@jest/schemas': 29.6.3 + '@types/istanbul-lib-coverage': 2.0.6 + '@types/istanbul-reports': 3.0.4 + '@types/node': 25.6.0 + '@types/yargs': 17.0.35 + chalk: 4.1.2 + '@jridgewell/gen-mapping@0.3.13': dependencies: '@jridgewell/sourcemap-codec': 1.5.5 @@ -1399,6 +2849,35 @@ snapshots: '@jridgewell/resolve-uri': 3.1.2 '@jridgewell/sourcemap-codec': 1.5.5 + '@langchain/core@1.2.3(@opentelemetry/api@1.9.1)(ws@8.21.0)': + dependencies: + '@cfworker/json-schema': 4.1.1 + '@standard-schema/spec': 1.1.0 + js-tiktoken: 1.0.21 + langsmith: 0.8.3(@opentelemetry/api@1.9.1)(ws@8.21.0) + mustache: 4.2.0 + p-queue: 6.6.2 + zod: 4.4.3 + transitivePeerDependencies: + - '@opentelemetry/api' + - '@opentelemetry/exporter-trace-otlp-proto' + - '@opentelemetry/sdk-trace-base' + - openai + - ws + + '@mistralai/mistralai@1.15.1': + dependencies: + ws: 8.21.0 + zod: 4.4.3 + zod-to-json-schema: 3.25.2(zod@4.4.3) + transitivePeerDependencies: + - bufferutil + - utf-8-validate + + '@mongodb-js/saslprep@1.4.12': + dependencies: + sparse-bitfield: 3.0.3 + '@neo4j-labs/agent-memory@0.4.0': {} '@noble/ciphers@1.3.0': {} @@ -1417,6 +2896,82 @@ snapshots: '@opentelemetry/api@1.9.1': {} + '@protobufjs/aspromise@1.1.2': {} + + '@protobufjs/base64@1.1.2': {} + + '@protobufjs/codegen@2.0.5': {} + + '@protobufjs/eventemitter@1.1.1': {} + + '@protobufjs/fetch@1.1.1': + dependencies: + '@protobufjs/aspromise': 1.1.2 + + '@protobufjs/float@1.0.2': {} + + '@protobufjs/path@1.1.2': {} + + '@protobufjs/pool@1.1.0': {} + + '@protobufjs/utf8@1.1.2': {} + + '@qdrant/js-client-rest@1.18.0(typescript@5.9.3)': + dependencies: + '@qdrant/openapi-typescript-fetch': 1.2.6 + typescript: 5.9.3 + undici: 6.27.0 + + '@qdrant/openapi-typescript-fetch@1.2.6': {} + + '@redis/bloom@1.2.0(@redis/client@1.6.1)': + dependencies: + '@redis/client': 1.6.1 + + '@redis/bloom@5.12.1(@redis/client@5.12.1(@opentelemetry/api@1.9.1))': + dependencies: + '@redis/client': 5.12.1(@opentelemetry/api@1.9.1) + + '@redis/client@1.6.1': + dependencies: + cluster-key-slot: 1.1.2 + generic-pool: 3.9.0 + yallist: 4.0.0 + + '@redis/client@5.12.1(@opentelemetry/api@1.9.1)': + dependencies: + cluster-key-slot: 1.1.2 + optionalDependencies: + '@opentelemetry/api': 1.9.1 + + '@redis/graph@1.1.1(@redis/client@1.6.1)': + dependencies: + '@redis/client': 1.6.1 + + '@redis/json@1.0.7(@redis/client@1.6.1)': + dependencies: + '@redis/client': 1.6.1 + + '@redis/json@5.12.1(@redis/client@5.12.1(@opentelemetry/api@1.9.1))': + dependencies: + '@redis/client': 5.12.1(@opentelemetry/api@1.9.1) + + '@redis/search@1.2.0(@redis/client@1.6.1)': + dependencies: + '@redis/client': 1.6.1 + + '@redis/search@5.12.1(@redis/client@5.12.1(@opentelemetry/api@1.9.1))': + dependencies: + '@redis/client': 5.12.1(@opentelemetry/api@1.9.1) + + '@redis/time-series@1.1.0(@redis/client@1.6.1)': + dependencies: + '@redis/client': 1.6.1 + + '@redis/time-series@5.12.1(@redis/client@5.12.1(@opentelemetry/api@1.9.1))': + dependencies: + '@redis/client': 5.12.1(@opentelemetry/api@1.9.1) + '@rollup/rollup-android-arm-eabi@4.60.2': optional: true @@ -1518,6 +3073,42 @@ snapshots: '@noble/hashes': 2.2.0 '@scure/base': 2.2.0 + '@sinclair/typebox@0.27.12': {} + + '@standard-schema/spec@1.1.0': {} + + '@supabase/auth-js@2.110.7': + dependencies: + tslib: 2.8.1 + + '@supabase/functions-js@2.110.7': + dependencies: + tslib: 2.8.1 + + '@supabase/phoenix@0.4.5': {} + + '@supabase/postgrest-js@2.110.7': + dependencies: + tslib: 2.8.1 + + '@supabase/realtime-js@2.110.7': + dependencies: + '@supabase/phoenix': 0.4.5 + tslib: 2.8.1 + + '@supabase/storage-js@2.110.7': + dependencies: + iceberg-js: 0.8.1 + tslib: 2.8.1 + + '@supabase/supabase-js@2.110.7': + dependencies: + '@supabase/auth-js': 2.110.7 + '@supabase/functions-js': 2.110.7 + '@supabase/postgrest-js': 2.110.7 + '@supabase/realtime-js': 2.110.7 + '@supabase/storage-js': 2.110.7 + '@tangle-network/agent-core@0.3.8': dependencies: '@tangle-network/agent-interface': 0.17.1 @@ -1601,16 +3192,70 @@ snapshots: '@types/estree@1.0.8': {} + '@types/istanbul-lib-coverage@2.0.6': {} + + '@types/istanbul-lib-report@3.0.3': + dependencies: + '@types/istanbul-lib-coverage': 2.0.6 + + '@types/istanbul-reports@3.0.4': + dependencies: + '@types/istanbul-lib-report': 3.0.3 + + '@types/jest@29.5.14': + dependencies: + expect: 29.7.0 + pretty-format: 29.7.0 + + '@types/node-fetch@2.6.13': + dependencies: + '@types/node': 25.6.0 + form-data: 4.0.6 + + '@types/node@18.19.130': + dependencies: + undici-types: 5.26.5 + '@types/node@25.6.0': dependencies: undici-types: 7.19.2 + '@types/pg@8.11.0': + dependencies: + '@types/node': 25.6.0 + pg-protocol: 1.15.0 + pg-types: 4.1.0 + '@types/proper-lockfile@4.1.4': dependencies: '@types/retry': 0.12.5 + '@types/retry@0.12.0': {} + '@types/retry@0.12.5': {} + '@types/stack-utils@2.0.3': {} + + '@types/webidl-conversions@7.0.3': {} + + '@types/whatwg-url@13.0.0': + dependencies: + '@types/webidl-conversions': 7.0.3 + + '@types/yargs-parser@21.0.3': {} + + '@types/yargs@17.0.35': + dependencies: + '@types/yargs-parser': 21.0.3 + + '@typespec/ts-http-runtime@0.3.7': + dependencies: + http-proxy-agent: 7.0.2 + https-proxy-agent: 7.0.6 + tslib: 2.8.1 + transitivePeerDependencies: + - supports-color + '@vitest/expect@3.2.4': dependencies: '@types/chai': 5.2.3 @@ -1658,12 +3303,94 @@ snapshots: typescript: 5.9.3 zod: 4.4.3 + abort-controller@3.0.0: + dependencies: + event-target-shim: 5.0.1 + acorn@8.16.0: {} + afinn-165-financialmarketnews@3.0.0: {} + + afinn-165@2.0.2: {} + + agent-base@6.0.2: + dependencies: + debug: 4.4.3 + transitivePeerDependencies: + - supports-color + + agent-base@7.1.4: {} + + agentkeepalive@4.6.0: + dependencies: + humanize-ms: 1.2.1 + + ansi-styles@4.3.0: + dependencies: + color-convert: 2.0.1 + + ansi-styles@5.2.0: {} + any-promise@1.3.0: {} + apparatus@0.0.10: + dependencies: + sylvester: 0.0.21 + assertion-error@2.0.1: {} + asynckit@0.4.0: {} + + axios@1.18.1: + dependencies: + follow-redirects: 1.16.0 + form-data: 4.0.6 + https-proxy-agent: 5.0.1 + proxy-from-env: 2.1.0 + transitivePeerDependencies: + - debug + - supports-color + + base-64@0.1.0: {} + + base64-js@1.5.1: {} + + better-sqlite3@12.11.1: + dependencies: + bindings: 1.5.0 + prebuild-install: 7.1.3 + + bignumber.js@9.3.1: {} + + bindings@1.5.0: + dependencies: + file-uri-to-path: 1.0.0 + + bl@4.1.0: + dependencies: + buffer: 5.7.1 + inherits: 2.0.4 + readable-stream: 3.6.2 + + braces@3.0.3: + dependencies: + fill-range: 7.1.1 + + bson@7.3.1: {} + + buffer-equal-constant-time@1.0.1: {} + + buffer-writer@2.0.0: {} + + buffer@5.7.1: + dependencies: + base64-js: 1.5.1 + ieee754: 1.2.1 + + bundle-name@4.1.0: + dependencies: + run-applescript: 7.1.0 + bundle-require@5.1.0(esbuild@0.27.7): dependencies: esbuild: 0.27.7 @@ -1671,6 +3398,11 @@ snapshots: cac@6.7.14: {} + call-bind-apply-helpers@1.0.2: + dependencies: + es-errors: 1.3.0 + function-bind: 1.1.2 + chai@5.3.3: dependencies: assertion-error: 2.0.1 @@ -1679,28 +3411,132 @@ snapshots: loupe: 3.2.1 pathval: 2.0.1 + chalk@4.1.2: + dependencies: + ansi-styles: 4.3.0 + supports-color: 7.2.0 + + charenc@0.0.2: {} + check-error@2.1.3: {} chokidar@4.0.3: dependencies: readdirp: 4.1.2 + chownr@1.1.4: {} + + ci-info@3.9.0: {} + + cloudflare@4.5.0: + dependencies: + '@types/node': 18.19.130 + '@types/node-fetch': 2.6.13 + abort-controller: 3.0.0 + agentkeepalive: 4.6.0 + form-data-encoder: 1.7.2 + formdata-node: 4.4.1 + node-fetch: 2.7.0 + transitivePeerDependencies: + - encoding + + cluster-key-slot@1.1.2: {} + + color-convert@2.0.1: + dependencies: + color-name: 1.1.4 + + color-name@1.1.4: {} + + combined-stream@1.0.8: + dependencies: + delayed-stream: 1.0.0 + commander@14.0.3: {} commander@4.1.1: {} + compromise@14.16.0: + dependencies: + efrt: 2.7.0 + grad-school: 0.0.5 + suffix-thumb: 5.0.2 + confbox@0.1.8: {} consola@3.4.2: {} + crypt@0.0.2: {} + + data-uri-to-buffer@4.0.1: {} + debug@4.4.3: dependencies: ms: 2.1.3 + decompress-response@6.0.0: + dependencies: + mimic-response: 3.1.0 + deep-eql@5.0.2: {} + deep-extend@0.6.0: {} + + default-browser-id@5.0.1: {} + + default-browser@5.5.0: + dependencies: + bundle-name: 4.1.0 + default-browser-id: 5.0.1 + + define-lazy-prop@3.0.0: {} + + delayed-stream@1.0.0: {} + + detect-libc@2.1.2: {} + + diff-sequences@29.6.3: {} + + digest-fetch@1.3.0: + dependencies: + base-64: 0.1.0 + md5: 2.3.0 + + dotenv@17.4.2: {} + + dunder-proto@1.0.1: + dependencies: + call-bind-apply-helpers: 1.0.2 + es-errors: 1.3.0 + gopd: 1.2.0 + + ecdsa-sig-formatter@1.0.11: + dependencies: + safe-buffer: 5.2.1 + + efrt@2.7.0: {} + + end-of-stream@1.4.5: + dependencies: + once: 1.4.0 + + es-define-property@1.0.1: {} + + es-errors@1.3.0: {} + es-module-lexer@1.7.0: {} + es-object-atoms@1.1.2: + dependencies: + es-errors: 1.3.0 + + es-set-tostringtag@2.1.0: + dependencies: + es-errors: 1.3.0 + get-intrinsic: 1.3.0 + has-tostringtag: 1.0.2 + hasown: 2.0.4 + esbuild@0.27.7: optionalDependencies: '@esbuild/aix-ppc64': 0.27.7 @@ -1759,51 +3595,393 @@ snapshots: '@esbuild/win32-ia32': 0.28.1 '@esbuild/win32-x64': 0.28.1 - estree-walker@3.0.3: + escape-string-regexp@2.0.0: {} + + estree-walker@3.0.3: + dependencies: + '@types/estree': 1.0.8 + + event-target-shim@5.0.1: {} + + eventemitter3@4.0.7: {} + + eventemitter3@5.0.1: {} + + events@3.3.0: {} + + expand-template@2.0.3: {} + + expect-type@1.3.0: {} + + expect@29.7.0: + dependencies: + '@jest/expect-utils': 29.7.0 + jest-get-type: 29.6.3 + jest-matcher-utils: 29.7.0 + jest-message-util: 29.7.0 + jest-util: 29.7.0 + + extend@3.0.2: {} + + fdir@6.5.0(picomatch@4.0.4): + optionalDependencies: + picomatch: 4.0.4 + + fetch-blob@3.2.0: + dependencies: + node-domexception: 1.0.0 + web-streams-polyfill: 3.3.3 + + file-uri-to-path@1.0.0: {} + + fill-range@7.1.1: + dependencies: + to-regex-range: 5.0.1 + + fix-dts-default-cjs-exports@1.0.1: + dependencies: + magic-string: 0.30.21 + mlly: 1.8.2 + rollup: 4.60.2 + + follow-redirects@1.16.0: {} + + form-data-encoder@1.7.2: {} + + form-data@4.0.6: + dependencies: + asynckit: 0.4.0 + combined-stream: 1.0.8 + es-set-tostringtag: 2.1.0 + hasown: 2.0.4 + mime-types: 2.1.35 + + formdata-node@4.4.1: + dependencies: + node-domexception: 1.0.0 + web-streams-polyfill: 4.0.0-beta.3 + + formdata-polyfill@4.0.10: + dependencies: + fetch-blob: 3.2.0 + + fs-constants@1.0.0: {} + + fsevents@2.3.3: + optional: true + + function-bind@1.1.2: {} + + gaxios@7.2.0: + dependencies: + extend: 3.0.2 + https-proxy-agent: 7.0.6 + node-fetch: 3.3.2 + transitivePeerDependencies: + - supports-color + + gcp-metadata@8.1.2: + dependencies: + gaxios: 7.2.0 + google-logging-utils: 1.1.3 + json-bigint: 1.0.0 + transitivePeerDependencies: + - supports-color + + generic-pool@3.9.0: {} + + get-intrinsic@1.3.0: + dependencies: + call-bind-apply-helpers: 1.0.2 + es-define-property: 1.0.1 + es-errors: 1.3.0 + es-object-atoms: 1.1.2 + function-bind: 1.1.2 + get-proto: 1.0.1 + gopd: 1.2.0 + has-symbols: 1.1.0 + hasown: 2.0.4 + math-intrinsics: 1.1.0 + + get-proto@1.0.1: + dependencies: + dunder-proto: 1.0.1 + es-object-atoms: 1.1.2 + + github-from-package@0.0.0: {} + + google-auth-library@10.9.0: + dependencies: + base64-js: 1.5.1 + ecdsa-sig-formatter: 1.0.11 + gaxios: 7.2.0 + gcp-metadata: 8.1.2 + google-logging-utils: 1.1.3 + jws: 4.0.1 + transitivePeerDependencies: + - supports-color + + google-logging-utils@1.1.3: {} + + gopd@1.2.0: {} + + graceful-fs@4.2.11: {} + + grad-school@0.0.5: {} + + groq-sdk@0.3.0: dependencies: - '@types/estree': 1.0.8 + '@types/node': 18.19.130 + '@types/node-fetch': 2.6.13 + abort-controller: 3.0.0 + agentkeepalive: 4.6.0 + digest-fetch: 1.3.0 + form-data-encoder: 1.7.2 + formdata-node: 4.4.1 + node-fetch: 2.7.0 + web-streams-polyfill: 3.3.3 + transitivePeerDependencies: + - encoding - eventemitter3@5.0.1: {} + has-flag@4.0.0: {} - expect-type@1.3.0: {} + has-symbols@1.1.0: {} - fdir@6.5.0(picomatch@4.0.4): - optionalDependencies: - picomatch: 4.0.4 + has-tostringtag@1.0.2: + dependencies: + has-symbols: 1.1.0 - fix-dts-default-cjs-exports@1.0.1: + hasown@2.0.4: dependencies: - magic-string: 0.30.21 - mlly: 1.8.2 - rollup: 4.60.2 + function-bind: 1.1.2 - fsevents@2.3.3: - optional: true + hono@4.12.30: {} - graceful-fs@4.2.11: {} + http-proxy-agent@7.0.2: + dependencies: + agent-base: 7.1.4 + debug: 4.4.3 + transitivePeerDependencies: + - supports-color - hono@4.12.30: {} + https-proxy-agent@5.0.1: + dependencies: + agent-base: 6.0.2 + debug: 4.4.3 + transitivePeerDependencies: + - supports-color + + https-proxy-agent@7.0.6: + dependencies: + agent-base: 7.1.4 + debug: 4.4.3 + transitivePeerDependencies: + - supports-color + + humanize-ms@1.2.1: + dependencies: + ms: 2.1.3 + + iceberg-js@0.8.1: {} + + ieee754@1.2.1: {} + + inherits@2.0.4: {} + + ini@1.3.8: {} + + is-buffer@1.1.6: {} + + is-docker@3.0.0: {} + + is-inside-container@1.0.0: + dependencies: + is-docker: 3.0.0 + + is-number@7.0.0: {} + + is-wsl@3.1.1: + dependencies: + is-inside-container: 1.0.0 isows@1.0.7(ws@8.21.0): dependencies: ws: 8.21.0 + jest-diff@29.7.0: + dependencies: + chalk: 4.1.2 + diff-sequences: 29.6.3 + jest-get-type: 29.6.3 + pretty-format: 29.7.0 + + jest-get-type@29.6.3: {} + + jest-matcher-utils@29.7.0: + dependencies: + chalk: 4.1.2 + jest-diff: 29.7.0 + jest-get-type: 29.6.3 + pretty-format: 29.7.0 + + jest-message-util@29.7.0: + dependencies: + '@babel/code-frame': 7.29.7 + '@jest/types': 29.6.3 + '@types/stack-utils': 2.0.3 + chalk: 4.1.2 + graceful-fs: 4.2.11 + micromatch: 4.0.8 + pretty-format: 29.7.0 + slash: 3.0.0 + stack-utils: 2.0.6 + + jest-util@29.7.0: + dependencies: + '@jest/types': 29.6.3 + '@types/node': 25.6.0 + chalk: 4.1.2 + ci-info: 3.9.0 + graceful-fs: 4.2.11 + picomatch: 2.3.2 + joycon@3.1.1: {} + js-tiktoken@1.0.21: + dependencies: + base64-js: 1.5.1 + + js-tokens@4.0.0: {} + js-tokens@9.0.1: {} + json-bigint@1.0.0: + dependencies: + bignumber.js: 9.3.1 + + jsonwebtoken@9.0.3: + dependencies: + jws: 4.0.1 + lodash.includes: 4.3.0 + lodash.isboolean: 3.0.3 + lodash.isinteger: 4.0.4 + lodash.isnumber: 3.0.3 + lodash.isplainobject: 4.0.6 + lodash.isstring: 4.0.1 + lodash.once: 4.1.1 + ms: 2.1.3 + semver: 7.8.5 + + jwa@2.0.1: + dependencies: + buffer-equal-constant-time: 1.0.1 + ecdsa-sig-formatter: 1.0.11 + safe-buffer: 5.2.1 + + jws@4.0.1: + dependencies: + jwa: 2.0.1 + safe-buffer: 5.2.1 + + kareem@3.3.0: {} + + langsmith@0.8.3(@opentelemetry/api@1.9.1)(ws@8.21.0): + dependencies: + p-queue: 6.6.2 + optionalDependencies: + '@opentelemetry/api': 1.9.1 + ws: 8.21.0 + lilconfig@3.1.3: {} lines-and-columns@1.2.4: {} load-tsconfig@0.2.5: {} + lodash.includes@4.3.0: {} + + lodash.isboolean@3.0.3: {} + + lodash.isinteger@4.0.4: {} + + lodash.isnumber@3.0.3: {} + + lodash.isplainobject@4.0.6: {} + + lodash.isstring@4.0.1: {} + + lodash.once@4.1.1: {} + + long@5.3.2: {} + loupe@3.2.1: {} magic-string@0.30.21: dependencies: '@jridgewell/sourcemap-codec': 1.5.5 + math-intrinsics@1.1.0: {} + + md5@2.3.0: + dependencies: + charenc: 0.0.2 + crypt: 0.0.2 + is-buffer: 1.1.6 + + mem0ai@3.1.0(@anthropic-ai/sdk@0.40.1)(@azure/identity@4.13.1)(@azure/search-documents@12.2.0)(@cloudflare/workers-types@4.20260702.1)(@google/genai@1.52.0)(@langchain/core@1.2.3(@opentelemetry/api@1.9.1)(ws@8.21.0))(@mistralai/mistralai@1.15.1)(@qdrant/js-client-rest@1.18.0(typescript@5.9.3))(@supabase/supabase-js@2.110.7)(@types/jest@29.5.14)(@types/pg@8.11.0)(better-sqlite3@12.11.1)(cloudflare@4.5.0)(compromise@14.16.0)(groq-sdk@0.3.0)(mongodb@7.2.0)(natural@8.1.1(@opentelemetry/api@1.9.1))(ollama@0.5.18)(pg@8.11.3)(redis@4.7.1)(ws@8.21.0): + dependencies: + '@anthropic-ai/sdk': 0.40.1 + '@azure/identity': 4.13.1 + '@azure/search-documents': 12.2.0 + '@cloudflare/workers-types': 4.20260702.1 + '@google/genai': 1.52.0 + '@langchain/core': 1.2.3(@opentelemetry/api@1.9.1)(ws@8.21.0) + '@mistralai/mistralai': 1.15.1 + '@qdrant/js-client-rest': 1.18.0(typescript@5.9.3) + '@supabase/supabase-js': 2.110.7 + '@types/jest': 29.5.14 + '@types/pg': 8.11.0 + axios: 1.18.1 + better-sqlite3: 12.11.1 + cloudflare: 4.5.0 + compromise: 14.16.0 + groq-sdk: 0.3.0 + natural: 8.1.1(@opentelemetry/api@1.9.1) + ollama: 0.5.18 + openai: 4.104.0(ws@8.21.0)(zod@3.25.76) + pg: 8.11.3 + redis: 4.7.1 + uuid: 11.1.1 + zod: 3.25.76 + optionalDependencies: + mongodb: 7.2.0 + transitivePeerDependencies: + - debug + - encoding + - supports-color + - ws + + memjs@1.3.2: {} + + memory-pager@1.5.0: {} + + micromatch@4.0.8: + dependencies: + braces: 3.0.3 + picomatch: 2.3.2 + + mime-db@1.52.0: {} + + mime-types@2.1.35: + dependencies: + mime-db: 1.52.0 + + mimic-response@3.1.0: {} + + minimist@1.2.8: {} + + mkdirp-classic@0.5.3: {} + mlly@1.8.2: dependencies: acorn: 8.16.0 @@ -1811,8 +3989,43 @@ snapshots: pkg-types: 1.3.1 ufo: 1.6.4 + mongodb-connection-string-url@7.0.2: + dependencies: + '@types/whatwg-url': 13.0.0 + whatwg-url: 14.2.0 + + mongodb@7.2.0: + dependencies: + '@mongodb-js/saslprep': 1.4.12 + bson: 7.3.1 + mongodb-connection-string-url: 7.0.2 + + mongoose@9.7.4: + dependencies: + '@standard-schema/spec': 1.1.0 + kareem: 3.3.0 + mongodb: 7.2.0 + mpath: 0.9.0 + mquery: 6.0.0 + ms: 2.1.3 + sift: 17.1.3 + transitivePeerDependencies: + - '@aws-sdk/credential-providers' + - '@mongodb-js/zstd' + - gcp-metadata + - kerberos + - mongodb-client-encryption + - snappy + - socks + + mpath@0.9.0: {} + + mquery@6.0.0: {} + ms@2.1.3: {} + mustache@4.2.0: {} + mz@2.7.0: dependencies: any-promise: 1.3.0 @@ -1821,8 +4034,86 @@ snapshots: nanoid@3.3.12: {} + napi-build-utils@2.0.0: {} + + natural@8.1.1(@opentelemetry/api@1.9.1): + dependencies: + afinn-165: 2.0.2 + afinn-165-financialmarketnews: 3.0.0 + apparatus: 0.0.10 + dotenv: 17.4.2 + memjs: 1.3.2 + mongoose: 9.7.4 + pg: 8.22.0 + redis: 5.12.1(@opentelemetry/api@1.9.1) + safe-stable-stringify: 2.5.0 + stopwords-iso: 1.1.0 + sylvester: 0.0.21 + underscore: 1.13.8 + uuid: 13.0.2 + wordnet-db: 3.1.14 + transitivePeerDependencies: + - '@aws-sdk/credential-providers' + - '@mongodb-js/zstd' + - '@node-rs/xxhash' + - '@opentelemetry/api' + - gcp-metadata + - kerberos + - mongodb-client-encryption + - pg-native + - snappy + - socks + + node-abi@3.94.0: + dependencies: + semver: 7.8.5 + + node-domexception@1.0.0: {} + + node-fetch@2.7.0: + dependencies: + whatwg-url: 5.0.0 + + node-fetch@3.3.2: + dependencies: + data-uri-to-buffer: 4.0.1 + fetch-blob: 3.2.0 + formdata-polyfill: 4.0.10 + object-assign@4.1.1: {} + obuf@1.1.2: {} + + ollama@0.5.18: + dependencies: + whatwg-fetch: 3.6.20 + + once@1.4.0: + dependencies: + wrappy: 1.0.2 + + open@10.2.0: + dependencies: + default-browser: 5.5.0 + define-lazy-prop: 3.0.0 + is-inside-container: 1.0.0 + wsl-utils: 0.1.0 + + openai@4.104.0(ws@8.21.0)(zod@3.25.76): + dependencies: + '@types/node': 18.19.130 + '@types/node-fetch': 2.6.13 + abort-controller: 3.0.0 + agentkeepalive: 4.6.0 + form-data-encoder: 1.7.2 + formdata-node: 4.4.1 + node-fetch: 2.7.0 + optionalDependencies: + ws: 8.21.0 + zod: 3.25.76 + transitivePeerDependencies: + - encoding + openapi3-ts@4.5.0: dependencies: yaml: 2.8.4 @@ -1842,12 +4133,97 @@ snapshots: transitivePeerDependencies: - zod + p-finally@1.0.0: {} + + p-queue@6.6.2: + dependencies: + eventemitter3: 4.0.7 + p-timeout: 3.2.0 + + p-retry@4.6.2: + dependencies: + '@types/retry': 0.12.0 + retry: 0.13.1 + + p-timeout@3.2.0: + dependencies: + p-finally: 1.0.0 + + packet-reader@1.0.0: {} + pathe@2.0.3: {} pathval@2.0.1: {} + pg-cloudflare@1.4.0: + optional: true + + pg-connection-string@2.14.0: {} + + pg-int8@1.0.1: {} + + pg-numeric@1.0.2: {} + + pg-pool@3.14.0(pg@8.11.3): + dependencies: + pg: 8.11.3 + + pg-pool@3.14.0(pg@8.22.0): + dependencies: + pg: 8.22.0 + + pg-protocol@1.15.0: {} + + pg-types@2.2.0: + dependencies: + pg-int8: 1.0.1 + postgres-array: 2.0.0 + postgres-bytea: 1.0.1 + postgres-date: 1.0.7 + postgres-interval: 1.2.0 + + pg-types@4.1.0: + dependencies: + pg-int8: 1.0.1 + pg-numeric: 1.0.2 + postgres-array: 3.0.4 + postgres-bytea: 3.0.0 + postgres-date: 2.1.0 + postgres-interval: 3.0.0 + postgres-range: 1.1.4 + + pg@8.11.3: + dependencies: + buffer-writer: 2.0.0 + packet-reader: 1.0.0 + pg-connection-string: 2.14.0 + pg-pool: 3.14.0(pg@8.11.3) + pg-protocol: 1.15.0 + pg-types: 2.2.0 + pgpass: 1.0.6 + optionalDependencies: + pg-cloudflare: 1.4.0 + + pg@8.22.0: + dependencies: + pg-connection-string: 2.14.0 + pg-pool: 3.14.0(pg@8.22.0) + pg-protocol: 1.15.0 + pg-types: 2.2.0 + pgpass: 1.0.5 + optionalDependencies: + pg-cloudflare: 1.4.0 + + pgpass@1.0.5: + dependencies: + split2: 4.2.0 + + pgpass@1.0.6: {} + picocolors@1.1.1: {} + picomatch@2.3.2: {} + picomatch@4.0.4: {} pirates@4.0.7: {} @@ -1872,18 +4248,121 @@ snapshots: picocolors: 1.1.1 source-map-js: 1.2.1 + postgres-array@2.0.0: {} + + postgres-array@3.0.4: {} + + postgres-bytea@1.0.1: {} + + postgres-bytea@3.0.0: + dependencies: + obuf: 1.1.2 + + postgres-date@1.0.7: {} + + postgres-date@2.1.0: {} + + postgres-interval@1.2.0: + dependencies: + xtend: 4.0.2 + + postgres-interval@3.0.0: {} + + postgres-range@1.1.4: {} + + prebuild-install@7.1.3: + dependencies: + detect-libc: 2.1.2 + expand-template: 2.0.3 + github-from-package: 0.0.0 + minimist: 1.2.8 + mkdirp-classic: 0.5.3 + napi-build-utils: 2.0.0 + node-abi: 3.94.0 + pump: 3.0.4 + rc: 1.2.8 + simple-get: 4.0.1 + tar-fs: 2.1.5 + tunnel-agent: 0.6.0 + + pretty-format@29.7.0: + dependencies: + '@jest/schemas': 29.6.3 + ansi-styles: 5.2.0 + react-is: 18.3.1 + proper-lockfile@4.1.2: dependencies: graceful-fs: 4.2.11 retry: 0.12.0 signal-exit: 3.0.7 + protobufjs@7.6.5: + dependencies: + '@protobufjs/aspromise': 1.1.2 + '@protobufjs/base64': 1.1.2 + '@protobufjs/codegen': 2.0.5 + '@protobufjs/eventemitter': 1.1.1 + '@protobufjs/fetch': 1.1.1 + '@protobufjs/float': 1.0.2 + '@protobufjs/path': 1.1.2 + '@protobufjs/pool': 1.1.0 + '@protobufjs/utf8': 1.1.2 + '@types/node': 25.6.0 + long: 5.3.2 + + proxy-from-env@2.1.0: {} + + pump@3.0.4: + dependencies: + end-of-stream: 1.4.5 + once: 1.4.0 + + punycode@2.3.1: {} + + rc@1.2.8: + dependencies: + deep-extend: 0.6.0 + ini: 1.3.8 + minimist: 1.2.8 + strip-json-comments: 2.0.1 + + react-is@18.3.1: {} + + readable-stream@3.6.2: + dependencies: + inherits: 2.0.4 + string_decoder: 1.3.0 + util-deprecate: 1.0.2 + readdirp@4.1.2: {} + redis@4.7.1: + dependencies: + '@redis/bloom': 1.2.0(@redis/client@1.6.1) + '@redis/client': 1.6.1 + '@redis/graph': 1.1.1(@redis/client@1.6.1) + '@redis/json': 1.0.7(@redis/client@1.6.1) + '@redis/search': 1.2.0(@redis/client@1.6.1) + '@redis/time-series': 1.1.0(@redis/client@1.6.1) + + redis@5.12.1(@opentelemetry/api@1.9.1): + dependencies: + '@redis/bloom': 5.12.1(@redis/client@5.12.1(@opentelemetry/api@1.9.1)) + '@redis/client': 5.12.1(@opentelemetry/api@1.9.1) + '@redis/json': 5.12.1(@redis/client@5.12.1(@opentelemetry/api@1.9.1)) + '@redis/search': 5.12.1(@redis/client@5.12.1(@opentelemetry/api@1.9.1)) + '@redis/time-series': 5.12.1(@redis/client@5.12.1(@opentelemetry/api@1.9.1)) + transitivePeerDependencies: + - '@node-rs/xxhash' + - '@opentelemetry/api' + resolve-from@5.0.0: {} retry@0.12.0: {} + retry@0.13.1: {} + rollup@4.60.2: dependencies: '@types/estree': 1.0.8 @@ -1915,18 +4394,56 @@ snapshots: '@rollup/rollup-win32-x64-msvc': 4.60.2 fsevents: 2.3.3 + run-applescript@7.1.0: {} + + safe-buffer@5.2.1: {} + + safe-stable-stringify@2.5.0: {} + + semver@7.8.5: {} + + sift@17.1.3: {} + siginfo@2.0.0: {} signal-exit@3.0.7: {} + simple-concat@1.0.1: {} + + simple-get@4.0.1: + dependencies: + decompress-response: 6.0.0 + once: 1.4.0 + simple-concat: 1.0.1 + + slash@3.0.0: {} + source-map-js@1.2.1: {} source-map@0.7.6: {} + sparse-bitfield@3.0.3: + dependencies: + memory-pager: 1.5.0 + + split2@4.2.0: {} + + stack-utils@2.0.6: + dependencies: + escape-string-regexp: 2.0.0 + stackback@0.0.2: {} std-env@3.10.0: {} + stopwords-iso@1.1.0: {} + + string_decoder@1.3.0: + dependencies: + safe-buffer: 5.2.1 + + strip-json-comments@2.0.1: {} + strip-literal@3.1.0: dependencies: js-tokens: 9.0.1 @@ -1941,6 +4458,29 @@ snapshots: tinyglobby: 0.2.16 ts-interface-checker: 0.1.13 + suffix-thumb@5.0.2: {} + + supports-color@7.2.0: + dependencies: + has-flag: 4.0.0 + + sylvester@0.0.21: {} + + tar-fs@2.1.5: + dependencies: + chownr: 1.1.4 + mkdirp-classic: 0.5.3 + pump: 3.0.4 + tar-stream: 2.2.0 + + tar-stream@2.2.0: + dependencies: + bl: 4.1.0 + end-of-stream: 1.4.5 + fs-constants: 1.0.0 + inherits: 2.0.4 + readable-stream: 3.6.2 + thenify-all@1.6.0: dependencies: thenify: 3.3.1 @@ -1964,10 +4504,22 @@ snapshots: tinyspy@4.0.4: {} + to-regex-range@5.0.1: + dependencies: + is-number: 7.0.0 + + tr46@0.0.3: {} + + tr46@5.1.1: + dependencies: + punycode: 2.3.1 + tree-kill@1.2.2: {} ts-interface-checker@0.1.13: {} + tslib@2.8.1: {} + tsup@8.5.1(postcss@8.5.13)(tsx@4.22.4)(typescript@5.9.3)(yaml@2.8.4): dependencies: bundle-require: 5.1.0(esbuild@0.27.7) @@ -2002,12 +4554,28 @@ snapshots: optionalDependencies: fsevents: 2.3.3 + tunnel-agent@0.6.0: + dependencies: + safe-buffer: 5.2.1 + typescript@5.9.3: {} ufo@1.6.4: {} + underscore@1.13.8: {} + + undici-types@5.26.5: {} + undici-types@7.19.2: {} + undici@6.27.0: {} + + util-deprecate@1.0.2: {} + + uuid@11.1.1: {} + + uuid@13.0.2: {} + viem@2.48.8(typescript@5.9.3)(zod@4.4.3): dependencies: '@noble/curves': 1.9.1 @@ -2101,15 +4669,53 @@ snapshots: - tsx - yaml + web-streams-polyfill@3.3.3: {} + + web-streams-polyfill@4.0.0-beta.3: {} + + webidl-conversions@3.0.1: {} + + webidl-conversions@7.0.0: {} + + whatwg-fetch@3.6.20: {} + + whatwg-url@14.2.0: + dependencies: + tr46: 5.1.1 + webidl-conversions: 7.0.0 + + whatwg-url@5.0.0: + dependencies: + tr46: 0.0.3 + webidl-conversions: 3.0.1 + why-is-node-running@2.3.0: dependencies: siginfo: 2.0.0 stackback: 0.0.2 + wordnet-db@3.1.14: {} + + wrappy@1.0.2: {} + ws@8.21.0: {} + wsl-utils@0.1.0: + dependencies: + is-wsl: 3.1.1 + + xtend@4.0.2: {} + + yallist@4.0.0: {} + yaml@2.8.4: {} + zod-to-json-schema@3.25.2(zod@4.4.3): + dependencies: + zod: 4.4.3 + + zod@3.25.76: {} + zod@4.4.2: {} zod@4.4.3: {} diff --git a/src/benchmarks/index.ts b/src/benchmarks/index.ts index c64832e..d324e9c 100644 --- a/src/benchmarks/index.ts +++ b/src/benchmarks/index.ts @@ -925,48 +925,81 @@ export function createMemoryAdapterBenchmarkResponder(options: { export async function runMemoryAdapterBenchmark( options: RunMemoryAdapterBenchmarkOptions, ): Promise { + if (options.candidates.length === 0) + throw new Error('memory adapter benchmark requires candidates') + assertUniqueNonEmptyStrings( + options.candidates.map((candidate) => candidate.id), + 'memory adapter candidate id', + ) + for (const candidate of options.candidates) { + if (!/^[A-Za-z0-9][A-Za-z0-9._-]*$/.test(candidate.id)) { + throw new Error( + `memory adapter candidate id '${candidate.id}' must be a safe directory segment`, + ) + } + } const storage = options.storage ?? fsCampaignStorage() const rows: MemoryAdapterBenchmarkRankingRow[] = [] for (const candidate of options.candidates) { const adapter = await candidate.createAdapter() - const run = await runKnowledgeBenchmarkSuite({ - cases: options.cases, - respond: createMemoryAdapterBenchmarkResponder({ - adapter, - candidateId: candidate.id, - searchLimit: candidate.searchLimit, - scope: candidate.scope, - costUsdPerCase: candidate.costUsdPerCase, + let primaryError: unknown + try { + const run = await runKnowledgeBenchmarkSuite({ + cases: options.cases, + respond: createMemoryAdapterBenchmarkResponder({ + adapter, + candidateId: candidate.id, + searchLimit: candidate.searchLimit, + scope: candidate.scope, + costUsdPerCase: candidate.costUsdPerCase, + now: options.now, + }), + runDir: join(options.runDir, candidate.id), + storage, + repo: options.repo, + seed: options.seed, + reps: options.reps, + resumable: options.resumable, + costCeiling: options.costCeiling, + maxConcurrency: options.maxConcurrency, + dispatchTimeoutMs: options.dispatchTimeoutMs, + expectUsage: options.expectUsage ?? 'off', now: options.now, - }), - runDir: join(options.runDir, candidate.id), - storage, - repo: options.repo, - seed: options.seed, - reps: options.reps, - resumable: options.resumable, - costCeiling: options.costCeiling, - maxConcurrency: options.maxConcurrency, - dispatchTimeoutMs: options.dispatchTimeoutMs, - expectUsage: options.expectUsage ?? 'off', - now: options.now, - }) - rows.push({ - rank: 0, - candidateId: candidate.id, - label: candidate.label ?? candidate.id, - adapterId: adapter.id, - scoreMean: run.report.score.mean, - passRate: run.report.dimensions.passed?.mean ?? 0, - totalCases: run.report.totalCases, - totalCells: run.report.totalCells, - cellsFailed: run.report.cellsFailed, - totalCostUsd: run.report.totalCostUsd, - reportJsonPath: run.reportJsonPath, - reportMarkdownPath: run.reportMarkdownPath, - report: run.report, - }) - await adapter.flush?.() + }) + rows.push({ + rank: 0, + candidateId: candidate.id, + label: candidate.label ?? candidate.id, + adapterId: adapter.id, + scoreMean: run.report.score.mean, + passRate: run.report.dimensions.passed?.mean ?? 0, + totalCases: run.report.totalCases, + totalCells: run.report.totalCells, + cellsFailed: run.report.cellsFailed, + totalCostUsd: run.report.totalCostUsd, + reportJsonPath: run.reportJsonPath, + reportMarkdownPath: run.reportMarkdownPath, + report: run.report, + }) + } catch (error) { + primaryError = error + } + const cleanupErrors: unknown[] = [] + try { + await adapter.flush?.() + } catch (error) { + cleanupErrors.push(error) + } + try { + await adapter.close?.() + } catch (error) { + cleanupErrors.push(error) + } + if (primaryError || cleanupErrors.length > 0) { + const errors = [...(primaryError ? [primaryError] : []), ...cleanupErrors] + if (errors.length === 1) throw errors[0] + throw new AggregateError(errors, `${candidate.id}: memory adapter benchmark cleanup failed`) + } } const ranked = rows @@ -986,6 +1019,7 @@ export async function runMemoryAdapterBenchmark( export function createNoopMemoryBenchmarkAdapter(id = 'no-memory'): AgentMemoryAdapter { return { id, + branchIsolation: { mode: 'scoped' }, async search() { return [] }, @@ -1000,6 +1034,7 @@ export function createNoopMemoryBenchmarkAdapter(id = 'no-memory'): AgentMemoryA kind: input.kind, } }, + async clear() {}, async flush() {}, } } @@ -1014,6 +1049,7 @@ export function createInMemoryBenchmarkAdapter(options: { id?: string } = {}): A let seq = 0 const adapter: AgentMemoryAdapter = { id, + branchIsolation: { mode: 'scoped' }, async search(query, searchOptions = {}) { const scored = rows .filter((row) => memoryScopeMatches(row.input.scope, searchOptions.scope)) @@ -1081,10 +1117,12 @@ export function createInMemoryBenchmarkAdapter(options: { id?: string } = {}): A metadata: hit.metadata, } }, - async flush() { - rows.length = 0 - seq = 0 + async clear(scope) { + for (let index = rows.length - 1; index >= 0; index -= 1) { + if (memoryScopeMatches(rows[index]!.input.scope, scope)) rows.splice(index, 1) + } }, + async flush() {}, } return adapter } @@ -1164,6 +1202,7 @@ export function buildRetrievalBenchmarkCasesFromQrels( export async function runKnowledgeBenchmarkSuite( options: RunKnowledgeBenchmarkSuiteOptions, ): Promise> { + assertKnowledgeBenchmarkCases(options.cases) const storage = options.storage ?? fsCampaignStorage() const scenarios = buildKnowledgeBenchmarkScenarios(options.cases, options.splits) const dispatch: RunCampaignOptions['dispatch'] = async ( @@ -1409,7 +1448,7 @@ export function scoreMemoryBenchmarkArtifact( required.totalWeight > 0 ? required.recall : undefined, testCase.expectedEventIds && testCase.expectedEventIds.length > 0 ? eventRecall : undefined, testCase.expectedActorIds && testCase.expectedActorIds.length > 0 ? actorRecall : undefined, - forbidden.safe, + testCase.forbiddenFacts && testCase.forbiddenFacts.length > 0 ? forbidden.safe : undefined, ].filter((value): value is number => value !== undefined) const score = mean(components) return { @@ -1654,6 +1693,9 @@ function memoryScopeMatches(stored?: AgentMemoryScope, requested?: AgentMemorySc if (!requested) return true if (requested.tenantId !== undefined && stored?.tenantId !== requested.tenantId) return false if (requested.userId !== undefined && stored?.userId !== requested.userId) return false + if (requested.agentId !== undefined && stored?.agentId !== requested.agentId) return false + if (requested.teamId !== undefined && stored?.teamId !== requested.teamId) return false + if (requested.runId !== undefined && stored?.runId !== requested.runId) return false if (requested.sessionId !== undefined && stored?.sessionId !== requested.sessionId) return false if (requested.namespace !== undefined && stored?.namespace !== requested.namespace) return false for (const [key, value] of Object.entries(requested.tags ?? {})) { @@ -1718,6 +1760,74 @@ export function isKnowledgeMemoryBenchmarkCase( return testCase.taskKind.startsWith('memory-') } +function assertKnowledgeBenchmarkCases(cases: readonly KnowledgeBenchmarkCase[]): void { + if (cases.length === 0) throw new Error('knowledge benchmark requires cases') + assertUniqueNonEmptyStrings( + cases.map((testCase) => testCase.id), + 'knowledge benchmark case id', + ) + for (const testCase of cases) { + if (typeof testCase.family !== 'string' || !testCase.family.trim()) { + throw new Error(`knowledge benchmark case ${testCase.id} requires a family`) + } + if (testCase.taskKind === 'retrieval') continue + if (isKnowledgeMemoryBenchmarkCase(testCase)) { + assertUniqueNonEmptyStrings( + testCase.events.map((event) => event.id), + `${testCase.id} memory event id`, + ) + for (const event of testCase.events) { + assertNonEmptyBenchmarkString(event.text, `${testCase.id} memory event ${event.id} text`) + } + assertClaimMatchers(testCase.requiredFacts ?? [], `${testCase.id} requiredFacts`) + assertClaimMatchers(testCase.forbiddenFacts ?? [], `${testCase.id} forbiddenFacts`) + assertUniqueNonEmptyStrings( + testCase.expectedEventIds ?? [], + `${testCase.id} expected event id`, + ) + assertUniqueNonEmptyStrings( + testCase.expectedActorIds ?? [], + `${testCase.id} expected actor id`, + ) + } else { + assertClaimMatchers(testCase.requiredClaims ?? [], `${testCase.id} requiredClaims`) + assertClaimMatchers(testCase.forbiddenClaims ?? [], `${testCase.id} forbiddenClaims`) + assertUniqueNonEmptyStrings(testCase.expectedSourceIds ?? [], `${testCase.id} source id`) + } + } +} + +function assertClaimMatchers(claims: readonly KnowledgeClaimMatcher[], label: string): void { + assertUniqueNonEmptyStrings( + claims.map((claim) => claim.id), + `${label} matcher id`, + ) + for (const claim of claims) { + if (claim.anyOf.length === 0) throw new Error(`${label} matcher ${claim.id} requires anyOf`) + assertUniqueNonEmptyStrings(claim.anyOf, `${label} matcher ${claim.id} anyOf`) + if (claim.weight !== undefined && (!Number.isFinite(claim.weight) || claim.weight <= 0)) { + throw new Error(`${label} matcher ${claim.id} weight must be a positive finite number`) + } + const sourceEventIds = (claim as Partial).sourceEventIds + if (sourceEventIds !== undefined) { + assertUniqueNonEmptyStrings(sourceEventIds, `${label} matcher ${claim.id} source event id`) + } + } +} + +function assertUniqueNonEmptyStrings(values: readonly string[], label: string): void { + const seen = new Set() + for (const value of values) { + assertNonEmptyBenchmarkString(value, label) + if (seen.has(value)) throw new Error(`duplicate ${label}: ${value}`) + seen.add(value) + } +} + +function assertNonEmptyBenchmarkString(value: unknown, label: string): asserts value is string { + if (typeof value !== 'string' || !value.trim()) throw new Error(`${label} must be non-empty`) +} + function scoreClaims(text: string, claims: readonly KnowledgeClaimMatcher[]) { let matched = 0 let matchedWeight = 0 @@ -1725,7 +1835,19 @@ function scoreClaims(text: string, claims: readonly KnowledgeClaimMatcher[]) { const matchedIds: string[] = [] const haystack = text.toLowerCase() for (const claim of claims) { + if ( + !claim.id.trim() || + claim.anyOf.length === 0 || + claim.anyOf.some((value) => !value.trim()) + ) { + throw new Error( + 'claim matchers require a non-empty id and at least one non-empty alternative', + ) + } const weight = claim.weight ?? 1 + if (!Number.isFinite(weight) || weight <= 0) { + throw new Error(`claim matcher ${claim.id} weight must be a positive finite number`) + } totalWeight += weight if (claim.anyOf.some((fragment) => haystack.includes(fragment.toLowerCase()))) { matched += 1 diff --git a/src/memory/branch.ts b/src/memory/branch.ts new file mode 100644 index 0000000..7bede10 --- /dev/null +++ b/src/memory/branch.ts @@ -0,0 +1,707 @@ +import { canonicalJson } from '@tangle-network/agent-eval' +import { sha256, stableId } from '../ids' +import { defaultGetMemoryContext } from './adapter' +import type { + AgentMemoryAdapter, + AgentMemoryHit, + AgentMemoryScope, + AgentMemoryWriteInput, + AgentMemoryWriteResult, +} from './types' + +export type AgentMemoryVisibility = 'private' | 'team' | 'shared' + +export interface AgentMemorySharingPolicy { + read: readonly AgentMemoryVisibility[] + write: AgentMemoryVisibility +} + +export interface AgentMemoryJournalEntry { + sequence: number + input: AgentMemoryWriteInput + result: AgentMemoryWriteResult +} + +export interface AgentMemoryBranchSnapshot { + branchId: string + parentBranchId?: string + adapterId: string + policy: AgentMemorySharingPolicy + baseScope: AgentMemoryScope + journal: readonly AgentMemoryJournalEntry[] + digest: `sha256:${string}` +} + +export interface AgentMemoryBranch extends AgentMemoryAdapter { + readonly branchId: string + readonly parentBranchId?: string + readonly policy: AgentMemorySharingPolicy + snapshot(): Promise + fork(options: { + branchId: string + adapter?: AgentMemoryAdapter + policy?: AgentMemorySharingPolicy + baseScope?: AgentMemoryScope + }): Promise +} + +export interface CreateAgentMemoryBranchOptions { + adapter: AgentMemoryAdapter + branchId: string + parentBranchId?: string + policy?: AgentMemorySharingPolicy + baseScope?: AgentMemoryScope + /** Rebind a persisted branch whose external memory is still present. */ + snapshot?: AgentMemoryBranchSnapshot + /** Optional exact logical scopes allowed to write, used by crash-safe experiments. */ + allowedWriteScopes?: readonly AgentMemoryScope[] +} + +const DEFAULT_POLICY: AgentMemorySharingPolicy = { + read: ['private'], + write: 'private', +} + +interface AdapterLeaseState { + references: number + closed: boolean + closePromise?: Promise +} + +const adapterLeases = new WeakMap() + +/** + * Isolates one candidate branch and records accepted writes for durable resume + * or replay into a child branch, including a child backed by another provider. + */ +export function createAgentMemoryBranch( + options: CreateAgentMemoryBranchOptions, +): AgentMemoryBranch { + if (typeof options.branchId !== 'string' || !options.branchId.trim()) { + throw new Error('memory branchId must be a non-empty string') + } + const snapshot = options.snapshot + if (snapshot && snapshot.branchId !== options.branchId) { + throw new Error( + `memory branch snapshot belongs to ${snapshot.branchId}, not ${options.branchId}`, + ) + } + if (snapshot && snapshot.adapterId !== options.adapter.id) { + throw new Error( + `memory branch snapshot uses ${snapshot.adapterId}, not ${options.adapter.id}; fork it to replay into another adapter`, + ) + } + if (snapshot) { + assertDurableValue(snapshot, `${options.branchId}: branch snapshot`) + assertSnapshotDigest(snapshot) + } + if ( + snapshot && + options.policy && + canonicalJson(stripUndefined(options.policy)) !== canonicalJson(stripUndefined(snapshot.policy)) + ) { + throw new Error('memory branch snapshot policy cannot change during resume; fork it instead') + } + if ( + snapshot && + options.baseScope && + canonicalJson(stripUndefined(options.baseScope)) !== + canonicalJson(stripUndefined(snapshot.baseScope)) + ) { + throw new Error('memory branch snapshot scope cannot change during resume; fork it instead') + } + + const branchId = options.branchId + const parentBranchId = options.parentBranchId ?? snapshot?.parentBranchId + const policy = normalizePolicy(options.policy ?? snapshot?.policy ?? DEFAULT_POLICY) + assertBranchIsolation(options.adapter, branchId, policy) + const mergedBaseScope = mergeScopes(undefined, snapshot?.baseScope ?? options.baseScope) + assertDurableValue(mergedBaseScope, `${branchId}: base scope`) + const baseScope = cloneDurableValue(mergedBaseScope) + const allowedWriteScopeKeys = options.allowedWriteScopes + ? new Set( + options.allowedWriteScopes.map((scope) => + canonicalJson(stripUndefined(mergeScopes(baseScope, scope))), + ), + ) + : undefined + const journal = [...(snapshot?.journal ?? [])].map(cloneJournalEntry) + let nextSequence = journal.reduce((max, entry) => Math.max(max, entry.sequence + 1), 0) + const queues = new Map>() + const pending = new Set>() + const reads = new Set>() + const active = new Set>() + const touchedScopes = new Map() + for (const entry of journal) { + const scope = mergeScopes(baseScope, entry.input.scope) + touchedScopes.set(canonicalJson(stripUndefined(scope)), scope) + } + let mutationBarrier = Promise.resolve() + const releaseAdapter = retainAdapter(options.adapter) + let lifecycle: 'open' | 'closing' | 'closed' = 'open' + let closePromise: Promise | undefined + + const assertOpen = (operation: string): void => { + if (lifecycle !== 'open') throw new Error(`${branchId}: cannot ${operation} a closed branch`) + } + const track = (operation: Promise): Promise => { + active.add(operation) + void operation.then( + () => active.delete(operation), + () => active.delete(operation), + ) + return operation + } + const trackRead = (operation: Promise): Promise => { + reads.add(operation) + void operation.then( + () => reads.delete(operation), + () => reads.delete(operation), + ) + return track(operation) + } + const flushWrites = async (): Promise => { + const results = await Promise.allSettled([...pending]) + const failures = results.flatMap((result) => + result.status === 'rejected' ? [result.reason] : [], + ) + if (failures.length === 1) throw failures[0] + if (failures.length > 1) { + throw new AggregateError(failures, `${branchId}: pending memory writes failed`) + } + } + const flushAll = async (): Promise => { + await flushWrites() + await options.adapter.flush?.() + } + const runExclusive = async ( + operation: () => Promise, + exclusiveOptions: { failOnOperationError?: boolean; flushAdapter?: boolean } = {}, + ): Promise => { + const precedingBarrier = mutationBarrier + const precedingWrites = [...pending] + const precedingReads = [...reads] + let releaseBarrier!: () => void + const ownBarrier = new Promise((resolve) => { + releaseBarrier = resolve + }) + mutationBarrier = precedingBarrier.then(() => ownBarrier) + await precedingBarrier + const settled = await Promise.allSettled([...precedingWrites, ...precedingReads]) + const failures = settled.flatMap((result) => + result.status === 'rejected' ? [result.reason] : [], + ) + try { + if ((exclusiveOptions.failOnOperationError ?? true) && failures.length > 0) { + if (failures.length === 1) throw failures[0] + throw new AggregateError(failures, `${branchId}: pending memory operations failed`) + } + if (exclusiveOptions.flushAdapter ?? true) await options.adapter.flush?.() + return await operation() + } finally { + releaseBarrier() + } + } + + const branch: AgentMemoryBranch = { + id: `${options.adapter.id}:branch:${stableId('branch', branchId)}`, + branchId, + ...(parentBranchId !== undefined ? { parentBranchId } : {}), + policy, + async search(query, searchOptions = {}) { + assertOpen('search') + const precedingBarrier = mutationBarrier + return trackRead( + (async () => { + await precedingBarrier + const logicalScope = mergeScopes(baseScope, searchOptions.scope) + const scopes = policy.read.map((visibility) => + physicalScope(branchId, logicalScope, visibility), + ) + const groups = await Promise.all( + scopes.map(async (scope) => { + const hits = await options.adapter.search(query, { + ...searchOptions, + scope, + holdout: undefined, + }) + return hits.map((hit) => ({ + ...hit, + metadata: { + ...hit.metadata, + memoryBranchId: branchId, + memoryVisibility: scope.tags?.memoryVisibility, + }, + })) + }), + ) + return mergeHits(groups.flat(), searchOptions.limit) + })(), + ) + }, + async getContext(query, searchOptions = {}) { + assertOpen('read context from') + return track(defaultGetMemoryContext(branch, query, searchOptions)) + }, + async write(input) { + assertOpen('write to') + const sequence = nextSequence + nextSequence += 1 + assertDurableValue(input, `${branch.id}: write input`) + const logicalInput = cloneWriteInput(input) + const logicalScope = mergeScopes(baseScope, logicalInput.scope) + if ( + allowedWriteScopeKeys && + !allowedWriteScopeKeys.has(canonicalJson(stripUndefined(logicalScope))) + ) { + throw new Error( + `${branch.id}: write scope was not declared in the experiment sequence cleanupScopes`, + ) + } + const scope = physicalScope(branchId, logicalScope, policy.write) + const queueKey = writeQueueKey(branchId, logicalScope, policy.write) + touchedScopes.set(canonicalJson(stripUndefined(logicalScope)), logicalScope) + const previous = queues.get(queueKey) ?? Promise.resolve() + const precedingBarrier = mutationBarrier + let resultPromise: Promise + const queued = Promise.all([previous.catch(() => undefined), precedingBarrier]).then( + async () => { + const result = await options.adapter.write({ ...logicalInput, scope }) + if (result.accepted) { + assertDurableValue(result, `${branch.id}: write result`) + journal.push({ + sequence, + input: logicalInput, + result: cloneWriteResult(result), + }) + } + return result + }, + ) + resultPromise = queued + const tail = queued.then( + () => undefined, + () => undefined, + ) + queues.set(queueKey, tail) + pending.add(resultPromise) + void resultPromise.then( + () => { + pending.delete(resultPromise) + if (queues.get(queueKey) === tail) queues.delete(queueKey) + }, + () => { + pending.delete(resultPromise) + if (queues.get(queueKey) === tail) queues.delete(queueKey) + }, + ) + return track(resultPromise) + }, + async clear(scope) { + assertOpen('clear') + return track( + runExclusive( + async () => { + if (!options.adapter.clear) { + throw new Error(`${options.adapter.id}: adapter does not support scoped clear`) + } + const flushErrors: unknown[] = [] + try { + await options.adapter.flush?.() + } catch (error) { + flushErrors.push(error) + } + const logicalScopes = scope + ? [mergeScopes(baseScope, scope)] + : touchedScopes.size > 0 + ? [...touchedScopes.values()] + : [baseScope] + const visibilities = new Set([...policy.read, policy.write]) + const physicalScopes = new Map() + for (const logicalScope of logicalScopes) { + for (const visibility of visibilities) { + const physical = physicalScope(branchId, logicalScope, visibility) + physicalScopes.set(canonicalJson(stripUndefined(physical)), physical) + } + } + const clearErrors: unknown[] = [] + for (const physical of physicalScopes.values()) { + try { + await options.adapter.clear(physical) + } catch (error) { + clearErrors.push(error) + } + } + if (clearErrors.length === 0) { + if (!scope) { + journal.splice(0, journal.length) + touchedScopes.clear() + } else { + for (let index = journal.length - 1; index >= 0; index -= 1) { + if (scopeMatches(mergeScopes(baseScope, journal[index]!.input.scope), scope)) { + journal.splice(index, 1) + } + } + for (const [key, touched] of touchedScopes) { + if (scopeMatches(touched, scope)) touchedScopes.delete(key) + } + } + } + const failures = [...flushErrors, ...clearErrors] + if (failures.length === 1) throw failures[0] + if (failures.length > 1) { + throw new AggregateError(failures, `${branchId}: memory clear failed`) + } + }, + { failOnOperationError: false, flushAdapter: false }, + ), + ) + }, + async flush() { + assertOpen('flush') + return track(runExclusive(async () => undefined)) + }, + close() { + if (closePromise) return closePromise + lifecycle = 'closing' + closePromise = (async () => { + const errors: unknown[] = [] + const operations = await Promise.allSettled([...active]) + for (const result of operations) { + if (result.status === 'rejected') errors.push(result.reason) + } + try { + await flushAll() + } catch (error) { + errors.push(error) + } + try { + await releaseAdapter() + } catch (error) { + errors.push(error) + } + lifecycle = 'closed' + if (errors.length === 1) throw errors[0] + if (errors.length > 1) throw new AggregateError(errors, `${branch.id}: close failed`) + })() + return closePromise + }, + async snapshot() { + assertOpen('snapshot') + return track( + runExclusive(async () => { + return createSnapshot({ + branchId, + adapterId: options.adapter.id, + policy, + baseScope, + journal, + ...(parentBranchId !== undefined ? { parentBranchId } : {}), + }) + }), + ) + }, + async fork(forkOptions) { + assertOpen('fork') + if (forkOptions.branchId === branchId) { + throw new Error(`memory branch ${branchId}: child branchId must differ from its parent`) + } + return track( + (async () => { + const parent = await branch.snapshot() + const child = createAgentMemoryBranch({ + adapter: forkOptions.adapter ?? options.adapter, + branchId: forkOptions.branchId, + parentBranchId: branchId, + policy: forkOptions.policy ?? policy, + baseScope: mergeScopes(baseScope, forkOptions.baseScope), + }) + try { + for (const entry of parent.journal) { + const result = await child.write(entry.input) + if (!result.accepted) { + throw new Error( + `memory branch ${forkOptions.branchId}: adapter rejected replay of journal entry ${entry.sequence}`, + ) + } + } + await child.flush?.() + return child + } catch (error) { + const cleanupErrors: unknown[] = [] + try { + await child.clear?.() + } catch (cleanupError) { + cleanupErrors.push(cleanupError) + } + try { + await child.close?.() + } catch (cleanupError) { + cleanupErrors.push(cleanupError) + } + if (cleanupErrors.length > 0) { + throw new AggregateError( + [error, ...cleanupErrors], + `memory branch ${forkOptions.branchId}: replay and cleanup failed`, + ) + } + throw error + } + })(), + ) + }, + } + return branch +} + +function assertBranchIsolation( + adapter: AgentMemoryAdapter, + branchId: string, + policy: AgentMemorySharingPolicy, +): void { + const isolation = adapter.branchIsolation + if (!isolation) { + throw new Error( + `${adapter.id}: cannot isolate memory branch '${branchId}': adapter must declare branchIsolation`, + ) + } + if (isolation.mode === 'unsupported') { + throw new Error( + `${adapter.id}: cannot isolate memory branch '${branchId}': ${isolation.reason}`, + ) + } + if (isolation.mode === 'instance' && isolation.branchId !== branchId) { + throw new Error( + `${adapter.id}: adapter instance belongs to branch '${isolation.branchId}', not '${branchId}'`, + ) + } + if ( + isolation.mode === 'instance' && + isolation.supportsLogicalScopes !== true && + (policy.write !== 'shared' || policy.read.length !== 1 || policy.read[0] !== 'shared') + ) { + throw new Error( + `${adapter.id}: dedicated branch instances without logical scope isolation support only shared memory policy`, + ) + } +} + +function retainAdapter(adapter: AgentMemoryAdapter): () => Promise { + const existing = adapterLeases.get(adapter) + if (existing?.closed) throw new Error(`${adapter.id}: adapter is already closed`) + const state = existing ?? { references: 0, closed: false } + state.references += 1 + adapterLeases.set(adapter, state) + let released = false + return async () => { + if (released) return + released = true + state.references -= 1 + if (state.references > 0) return + if (!adapter.close) { + adapterLeases.delete(adapter) + return + } + state.closed = true + state.closePromise ??= adapter.close() + await state.closePromise + } +} + +function physicalScope( + branchId: string, + logical: AgentMemoryScope, + visibility: AgentMemoryVisibility, +): AgentMemoryScope { + if (visibility === 'private' && !logical.agentId) { + throw new Error('private memory requires scope.agentId') + } + if (visibility === 'team' && !logical.teamId) { + throw new Error('team memory requires scope.teamId') + } + const principal = + visibility === 'private' ? logical.agentId! : visibility === 'team' ? logical.teamId! : 'all' + const namespace = [ + logical.namespace ?? 'default', + stableId('branch', branchId), + visibility, + stableId('principal', principal), + ].join('/') + const physical: AgentMemoryScope = { + ...logical, + namespace, + tags: { + ...(logical.tags ?? {}), + memoryBranchId: branchId, + memoryVisibility: visibility, + logicalNamespace: logical.namespace ?? 'default', + }, + } + if (visibility === 'shared') { + delete physical.agentId + delete physical.teamId + } else if (visibility === 'team') { + delete physical.agentId + } + return physical +} + +function writeQueueKey( + branchId: string, + logical: AgentMemoryScope, + visibility: AgentMemoryVisibility, +): string { + return canonicalJson( + stripUndefined({ + branchId, + visibility, + tenantId: logical.tenantId, + userId: logical.userId, + agentId: logical.agentId, + fallbackTeamId: logical.agentId === undefined ? logical.teamId : undefined, + fallbackPrincipal: + logical.agentId === undefined && logical.teamId === undefined ? 'unattributed' : undefined, + }), + ) +} + +function normalizePolicy(policy: AgentMemorySharingPolicy): AgentMemorySharingPolicy { + const allowed = new Set(['private', 'team', 'shared']) + if (!allowed.has(policy.write) || policy.read.some((visibility) => !allowed.has(visibility))) { + throw new Error('memory sharing policy contains an unsupported visibility') + } + const read = [...new Set(policy.read)] + if (read.length === 0) throw new Error('memory sharing policy must read at least one visibility') + return { read, write: policy.write } +} + +function mergeHits(hits: AgentMemoryHit[], limit?: number): AgentMemoryHit[] { + const byIdentity = new Map() + for (const hit of hits) { + const key = `${hit.uri}\n${hit.text}` + const prior = byIdentity.get(key) + const score = hit.normalizedScore ?? hit.score ?? Number.NEGATIVE_INFINITY + const priorScore = prior?.normalizedScore ?? prior?.score ?? Number.NEGATIVE_INFINITY + if (!prior || score > priorScore) byIdentity.set(key, hit) + } + return [...byIdentity.values()] + .sort( + (a, b) => + (b.normalizedScore ?? b.score ?? 0) - (a.normalizedScore ?? a.score ?? 0) || + a.id.localeCompare(b.id), + ) + .slice(0, limit ?? Number.POSITIVE_INFINITY) +} + +function createSnapshot( + input: Omit, +): AgentMemoryBranchSnapshot { + const payload = cloneDurableValue({ + ...input, + journal: [...input.journal].sort((a, b) => a.sequence - b.sequence), + }) + assertDurableValue(payload, `${input.branchId}: branch snapshot`) + return { + ...payload, + digest: `sha256:${sha256(canonicalJson(stripUndefined(payload)))}`, + } +} + +function assertSnapshotDigest(snapshot: AgentMemoryBranchSnapshot): void { + const { digest, ...payload } = snapshot + const actual = `sha256:${sha256(canonicalJson(stripUndefined(payload)))}` + if (actual !== digest) + throw new Error(`memory branch snapshot digest mismatch: ${digest} != ${actual}`) +} + +function mergeScopes(base?: AgentMemoryScope, extra?: AgentMemoryScope): AgentMemoryScope { + return { + ...(base ?? {}), + ...(extra ?? {}), + tags: { ...(base?.tags ?? {}), ...(extra?.tags ?? {}) }, + } +} + +function scopeMatches(actual: AgentMemoryScope, requested: AgentMemoryScope): boolean { + for (const key of [ + 'tenantId', + 'userId', + 'agentId', + 'teamId', + 'runId', + 'sessionId', + 'namespace', + ] as const) { + if (requested[key] !== undefined && actual[key] !== requested[key]) return false + } + for (const [key, value] of Object.entries(requested.tags ?? {})) { + if (actual.tags?.[key] !== value) return false + } + return true +} + +function stripUndefined(value: unknown): unknown { + if (Array.isArray(value)) return value.map(stripUndefined) + if (typeof value !== 'object' || value === null) return value + return Object.fromEntries( + Object.entries(value) + .filter(([, child]) => child !== undefined) + .map(([key, child]) => [key, stripUndefined(child)]), + ) +} + +function cloneWriteInput(input: AgentMemoryWriteInput): AgentMemoryWriteInput { + return cloneDurableValue(input) +} + +function cloneWriteResult(result: AgentMemoryWriteResult): AgentMemoryWriteResult { + return cloneDurableValue(result) +} + +function cloneJournalEntry(entry: AgentMemoryJournalEntry): AgentMemoryJournalEntry { + return { + sequence: entry.sequence, + input: cloneWriteInput(entry.input), + result: cloneWriteResult(entry.result), + } +} + +function assertDurableValue(value: unknown, label: string, seen = new WeakSet()): void { + if ( + value === null || + typeof value === 'string' || + typeof value === 'boolean' || + (typeof value === 'number' && Number.isFinite(value)) + ) { + return + } + if (value === undefined) return + if (typeof value !== 'object') { + throw new Error(`${label} must contain only JSON-compatible values`) + } + if (seen.has(value)) throw new Error(`${label} must not contain cycles`) + seen.add(value) + if (Array.isArray(value)) { + for (const child of value) { + if (child === undefined) { + throw new Error(`${label} arrays must not contain undefined values`) + } + assertDurableValue(child, label, seen) + } + } else { + const prototype = Object.getPrototypeOf(value) + if (prototype !== Object.prototype && prototype !== null) { + throw new Error(`${label} must contain only plain objects`) + } + for (const child of Object.values(value)) assertDurableValue(child, label, seen) + } + seen.delete(value) +} + +function cloneDurableValue(value: T): T { + if (Array.isArray(value)) return value.map((child) => cloneDurableValue(child)) as T + if (typeof value !== 'object' || value === null) return value + return Object.fromEntries( + Object.entries(value).map(([key, child]) => [key, cloneDurableValue(child)]), + ) as T +} diff --git a/src/memory/experiment.ts b/src/memory/experiment.ts new file mode 100644 index 0000000..f890b5c --- /dev/null +++ b/src/memory/experiment.ts @@ -0,0 +1,957 @@ +import { join } from 'node:path' +import { canonicalJson } from '@tangle-network/agent-eval' +import { + type CampaignResult, + type CampaignStorage, + type CostLedgerHandle, + type DispatchContext, + fsCampaignStorage, + type JudgeConfig, + resolveRunDir, + runCampaign, + type Scenario, +} from '@tangle-network/agent-eval/campaign' +import { + type KnowledgeBenchmarkFamily, + type KnowledgeBenchmarkSplit, + type KnowledgeMemoryBenchmarkCase, + type KnowledgeMemoryBenchmarkTaskKind, + type KnowledgeMemoryEvent, + type KnowledgeMemoryFactMatcher, + scoreMemoryBenchmarkArtifact, +} from '../benchmarks/index' +import { stableId } from '../ids' +import { + type AgentMemoryBranch, + type AgentMemoryBranchSnapshot, + type AgentMemorySharingPolicy, + createAgentMemoryBranch, +} from './branch' +import type { AgentMemoryAdapter, AgentMemoryScope, AgentMemoryWriteInput } from './types' + +export interface AgentMemorySequenceProbe { + id: string + query: string + scope?: AgentMemoryScope + limit?: number + taskKind?: KnowledgeMemoryBenchmarkTaskKind + requiredFacts?: readonly KnowledgeMemoryFactMatcher[] + forbiddenFacts?: readonly KnowledgeMemoryFactMatcher[] + expectedEventIds?: readonly string[] + expectedActorIds?: readonly string[] + referenceAnswer?: string +} + +export interface AgentMemorySequenceStep { + id: string + instruction?: string + scope?: AgentMemoryScope + writes?: readonly AgentMemoryWriteInput[] + parallelWrites?: boolean + probes?: readonly AgentMemorySequenceProbe[] + parallelProbes?: boolean + metadata?: Record +} + +export interface AgentMemorySequence { + id: string + family: KnowledgeBenchmarkFamily | string + split?: KnowledgeBenchmarkSplit + steps: readonly AgentMemorySequenceStep[] + /** Exact scopes a runtime callback may write beyond scopes declared by steps. */ + cleanupScopes?: readonly AgentMemoryScope[] + tags?: readonly string[] + metadata?: Record +} + +export interface BuildAgentMemorySequencesFromBenchmarkCasesOptions { + memoryAgentId?: string + eventScope?: (input: { + event: KnowledgeMemoryEvent + case: KnowledgeMemoryBenchmarkCase + eventIndex: number + }) => AgentMemoryScope + probeScope?: (testCase: KnowledgeMemoryBenchmarkCase) => AgentMemoryScope +} + +export interface AgentMemoryExperimentCandidate { + id: string + label?: string + /** Change when provider configuration changes so cached cells cannot be reused. */ + ref?: string + createAdapter(input: { + branchId: string + sequence: AgentMemorySequence + rep: number + seed: number + }): AgentMemoryAdapter | Promise + policy?: AgentMemorySharingPolicy + baseScope?: AgentMemoryScope + /** Conservative external provider charge for one complete history. */ + externalCostUsdPerSequence?: number + /** Release resources and, when cleanupBranches is false, delete the isolated state. */ + disposeAdapter?(adapter: AgentMemoryAdapter): Promise +} + +export interface AgentMemorySequenceProbeResult { + id: string + stepId: string + query: string + score: number + passed: boolean + dimensions: Record + notes: string + hitIds: readonly string[] +} + +export interface AgentMemorySequenceArtifact { + candidateId: string + sequenceId: string + score: number + passed: boolean + dimensions: Record + probes: readonly AgentMemorySequenceProbeResult[] + branchDigest: string + journalEntries: number + durationMs: number +} + +export interface AgentMemorySequenceScenario extends Scenario { + kind: 'agent-memory-sequence' + candidateId: string + sequenceId: string + sequence: AgentMemorySequence + seedGroup: string +} + +export interface AgentMemoryExperimentRankingRow { + rank: number + candidateId: string + label: string + scoreMean: number + passRate: number + totalSequences: number + totalCells: number + totalProbes: number + cellsFailed: number + totalCostUsd: number + durationMs: number + dimensions: Record +} + +export interface RunAgentMemoryExperimentOptions { + experimentId: string + /** Stable external branch namespace; distributed workers must use the same value. */ + experimentRunId?: string + sequences: readonly AgentMemorySequence[] + candidates: readonly AgentMemoryExperimentCandidate[] + runDir: string + executeStep?: (input: { + memory: AgentMemoryBranch + candidateId: string + sequence: AgentMemorySequence + step: AgentMemorySequenceStep + context: DispatchContext + }) => Promise + /** Required with executeStep; identify the runtime/profile behavior in cache keys. */ + executeStepRef?: string + onBranchSnapshot?: (input: { + candidateId: string + sequenceId: string + cellId: string + snapshot: AgentMemoryBranchSnapshot + }) => Promise | void + cleanupBranches?: boolean + storage?: CampaignStorage + repo?: string + seed?: number + reps?: number + resumable?: boolean + costCeiling?: number + /** Shared across nested experiments when an outer improvement run owns spend. */ + costLedger?: CostLedgerHandle + costPhase?: string + maxConcurrency?: number + dispatchTimeoutMs?: number + now?: () => Date +} + +export interface RunAgentMemoryExperimentResult { + campaign: CampaignResult + rows: readonly AgentMemoryExperimentRankingRow[] + leaderCandidateId?: string + rankingJsonPath: string + rankingMarkdownPath: string +} + +class AgentMemoryCleanupError extends AggregateError { + constructor( + errors: Iterable, + message: string, + readonly afterAbort: boolean, + ) { + super(errors, message) + this.name = 'AgentMemoryCleanupError' + } +} + +/** Converts existing ordered memory benchmark cases into executable histories. */ +export function buildAgentMemorySequencesFromBenchmarkCases( + cases: readonly KnowledgeMemoryBenchmarkCase[], + options: BuildAgentMemorySequencesFromBenchmarkCasesOptions = {}, +): AgentMemorySequence[] { + const memoryAgentId = options.memoryAgentId ?? 'benchmark-agent' + return cases.map((testCase) => ({ + id: testCase.id, + family: testCase.family, + ...(testCase.split !== undefined ? { split: testCase.split } : {}), + ...(testCase.tags !== undefined ? { tags: testCase.tags } : {}), + metadata: compactRecord({ + ...(testCase.metadata ?? {}), + taskKind: testCase.taskKind, + source: testCase.source, + }), + steps: [ + ...testCase.events.map((event, eventIndex) => ({ + id: `event:${event.id}`, + scope: compactScope( + options.eventScope?.({ event, case: testCase, eventIndex }) ?? { + agentId: memoryAgentId, + sessionId: testCase.id, + }, + ), + writes: [ + { + id: event.id, + kind: 'observation' as const, + text: event.text, + metadata: compactRecord({ + ...(event.metadata ?? {}), + eventId: event.id, + actorId: event.actorId, + sessionId: event.sessionId, + timestamp: event.timestamp, + }), + }, + ], + metadata: { eventIndex }, + })), + { + id: 'probe', + scope: compactScope( + options.probeScope?.(testCase) ?? { + agentId: memoryAgentId, + sessionId: testCase.id, + }, + ), + probes: [ + { + id: 'answer', + query: testCase.prompt, + taskKind: testCase.taskKind, + ...(testCase.requiredFacts !== undefined + ? { requiredFacts: testCase.requiredFacts } + : {}), + ...(testCase.forbiddenFacts !== undefined + ? { forbiddenFacts: testCase.forbiddenFacts } + : {}), + ...(testCase.expectedEventIds !== undefined + ? { expectedEventIds: testCase.expectedEventIds } + : {}), + ...(testCase.expectedActorIds !== undefined + ? { expectedActorIds: testCase.expectedActorIds } + : {}), + ...(testCase.referenceAnswer !== undefined + ? { referenceAnswer: testCase.referenceAnswer } + : {}), + }, + ], + }, + ], + })) +} + +/** Runs ordered, branch-isolated memory histories across candidate systems. */ +export async function runAgentMemoryExperiment( + options: RunAgentMemoryExperimentOptions, +): Promise { + assertNonEmptyString(options.experimentId, 'memory experiment experimentId') + assertNonEmptyString(options.runDir, 'memory experiment runDir') + if (options.experimentRunId !== undefined) { + assertNonEmptyString(options.experimentRunId, 'memory experiment experimentRunId') + } + if (options.sequences.length === 0) throw new Error('memory experiment requires sequences') + if (options.candidates.length === 0) throw new Error('memory experiment requires candidates') + if (options.executeStep && !options.executeStepRef) { + throw new Error('memory experiment executeStepRef is required when executeStep is configured') + } + assertUnique( + options.sequences.map((sequence) => sequence.id), + 'sequence', + ) + assertUnique( + options.candidates.map((candidate) => candidate.id), + 'candidate', + ) + assertMemorySequences(options.sequences) + for (const candidate of options.candidates) { + if ( + candidate.externalCostUsdPerSequence !== undefined && + (!Number.isFinite(candidate.externalCostUsdPerSequence) || + candidate.externalCostUsdPerSequence < 0) + ) { + throw new Error( + `${candidate.id}: externalCostUsdPerSequence must be a non-negative finite number`, + ) + } + if (candidate.ref !== undefined) assertNonEmptyString(candidate.ref, `${candidate.id} ref`) + } + + const storage = options.storage ?? fsCampaignStorage() + const runDir = resolveRunDir(options.runDir, options.repo) + const runIdentity = stableId( + 'memory_run', + canonicalJson({ + experimentId: options.experimentId, + experimentRunId: options.experimentRunId ?? runDir, + }), + ) + const maxConcurrency = options.maxConcurrency ?? 2 + if (!Number.isSafeInteger(maxConcurrency) || maxConcurrency <= 0) { + throw new Error('memory experiment maxConcurrency must be a positive safe integer') + } + const executionPool = createExecutionPool(maxConcurrency) + const dispatchedExecutions: Promise[] = [] + const candidateById = new Map(options.candidates.map((candidate) => [candidate.id, candidate])) + const scenarios = buildAgentMemorySequenceScenarios(options.sequences, options.candidates) + let campaign: CampaignResult + let settledExecutions: PromiseSettledResult[] = [] + try { + campaign = await runCampaign({ + scenarios, + dispatch: (scenario, context) => { + const candidate = candidateById.get(scenario.candidateId) + if (!candidate) throw new Error(`unknown memory candidate ${scenario.candidateId}`) + const operation = executionPool.run(() => + runSequenceCell({ options, candidate, scenario, context, runIdentity }), + ) + dispatchedExecutions.push(operation) + return operation + }, + dispatchRef: memoryExperimentDispatchRef(options), + judges: [agentMemorySequenceJudge()], + runDir, + storage, + seed: options.seed, + reps: options.reps, + resumable: options.resumable, + costCeiling: options.costCeiling, + costLedger: options.costLedger, + costPhase: options.costPhase, + maxConcurrency, + dispatchTimeoutMs: options.dispatchTimeoutMs, + expectUsage: 'off', + now: options.now, + }) + } finally { + settledExecutions = await Promise.allSettled(dispatchedExecutions) + } + const cleanupFailures = settledExecutions.flatMap((settled) => + settled.status === 'rejected' && + settled.reason instanceof AgentMemoryCleanupError && + settled.reason.afterAbort + ? [settled.reason] + : [], + ) + if (cleanupFailures.length > 0) { + throw new AggregateError(cleanupFailures, 'memory experiment cleanup failed after dispatch') + } + const rows = rankAgentMemoryExperiment(options.candidates, scenarios, campaign) + const rankingJsonPath = join(campaign.runDir, 'memory-experiment-ranking.json') + const rankingMarkdownPath = join(campaign.runDir, 'memory-experiment-ranking.md') + storage.write( + rankingJsonPath, + `${JSON.stringify({ experimentId: options.experimentId, rows }, null, 2)}\n`, + ) + storage.write(rankingMarkdownPath, renderAgentMemoryExperimentRanking(rows)) + return { + campaign, + rows, + leaderCandidateId: rows.find((row) => row.cellsFailed === 0)?.candidateId, + rankingJsonPath, + rankingMarkdownPath, + } +} + +export function buildAgentMemorySequenceScenarios( + sequences: readonly AgentMemorySequence[], + candidates: readonly Pick[], +): AgentMemorySequenceScenario[] { + return candidates.flatMap((candidate) => + sequences.map((sequence) => ({ + id: `${stableId('candidate', candidate.id)}:${sequence.id}`, + kind: 'agent-memory-sequence' as const, + candidateId: candidate.id, + sequenceId: sequence.id, + sequence, + seedGroup: sequence.id, + tags: [...new Set([sequence.split ?? 'dev', ...(sequence.tags ?? []), candidate.id])], + })), + ) +} + +export function agentMemorySequenceJudge(): JudgeConfig< + AgentMemorySequenceArtifact, + AgentMemorySequenceScenario +> { + return { + name: 'agent-memory-sequence', + dimensions: [ + { key: 'score', description: 'mean memory probe score' }, + { key: 'passed', description: '1 when every memory probe passes' }, + { key: 'memory_fact_recall', description: 'current memory fact coverage' }, + { key: 'memory_event_recall', description: 'memory source event coverage' }, + { key: 'memory_actor_recall', description: 'memory actor attribution coverage' }, + { key: 'memory_stale_safe', description: '1 when obsolete memory is not reused' }, + ], + score({ artifact }) { + return { + composite: artifact.score, + dimensions: { + score: artifact.score, + passed: artifact.passed ? 1 : 0, + ...artifact.dimensions, + }, + notes: `${artifact.probes.filter((probe) => probe.passed).length}/${artifact.probes.length} probes passed`, + } + }, + } +} + +async function runSequenceCell(input: { + options: RunAgentMemoryExperimentOptions + candidate: AgentMemoryExperimentCandidate + scenario: AgentMemorySequenceScenario + context: DispatchContext + runIdentity: string +}): Promise { + const { options, candidate, scenario, context, runIdentity } = input + const cleanupBranches = options.cleanupBranches ?? true + const costUsd = candidate.externalCostUsdPerSequence ?? 0 + if (!Number.isFinite(costUsd) || costUsd < 0) { + throw new Error( + `${candidate.id}: externalCostUsdPerSequence must be a non-negative finite number`, + ) + } + if (!cleanupBranches && !candidate.disposeAdapter) { + throw new Error( + `${candidate.id}: cleanupBranches=false requires disposeAdapter to delete isolated external state`, + ) + } + + const execute = async (): Promise => { + context.signal.throwIfAborted() + const startedAt = Date.now() + const branchId = stableId('memory_branch', `${runIdentity}:${context.cellId}:${context.seed}`) + const adapter = await candidate.createAdapter({ + branchId, + sequence: scenario.sequence, + rep: context.rep, + seed: context.seed, + }) + let memory: AgentMemoryBranch | undefined + let primaryError: unknown + let completedArtifact: AgentMemorySequenceArtifact | undefined + try { + if (cleanupBranches && !adapter.clear) { + throw new Error( + `${candidate.id}: cleanupBranches requires an adapter with scoped clear support`, + ) + } + memory = createAgentMemoryBranch({ + adapter, + branchId, + policy: candidate.policy, + allowedWriteScopes: sequenceCleanupScopes(scenario.sequence), + baseScope: mergeScopes(candidate.baseScope, { + tags: { + memoryExperimentId: options.experimentId, + memoryCandidateId: candidate.id, + memorySequenceId: scenario.sequenceId, + }, + }), + }) + if (cleanupBranches) await clearSequenceScopes(memory, scenario.sequence) + const probes: AgentMemorySequenceProbeResult[] = [] + for (const step of scenario.sequence.steps) { + context.signal.throwIfAborted() + await writeStep(memory, step) + await options.executeStep?.({ + memory, + candidateId: candidate.id, + sequence: scenario.sequence, + step, + context, + }) + context.signal.throwIfAborted() + const stepProbes = await probeStep(memory, scenario.sequence, step) + probes.push(...stepProbes) + } + const snapshot = await memory.snapshot() + await options.onBranchSnapshot?.({ + candidateId: candidate.id, + sequenceId: scenario.sequenceId, + cellId: context.cellId, + snapshot, + }) + const dimensions = meanDimensions(probes.map((probe) => probe.dimensions)) + const artifact: AgentMemorySequenceArtifact = { + candidateId: candidate.id, + sequenceId: scenario.sequenceId, + score: mean(probes.map((probe) => probe.score)), + passed: probes.length > 0 && probes.every((probe) => probe.passed), + dimensions, + probes, + branchDigest: snapshot.digest, + journalEntries: snapshot.journal.length, + durationMs: Math.max(0, Date.now() - startedAt), + } + if (cleanupBranches) await clearSequenceScopes(memory, scenario.sequence) + completedArtifact = artifact + } catch (error) { + primaryError = error + } + const cleanupErrors: unknown[] = [] + if (primaryError && memory && cleanupBranches && adapter.clear) { + try { + await clearSequenceScopes(memory, scenario.sequence) + } catch (error) { + cleanupErrors.push(error) + } + } + try { + if (memory) await memory.close?.() + else { + await adapter.flush?.() + await adapter.close?.() + } + } catch (error) { + cleanupErrors.push(error) + } + try { + await candidate.disposeAdapter?.(adapter) + } catch (error) { + cleanupErrors.push(error) + } + if (cleanupErrors.length > 0) { + const primaryMessage = + primaryError instanceof Error ? primaryError.message : String(primaryError ?? '') + throw new AgentMemoryCleanupError( + primaryError ? [primaryError, ...cleanupErrors] : cleanupErrors, + `${candidate.id}: memory branch cleanup failed${primaryMessage ? ` after: ${primaryMessage}` : ''}`, + context.signal.aborted, + ) + } + if (primaryError) throw primaryError + if (!completedArtifact) throw new Error(`${candidate.id}: memory sequence produced no result`) + return completedArtifact + } + + if (costUsd === 0) return execute() + const receipt = { + model: candidate.id, + inputTokens: 0, + outputTokens: 0, + usageUnknown: true, + actualCostUsd: costUsd, + } as const + const paid = await context.cost.runPaidCall({ + actor: `agent-knowledge:memory-experiment:${candidate.id}`, + model: candidate.id, + maximumCharge: { externallyEnforcedMaximumUsd: costUsd }, + execute, + receipt: () => receipt, + receiptFromError: () => receipt, + }) + if (!paid.succeeded) throw paid.error + return paid.value +} + +function createExecutionPool(limit: number): { + run(operation: () => Promise): Promise +} { + let active = 0 + const waiters: Array<() => void> = [] + return { + async run(operation: () => Promise): Promise { + if (active >= limit) await new Promise((resolve) => waiters.push(resolve)) + active += 1 + try { + return await operation() + } finally { + active -= 1 + waiters.shift()?.() + } + }, + } +} + +async function writeStep(memory: AgentMemoryBranch, step: AgentMemorySequenceStep): Promise { + const writes = (step.writes ?? []).map((write) => ({ + ...write, + scope: mergeScopes(step.scope, write.scope), + })) + if (step.parallelWrites) { + await Promise.all(writes.map((write) => memory.write(write))) + return + } + for (const write of writes) await memory.write(write) +} + +async function probeStep( + memory: AgentMemoryBranch, + sequence: AgentMemorySequence, + step: AgentMemorySequenceStep, +): Promise { + const run = async (probe: AgentMemorySequenceProbe): Promise => { + const scope = mergeScopes(step.scope, probe.scope) + const context = await memory.getContext(probe.query, { scope, limit: probe.limit }) + const artifact = { + answer: context.text, + rememberedFacts: context.hits.map((hit) => hit.text), + citedEventIds: uniqueStrings([ + ...context.hits.map((hit) => hit.metadata?.eventId), + ...context.hits.flatMap((hit) => stringArray(hit.metadata?.eventIds)), + ]), + usedMemoryIds: context.hits.map((hit) => hit.id), + actorIds: uniqueStrings([ + ...context.hits.map((hit) => hit.metadata?.actorId), + ...context.hits.flatMap((hit) => stringArray(hit.metadata?.actorIds)), + ]), + } + const evaluation = scoreMemoryBenchmarkArtifact( + { + id: `${sequence.id}:${step.id}:${probe.id}`, + family: sequence.family, + taskKind: probe.taskKind ?? 'memory-recall', + split: sequence.split, + events: [], + prompt: probe.query, + requiredFacts: + probe.requiredFacts && probe.requiredFacts.length > 0 + ? probe.requiredFacts + : probe.referenceAnswer + ? [{ id: `${probe.id}:reference`, anyOf: [probe.referenceAnswer] }] + : undefined, + forbiddenFacts: probe.forbiddenFacts, + expectedEventIds: probe.expectedEventIds, + expectedActorIds: probe.expectedActorIds, + referenceAnswer: probe.referenceAnswer, + }, + artifact, + ) + return { + id: probe.id, + stepId: step.id, + query: probe.query, + score: evaluation.score, + passed: evaluation.passed, + dimensions: evaluation.dimensions, + notes: evaluation.notes, + hitIds: context.hits.map((hit) => hit.id), + } + } + if (step.parallelProbes === false) { + const results: AgentMemorySequenceProbeResult[] = [] + for (const probe of step.probes ?? []) results.push(await run(probe)) + return results + } + return Promise.all((step.probes ?? []).map(run)) +} + +function rankAgentMemoryExperiment( + candidates: readonly AgentMemoryExperimentCandidate[], + scenarios: readonly AgentMemorySequenceScenario[], + campaign: CampaignResult, +): AgentMemoryExperimentRankingRow[] { + const scenarioById = new Map(scenarios.map((scenario) => [scenario.id, scenario])) + const rows = candidates.map((candidate): AgentMemoryExperimentRankingRow => { + const candidateScenarios = scenarios.filter((scenario) => scenario.candidateId === candidate.id) + const cells = campaign.cells.filter( + (cell) => scenarioById.get(cell.scenarioId)?.candidateId === candidate.id, + ) + const successful = cells.filter((cell) => !cell.error && cell.artifact) + const dimensionKeys = new Set([ + 'memory_fact_recall', + 'memory_event_recall', + 'memory_actor_recall', + 'memory_stale_safe', + ...successful.flatMap((cell) => Object.keys(cell.artifact.dimensions)), + ]) + const dimensionRows = cells.map((cell) => + Object.fromEntries( + [...dimensionKeys].map((key) => [ + key, + !cell.error && cell.artifact ? (cell.artifact.dimensions[key] ?? 0) : 0, + ]), + ), + ) + return { + rank: 0, + candidateId: candidate.id, + label: candidate.label ?? candidate.id, + scoreMean: mean( + cells.map((cell) => (!cell.error && cell.artifact ? cell.artifact.score : 0)), + ), + passRate: mean(cells.map((cell) => (!cell.error && cell.artifact?.passed ? 1 : 0))), + totalSequences: candidateScenarios.length, + totalCells: cells.length, + totalProbes: successful.reduce((sum, cell) => sum + cell.artifact.probes.length, 0), + cellsFailed: cells.filter((cell) => Boolean(cell.error)).length, + totalCostUsd: cells.reduce((sum, cell) => sum + cell.costUsd, 0), + durationMs: cells.reduce((sum, cell) => sum + cell.durationMs, 0), + dimensions: meanDimensions(dimensionRows), + } + }) + return rows + .sort( + (a, b) => + Number(a.cellsFailed > 0) - Number(b.cellsFailed > 0) || + b.scoreMean - a.scoreMean || + b.passRate - a.passRate || + a.totalCostUsd - b.totalCostUsd || + a.candidateId.localeCompare(b.candidateId), + ) + .map((row, index) => ({ ...row, rank: index + 1 })) +} + +function renderAgentMemoryExperimentRanking( + rows: readonly AgentMemoryExperimentRankingRow[], +): string { + return [ + '# Agent Memory Experiment', + '', + '| rank | candidate | sequences | cells | probes | failed | score | pass rate | cost | duration ms |', + '| ---: | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |', + ...rows.map( + (row) => + `| ${row.rank} | ${row.label} | ${row.totalSequences} | ${row.totalCells} | ${row.totalProbes} | ${row.cellsFailed} | ${format(row.scoreMean)} | ${format(row.passRate)} | $${format(row.totalCostUsd)} | ${format(row.durationMs)} |`, + ), + '', + ].join('\n') +} + +function memoryExperimentDispatchRef(options: RunAgentMemoryExperimentOptions): string { + return stableId( + 'memory_experiment', + canonicalJson({ + experimentId: options.experimentId, + experimentRunId: options.experimentRunId ?? null, + executeStepRef: options.executeStepRef ?? 'fixtures', + cleanupBranches: options.cleanupBranches ?? true, + candidates: options.candidates + .map((candidate) => ({ + id: candidate.id, + ref: candidate.ref ?? candidate.id, + policy: candidate.policy ?? null, + baseScope: candidate.baseScope ?? null, + externalCostUsdPerSequence: candidate.externalCostUsdPerSequence ?? 0, + })) + .sort((a, b) => a.id.localeCompare(b.id)), + }), + ) +} + +function meanDimensions(rows: readonly Record[]): Record { + const values = new Map() + for (const row of rows) { + for (const [key, value] of Object.entries(row)) { + if (!Number.isFinite(value)) continue + const bucket = values.get(key) ?? [] + bucket.push(value) + values.set(key, bucket) + } + } + return Object.fromEntries([...values].map(([key, bucket]) => [key, mean(bucket)])) +} + +function mean(values: readonly number[]): number { + return values.length === 0 ? 0 : values.reduce((sum, value) => sum + value, 0) / values.length +} + +function mergeScopes(base?: AgentMemoryScope, extra?: AgentMemoryScope): AgentMemoryScope { + return { + ...(base ?? {}), + ...(extra ?? {}), + tags: { ...(base?.tags ?? {}), ...(extra?.tags ?? {}) }, + } +} + +function sequenceCleanupScopes(sequence: AgentMemorySequence): AgentMemoryScope[] { + const scopes = new Map() + for (const scope of sequence.cleanupScopes ?? []) { + const normalized = normalizeCleanupScope(scope) + scopes.set(JSON.stringify(normalized), normalized) + } + for (const step of sequence.steps) { + const candidates = [ + ...(step.scope ? [step.scope] : []), + ...(step.writes ?? []).map((write) => mergeScopes(step.scope, write.scope)), + ...(step.probes ?? []).map((probe) => mergeScopes(step.scope, probe.scope)), + ] + for (const scope of candidates) { + const normalized = normalizeCleanupScope(scope) + scopes.set(JSON.stringify(normalized), normalized) + } + } + return [...scopes.values()] +} + +async function clearSequenceScopes( + memory: AgentMemoryBranch, + sequence: AgentMemorySequence, +): Promise { + for (const scope of sequenceCleanupScopes(sequence)) await memory.clear?.(scope) +} + +function assertMemorySequences(sequences: readonly AgentMemorySequence[]): void { + for (const sequence of sequences) { + assertNonEmptyString(sequence.family, `memory experiment sequence ${sequence.id} family`) + if (sequence.steps.length === 0) { + throw new Error(`memory experiment sequence ${sequence.id} has no steps`) + } + assertUnique( + sequence.steps.map((step) => step.id), + `step in sequence ${sequence.id}`, + ) + let probeCount = 0 + for (const step of sequence.steps) { + for (const write of step.writes ?? []) { + assertNonEmptyString(write.text, `memory experiment write in ${sequence.id}/${step.id}`) + if (write.id !== undefined) { + assertNonEmptyString(write.id, `memory experiment write id in ${sequence.id}/${step.id}`) + } + } + assertUnique( + (step.probes ?? []).map((probe) => probe.id), + `probe in sequence ${sequence.id} step ${step.id}`, + ) + for (const probe of step.probes ?? []) { + probeCount += 1 + assertNonEmptyString( + probe.query, + `memory experiment probe query ${sequence.id}/${step.id}/${probe.id}`, + ) + if (probe.limit !== undefined && (!Number.isSafeInteger(probe.limit) || probe.limit <= 0)) { + throw new Error( + `memory experiment probe limit ${sequence.id}/${step.id}/${probe.id} must be a positive safe integer`, + ) + } + assertMemoryFactMatchers( + probe.requiredFacts ?? [], + `${sequence.id}/${step.id}/${probe.id} requiredFacts`, + ) + assertMemoryFactMatchers( + probe.forbiddenFacts ?? [], + `${sequence.id}/${step.id}/${probe.id} forbiddenFacts`, + ) + assertStringList(probe.expectedEventIds, `${sequence.id}/${step.id}/${probe.id} event ids`) + assertStringList(probe.expectedActorIds, `${sequence.id}/${step.id}/${probe.id} actor ids`) + if (probe.referenceAnswer !== undefined) { + assertNonEmptyString( + probe.referenceAnswer, + `memory experiment reference answer ${sequence.id}/${step.id}/${probe.id}`, + ) + } + const hasTarget = + (probe.requiredFacts?.length ?? 0) > 0 || + (probe.forbiddenFacts?.length ?? 0) > 0 || + (probe.expectedEventIds?.length ?? 0) > 0 || + (probe.expectedActorIds?.length ?? 0) > 0 || + Boolean(probe.referenceAnswer?.trim()) + if (!hasTarget) { + throw new Error( + `memory experiment probe ${sequence.id}/${step.id}/${probe.id} has no measurable target`, + ) + } + } + } + if (probeCount === 0) { + throw new Error(`memory experiment sequence ${sequence.id} has no probes`) + } + } +} + +function normalizeCleanupScope(scope: AgentMemoryScope): AgentMemoryScope { + const normalized = compactScope(scope) + if (normalized.tags && Object.keys(normalized.tags).length === 0) { + delete normalized.tags + } + return normalized +} + +function compactScope(scope: AgentMemoryScope): AgentMemoryScope { + return Object.fromEntries( + Object.entries(scope).filter(([, value]) => value !== undefined), + ) as AgentMemoryScope +} + +function compactRecord(record: Record): Record { + return Object.fromEntries(Object.entries(record).filter(([, value]) => value !== undefined)) +} + +function uniqueStrings(values: readonly unknown[]): string[] { + return [...new Set(values.filter((value): value is string => typeof value === 'string'))] +} + +function stringArray(value: unknown): string[] { + return Array.isArray(value) + ? value.filter((entry): entry is string => typeof entry === 'string') + : [] +} + +function assertUnique(values: readonly string[], label: string): void { + const seen = new Set() + for (const value of values) { + assertNonEmptyString(value, `memory experiment ${label} id`) + if (seen.has(value)) throw new Error(`duplicate memory experiment ${label} id: ${value}`) + seen.add(value) + } +} + +function assertMemoryFactMatchers( + matchers: readonly KnowledgeMemoryFactMatcher[], + label: string, +): void { + assertUnique( + matchers.map((matcher) => matcher.id), + `${label} matcher`, + ) + for (const matcher of matchers) { + if (matcher.anyOf.length === 0) { + throw new Error(`memory experiment ${label} matcher ${matcher.id} requires anyOf`) + } + assertStringList(matcher.anyOf, `${label} matcher ${matcher.id} anyOf`) + assertStringList(matcher.sourceEventIds, `${label} matcher ${matcher.id} source event ids`) + if (matcher.weight !== undefined && (!Number.isFinite(matcher.weight) || matcher.weight <= 0)) { + throw new Error( + `memory experiment ${label} matcher ${matcher.id} weight must be a positive finite number`, + ) + } + } +} + +function assertStringList(values: readonly string[] | undefined, label: string): void { + if (values === undefined) return + assertUnique(values, label) +} + +function assertNonEmptyString(value: unknown, label: string): asserts value is string { + if (typeof value !== 'string' || value.trim().length === 0) { + throw new Error(`${label} must be a non-empty string`) + } +} + +function format(value: number): string { + return Number.isFinite(value) ? value.toFixed(4) : '0.0000' +} diff --git a/src/memory/graphiti.ts b/src/memory/graphiti.ts new file mode 100644 index 0000000..46131dd --- /dev/null +++ b/src/memory/graphiti.ts @@ -0,0 +1,552 @@ +import { canonicalJson } from '@tangle-network/agent-eval' +import { sha256, stableId } from '../ids' +import { defaultGetMemoryContext } from './adapter' +import { memoryWriteResultToSourceRecord } from './source-record' +import type { AgentMemoryAdapter, AgentMemoryHit, AgentMemoryScope } from './types' + +export interface GraphitiMcpClientLike { + callTool(request: { name: string; arguments?: Record }): Promise +} + +export interface GraphitiMemoryAdapterOptions { + client: GraphitiMcpClientLike + id?: string + defaultScope?: AgentMemoryScope + consistency?: 'queued' | 'visible' + ingestionTimeoutMs?: number + pollIntervalMs?: number + /** Maximum episodes inspected to prove a queued write became visible. */ + episodeScanLimit?: number + provenanceEpisodeLimit?: number + search?: readonly ('facts' | 'nodes')[] + toolNames?: Partial +} + +export interface GraphitiToolNames { + addMemory: string + searchFacts: string + searchNodes: string + getEpisodes: string + clearGraph: string +} + +interface GraphitiEpisode { + uuid: string + name?: string + content?: string + source_description?: string + group_id?: string +} + +/** Connects the official Graphiti MCP server without coupling to its Python internals. */ +export function createGraphitiMemoryAdapter( + options: GraphitiMemoryAdapterOptions, +): AgentMemoryAdapter { + assertGraphitiOptions(options) + const id = options.id ?? 'graphiti' + const consistency = options.consistency ?? 'visible' + const tools = graphitiToolNames(options) + const queued = new Map() + + const adapter: AgentMemoryAdapter = { + id, + branchIsolation: + consistency === 'visible' + ? { mode: 'scoped' } + : { + mode: 'unsupported', + reason: + 'queued Graphiti writes can become visible after a process restart; use consistency="visible" for branch experiments', + }, + async search(query, searchOptions = {}) { + if (searchOptions.minScore !== undefined && !Number.isFinite(searchOptions.minScore)) { + throw new Error(`${id}: minScore must be finite`) + } + const scope = mergeScopes(options.defaultScope, searchOptions.scope) + const groupId = graphitiGroupId(id, scope) + const modes = new Set(options.search ?? ['facts', 'nodes']) + const kinds = searchOptions.kinds ? new Set(searchOptions.kinds) : null + const searchFacts = modes.has('facts') && (!kinds || kinds.has('fact')) + const searchNodes = modes.has('nodes') && (!kinds || kinds.has('entity')) + const limit = searchOptions.limit ?? 10 + if (!Number.isSafeInteger(limit) || limit < 0) { + throw new Error(`${id}: search limit must be a non-negative safe integer`) + } + if (limit === 0) return [] + const [factPayload, nodePayload] = await Promise.all([ + searchFacts + ? callGraphiti(options.client, tools.searchFacts, { + query, + group_ids: [groupId], + max_facts: limit, + }) + : undefined, + searchNodes + ? callGraphiti(options.client, tools.searchNodes, { + query, + group_ids: [groupId], + max_nodes: limit, + ...(searchOptions.metadata?.entityTypes + ? { entity_types: searchOptions.metadata.entityTypes } + : {}), + }) + : undefined, + ]) + const hits = [ + ...normalizeGraphitiFacts(factPayload, id), + ...normalizeGraphitiNodes(nodePayload, id), + ] + await enrichGraphitiProvenance(options, groupId, hits) + return hits + .filter((hit) => { + if (searchOptions.minScore === undefined) return true + const score = hit.normalizedScore ?? hit.score + return score !== undefined && score >= searchOptions.minScore + }) + .sort( + (a, b) => + (b.normalizedScore ?? b.score ?? Number.NEGATIVE_INFINITY) - + (a.normalizedScore ?? a.score ?? Number.NEGATIVE_INFINITY), + ) + .slice(0, limit) + }, + async getContext(query, searchOptions = {}) { + return defaultGetMemoryContext(adapter, query, searchOptions) + }, + async write(input) { + const scope = mergeScopes(options.defaultScope, input.scope) + const groupId = graphitiGroupId(id, scope) + const eventKey = canonicalJson( + stripUndefined({ + id: input.id, + kind: input.kind, + text: input.text, + title: input.title, + role: input.role, + metadata: input.metadata, + }), + ) + const episodeUuid = deterministicUuid(`${id}:${groupId}:${eventKey}`) + const name = input.title ?? `${input.kind}:${input.id ?? stableId('event', eventKey)}` + const sourceDescription = canonicalJson( + stripUndefined({ + adapter: 'agent-knowledge', + memoryKind: input.kind, + eventId: input.id, + actorId: input.metadata?.actorId, + sessionId: scope.sessionId, + role: input.role, + scope, + metadata: input.metadata ?? {}, + }), + ) + await callGraphiti(options.client, tools.addMemory, { + name, + episode_body: input.text, + group_id: groupId, + source: input.kind === 'message' ? 'message' : 'text', + source_description: sourceDescription, + uuid: episodeUuid, + ...(typeof input.metadata?.timestamp === 'string' + ? { reference_time: input.metadata.timestamp } + : {}), + }) + queued.set(episodeUuid, { groupId, name }) + if (consistency === 'visible') { + await waitForGraphitiEpisode(options, groupId, episodeUuid) + queued.delete(episodeUuid) + } + const result = { + accepted: true, + id: episodeUuid, + uri: `memory://${id}/${episodeUuid}`, + kind: input.kind, + metadata: { + provider: 'graphiti', + groupId, + consistency, + queued: consistency === 'queued', + }, + } as const + return { + ...result, + sourceRecord: memoryWriteResultToSourceRecord(result, input.text, { scope }), + } + }, + async clear(scope) { + const groupId = graphitiGroupId(id, mergeScopes(options.defaultScope, scope)) + for (const [episodeUuid, pending] of queued) { + if (pending.groupId !== groupId) continue + await waitForGraphitiEpisode(options, groupId, episodeUuid) + queued.delete(episodeUuid) + } + await callGraphiti(options.client, tools.clearGraph, { group_ids: [groupId] }) + }, + async flush() { + for (const [episodeUuid, pending] of queued) { + await waitForGraphitiEpisode(options, pending.groupId, episodeUuid) + queued.delete(episodeUuid) + } + }, + } + return adapter +} + +async function callGraphiti( + client: GraphitiMcpClientLike, + name: string, + args: Record, +): Promise { + const raw = await client.callTool({ name, arguments: args }) + const payload = graphitiPayload(raw) + if (isRecord(payload) && typeof payload.error === 'string') { + throw new Error(`Graphiti ${name}: ${payload.error}`) + } + return payload +} + +function graphitiPayload(raw: unknown): unknown { + if (!isRecord(raw)) return raw + if (raw.isError === true) throw new Error(`Graphiti MCP: ${toolText(raw) || 'tool call failed'}`) + if (isRecord(raw.structuredContent)) return unwrapResult(raw.structuredContent) + const text = toolText(raw) + if (text) { + try { + return unwrapResult(JSON.parse(text)) + } catch { + return { message: text } + } + } + return unwrapResult(raw) +} + +function unwrapResult(value: unknown): unknown { + if (isRecord(value) && Object.keys(value).length === 1 && 'result' in value) { + return value.result + } + return value +} + +function toolText(raw: Record): string { + if (!Array.isArray(raw.content)) return '' + return raw.content + .filter(isRecord) + .map((part) => (typeof part.text === 'string' ? part.text : '')) + .filter(Boolean) + .join('\n') +} + +function normalizeGraphitiFacts(raw: unknown, adapterId: string): AgentMemoryHit[] { + const facts = isRecord(raw) && Array.isArray(raw.facts) ? raw.facts.filter(isRecord) : [] + return facts.flatMap((fact) => { + const text = stringField(fact, ['fact', 'text']) + if (!text) return [] + const factId = + stringField(fact, ['uuid', 'id']) ?? + stableId('fact', canonicalJson(stripUndefined({ adapterId, text, fact }))) + const episodeUuids = stringArray(fact.episodes ?? fact.episode_uuids) + const score = numberField(fact, ['score', 'relevance_score']) + const normalizedScore = normalizedScoreOf(score) + return [ + { + id: factId, + uri: `memory://${adapterId}/fact/${factId}`, + kind: 'fact' as const, + text, + ...(stringField(fact, ['name']) ? { title: stringField(fact, ['name']) } : {}), + ...(stringField(fact, ['created_at']) + ? { createdAt: stringField(fact, ['created_at']) } + : {}), + ...(stringField(fact, ['invalid_at']) + ? { validUntil: stringField(fact, ['invalid_at']) } + : stringField(fact, ['expired_at']) + ? { validUntil: stringField(fact, ['expired_at']) } + : {}), + ...(score !== undefined ? { score } : {}), + ...(normalizedScore !== undefined ? { normalizedScore } : {}), + metadata: { + provider: 'graphiti', + groupId: fact.group_id, + sourceNodeUuid: fact.source_node_uuid, + targetNodeUuid: fact.target_node_uuid, + validAt: fact.valid_at, + invalidAt: fact.invalid_at, + episodeUuids, + attributes: fact.attributes, + }, + }, + ] + }) +} + +function normalizeGraphitiNodes(raw: unknown, adapterId: string): AgentMemoryHit[] { + const nodes = isRecord(raw) && Array.isArray(raw.nodes) ? raw.nodes.filter(isRecord) : [] + return nodes.flatMap((node) => { + const name = stringField(node, ['name']) + const summary = stringField(node, ['summary']) + const text = summary ?? name + if (!text) return [] + const nodeId = + stringField(node, ['uuid', 'id']) ?? + stableId('node', canonicalJson(stripUndefined({ adapterId, text, node }))) + const score = numberField(node, ['score', 'relevance_score']) + const normalizedScore = normalizedScoreOf(score) + return [ + { + id: nodeId, + uri: `memory://${adapterId}/entity/${nodeId}`, + kind: 'entity' as const, + text, + ...(name ? { title: name } : {}), + ...(stringField(node, ['created_at']) + ? { createdAt: stringField(node, ['created_at']) } + : {}), + ...(score !== undefined ? { score } : {}), + ...(normalizedScore !== undefined ? { normalizedScore } : {}), + metadata: { + provider: 'graphiti', + groupId: node.group_id, + labels: node.labels, + attributes: node.attributes, + }, + }, + ] + }) +} + +async function enrichGraphitiProvenance( + options: GraphitiMemoryAdapterOptions, + groupId: string, + hits: AgentMemoryHit[], +): Promise { + const wanted = new Set(hits.flatMap((hit) => stringArray(hit.metadata?.episodeUuids))) + if (wanted.size === 0) return + const scan = await scanGraphitiEpisodes( + options, + groupId, + wanted, + options.provenanceEpisodeLimit ?? options.episodeScanLimit ?? 10_000, + ) + const episodes = scan.episodes + const byId = new Map(episodes.map((episode) => [episode.uuid, episode])) + for (const hit of hits) { + const episodeUuids = stringArray(hit.metadata?.episodeUuids) + const sourceEpisodes = episodeUuids + .map((uuid) => byId.get(uuid)) + .filter((episode): episode is GraphitiEpisode => episode !== undefined) + const provenanceIncomplete = episodeUuids.some((uuid) => !byId.has(uuid)) + if (sourceEpisodes.length === 0) { + if (provenanceIncomplete) { + hit.metadata = { ...hit.metadata, provenanceIncomplete: true } + } + continue + } + const descriptions = sourceEpisodes + .map((episode) => parseDescription(episode.source_description)) + .filter(isRecord) + const first = descriptions[0] + hit.metadata = { + ...hit.metadata, + sourceEpisodes, + ...(provenanceIncomplete || !scan.complete ? { provenanceIncomplete: true } : {}), + ...(first ?? {}), + eventIds: uniqueStrings(descriptions.map((value) => value.eventId)), + actorIds: uniqueStrings(descriptions.map((value) => value.actorId)), + ...(typeof first?.eventId === 'string' ? { eventId: first.eventId } : {}), + ...(typeof first?.actorId === 'string' ? { actorId: first.actorId } : {}), + } + } +} + +async function getGraphitiEpisodes( + options: GraphitiMemoryAdapterOptions, + groupId: string, + limit: number, +): Promise { + const raw = await callGraphiti(options.client, graphitiToolNames(options).getEpisodes, { + group_ids: [groupId], + max_episodes: limit, + }) + if (!isRecord(raw) || !Array.isArray(raw.episodes)) return [] + return raw.episodes.filter(isRecord).flatMap((episode) => { + const uuid = stringField(episode, ['uuid', 'id']) + return uuid ? [{ ...episode, uuid } as GraphitiEpisode] : [] + }) +} + +async function scanGraphitiEpisodes( + options: GraphitiMemoryAdapterOptions, + groupId: string, + wanted: ReadonlySet, + maximum: number, +): Promise<{ episodes: GraphitiEpisode[]; complete: boolean }> { + let limit = Math.min(100, maximum) + for (;;) { + const episodes = await getGraphitiEpisodes(options, groupId, limit) + const foundAll = [...wanted].every((uuid) => episodes.some((episode) => episode.uuid === uuid)) + if (foundAll || episodes.length < limit) return { episodes, complete: foundAll } + if (limit === maximum) return { episodes, complete: false } + limit = Math.min(maximum, limit * 2) + } +} + +async function waitForGraphitiEpisode( + options: GraphitiMemoryAdapterOptions, + groupId: string, + episodeUuid: string, +): Promise { + const timeoutMs = options.ingestionTimeoutMs ?? 120_000 + const pollIntervalMs = options.pollIntervalMs ?? 500 + const maximum = options.episodeScanLimit ?? 10_000 + const deadline = Date.now() + timeoutMs + let scanLimitReached = false + for (;;) { + const scan = await scanGraphitiEpisodes(options, groupId, new Set([episodeUuid]), maximum) + if (scan.complete) return + scanLimitReached ||= scan.episodes.length >= maximum + const remainingMs = deadline - Date.now() + if (remainingMs <= 0) break + await new Promise((resolve) => setTimeout(resolve, Math.min(pollIntervalMs, remainingMs))) + } + if (scanLimitReached) { + throw new Error( + `Graphiti episode ${episodeUuid} is outside episodeScanLimit=${maximum}; raise the limit or use a smaller group`, + ) + } + throw new Error(`Graphiti episode ${episodeUuid} was not visible after ${timeoutMs}ms`) +} + +function graphitiGroupId(adapterId: string, scope: AgentMemoryScope): string { + return `ak_${sha256( + canonicalJson( + stripUndefined({ + adapterId, + tenantId: scope.tenantId, + userId: scope.userId, + agentId: scope.agentId, + teamId: scope.teamId, + runId: scope.runId, + sessionId: scope.sessionId, + namespace: scope.namespace, + tags: scope.tags, + }), + ), + ).slice(0, 32)}` +} + +function graphitiToolNames(options: GraphitiMemoryAdapterOptions): GraphitiToolNames { + return { + addMemory: options.toolNames?.addMemory ?? 'add_memory', + searchFacts: options.toolNames?.searchFacts ?? 'search_memory_facts', + searchNodes: options.toolNames?.searchNodes ?? 'search_nodes', + getEpisodes: options.toolNames?.getEpisodes ?? 'get_episodes', + clearGraph: options.toolNames?.clearGraph ?? 'clear_graph', + } +} + +function assertGraphitiOptions(options: GraphitiMemoryAdapterOptions): void { + if (options.id !== undefined && (typeof options.id !== 'string' || !options.id.trim())) { + throw new Error('Graphiti id must be a non-empty string') + } + if (options.consistency !== undefined && !['queued', 'visible'].includes(options.consistency)) { + throw new Error('Graphiti consistency must be queued or visible') + } + for (const [name, value] of [ + ['ingestionTimeoutMs', options.ingestionTimeoutMs], + ['pollIntervalMs', options.pollIntervalMs], + ['episodeScanLimit', options.episodeScanLimit], + ['provenanceEpisodeLimit', options.provenanceEpisodeLimit], + ] as const) { + if (value !== undefined && (!Number.isSafeInteger(value) || value <= 0)) { + throw new Error(`Graphiti ${name} must be a positive safe integer`) + } + } + for (const [name, value] of Object.entries(options.toolNames ?? {})) { + if (typeof value !== 'string' || value.trim().length === 0) { + throw new Error(`Graphiti toolNames.${name} must be a non-empty string`) + } + } +} + +function deterministicUuid(value: string): string { + const hex = sha256(value) + return `${hex.slice(0, 8)}-${hex.slice(8, 12)}-5${hex.slice(13, 16)}-a${hex.slice(17, 20)}-${hex.slice(20, 32)}` +} + +function parseDescription(value: string | undefined): unknown { + if (!value) return undefined + try { + return JSON.parse(value) + } catch { + return undefined + } +} + +function mergeScopes(base?: AgentMemoryScope, extra?: AgentMemoryScope): AgentMemoryScope { + return { + ...(base ?? {}), + ...(extra ?? {}), + tags: { ...(base?.tags ?? {}), ...(extra?.tags ?? {}) }, + } +} + +function uniqueStrings(values: unknown[]): string[] { + return [...new Set(values.filter((value): value is string => typeof value === 'string'))] +} + +function stringArray(value: unknown): string[] { + return Array.isArray(value) + ? value.filter((entry): entry is string => typeof entry === 'string') + : [] +} + +function stringField(value: Record, keys: readonly string[]): string | undefined { + for (const key of keys) { + const field = value[key] + if (typeof field === 'string' && field.length > 0) return field + } + return undefined +} + +function numberField(value: Record, keys: readonly string[]): number | undefined { + for (const key of keys) { + const field = value[key] + if (typeof field === 'number' && Number.isFinite(field)) return field + } + return undefined +} + +function normalizedScoreOf(score: number | undefined): number | undefined { + return score !== undefined && score >= 0 && score <= 1 ? score : undefined +} + +function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value) +} + +function stripUndefined(value: unknown): unknown { + if (Array.isArray(value)) return value.map(stripUndefined) + if (!isRecord(value)) return value + return Object.fromEntries( + Object.entries(value) + .filter(([, child]) => child !== undefined) + .map(([key, child]) => [key, stripUndefined(child)]), + ) +} + +export function graphitiMemoryAdapterIdentity( + options: Pick< + GraphitiMemoryAdapterOptions, + | 'id' + | 'consistency' + | 'ingestionTimeoutMs' + | 'pollIntervalMs' + | 'episodeScanLimit' + | 'provenanceEpisodeLimit' + | 'search' + | 'toolNames' + | 'defaultScope' + >, +): string { + return stableId('graphiti', canonicalJson(stripUndefined(options))) +} diff --git a/src/memory/improvement.ts b/src/memory/improvement.ts new file mode 100644 index 0000000..ef9b483 --- /dev/null +++ b/src/memory/improvement.ts @@ -0,0 +1,1012 @@ +import { join } from 'node:path' +import { canonicalJson } from '@tangle-network/agent-eval' +import { + acquireSingleRunLock, + type CampaignStorage, + type CostLedgerHandle, + campaignLineageStore, + createRunCostLedger, + fsCampaignStorage, + type Governor, + type GovernorContext, + type GovernorOp, + type HeldoutSignificance, + type HeldoutSignificanceOptions, + heldoutSignificance, + type Lineage, + type LineageStore, + type MutableSurface, + memLineageStore, + type PairedHoldout, + resolveRunDir, + runLineageLoop, + type Scenario, + type SurfaceProposer, + surfaceHash, +} from '@tangle-network/agent-eval/campaign' +import type { + AgentMemoryExperimentCandidate, + AgentMemorySequence, + AgentMemorySequenceArtifact, + RunAgentMemoryExperimentOptions, + RunAgentMemoryExperimentResult, +} from './experiment' +import { runAgentMemoryExperiment } from './experiment' + +export interface AgentMemoryImprovementSeed { + config: TConfig + track: string + vision?: string + proposer: string +} + +export interface AgentMemoryDimensionComparison { + dimension: string + n: number + measured: boolean + meanDelta: number + low: number + high: number + tolerance: number + regressed: boolean +} + +export interface AgentMemoryPromotionDecision { + status: 'promote' | 'hold' | 'no-change' + reasons: readonly string[] + baselineScore: number + winnerScore: number + lift: number + significance?: HeldoutSignificance + criticalDimensions: readonly AgentMemoryDimensionComparison[] +} + +export interface AgentMemoryActivation { + id: string + status: 'not-eligible' | 'not-configured' | 'pending' | 'activated' | 'already-activated' + receiptPath: string +} + +export interface AgentMemoryImprovementRunLease { + assertOwned(): void | Promise + release(): void | Promise +} + +export interface AgentMemoryGovernor { + decide( + context: GovernorContext & { + costLedger: CostLedgerHandle + costPhase: string + }, + ): GovernorOp | Promise +} + +export interface RunAgentMemoryImprovementOptions { + experimentId: string + trainSequences: readonly AgentMemorySequence[] + holdoutSequences: readonly AgentMemorySequence[] + /** First entry is the current baseline; remaining entries seed independent search tracks. */ + seeds: readonly AgentMemoryImprovementSeed[] + createCandidate(input: { + config: TConfig + candidateId: string + surfaceHash: string + }): + | Omit + | Promise> + proposer: SurfaceProposer + /** Optional proposer implementations keyed by seed and branch proposer labels. */ + proposers?: Readonly> + /** Stable version or commit for the candidate factory, proposer, and governor. */ + improvementRef: string + governor?: AgentMemoryGovernor + budget: { maxSteps: number } + populationSize?: number + candidateConcurrency?: number + sequenceConcurrency?: number + runDir: string + repo?: string + storage?: CampaignStorage + lineageStore?: LineageStore + /** Required for distributed controllers using custom storage. Worker concurrency is independent. */ + acquireRunLease?: (input: { + experimentId: string + runDir: string + }) => AgentMemoryImprovementRunLease | Promise + seed?: number + reps?: number + resumable?: boolean + dispatchTimeoutMs?: number + maxTotalCostUsd?: number + executeStep?: RunAgentMemoryExperimentOptions['executeStep'] + executeStepRef?: string + onBranchSnapshot?: RunAgentMemoryExperimentOptions['onBranchSnapshot'] + cleanupBranches?: boolean + serializeConfig?: (config: TConfig) => string + parseConfig?: (surface: string) => TConfig + significance?: HeldoutSignificanceOptions + criticalDimensions?: readonly string[] + criticalDimensionTolerance?: number + minHoldoutScore?: number + onPromote?: (input: { + activationId: string + config: TConfig + decision: AgentMemoryPromotionDecision + lineage: Lineage + holdout: RunAgentMemoryExperimentResult + }) => Promise + now?: () => Date +} + +export interface RunAgentMemoryImprovementResult { + lineage: Lineage + baselineConfig: TConfig + winnerConfig: TConfig + baselineSurface: string + winnerSurface: string + baselineSurfaceHash: string + winnerSurfaceHash: string + decision: AgentMemoryPromotionDecision + activation: AgentMemoryActivation + holdout?: RunAgentMemoryExperimentResult + totalCostUsd: number + resultJsonPath: string +} + +interface MemoryConfigScenario extends Scenario { + kind: 'agent-memory-config-search' + sequenceId: string +} + +const DEFAULT_CRITICAL_DIMENSIONS = [ + 'memory_stale_safe', + 'memory_actor_recall', + 'memory_event_recall', +] as const + +/** Searches branchable memory configurations and activates only a fresh holdout win. */ +export async function runAgentMemoryImprovement( + options: RunAgentMemoryImprovementOptions, +): Promise> { + if (options.seeds.length === 0) throw new Error('memory improvement requires seed configs') + if (options.trainSequences.length === 0) { + throw new Error('memory improvement requires training sequences') + } + if (options.holdoutSequences.length === 0) { + throw new Error('memory improvement requires holdout sequences') + } + const trainIds = new Set(options.trainSequences.map((sequence) => sequence.id)) + const overlap = options.holdoutSequences + .map((sequence) => sequence.id) + .filter((id) => trainIds.has(id)) + if (overlap.length > 0) { + throw new Error(`memory improvement train/holdout overlap: ${overlap.join(', ')}`) + } + const trainFingerprints = new Map( + options.trainSequences.map((sequence) => [memorySequenceFingerprint(sequence), sequence.id]), + ) + const duplicateHistories = options.holdoutSequences.flatMap((sequence) => { + const trainId = trainFingerprints.get(memorySequenceFingerprint(sequence)) + return trainId ? [`${trainId}/${sequence.id}`] : [] + }) + if (duplicateHistories.length > 0) { + throw new Error( + `memory improvement train/holdout histories duplicate content: ${duplicateHistories.join(', ')}`, + ) + } + if (typeof options.improvementRef !== 'string' || !options.improvementRef.trim()) { + throw new Error('memory improvement improvementRef must be a non-empty string') + } + assertMemoryImprovementOptions(options) + const storage = options.storage ?? fsCampaignStorage() + const runDir = resolveRunDir(options.runDir, options.repo) + storage.ensureDir(runDir) + const lease = await acquireMemoryImprovementRunLease(options, storage, runDir) + let result: RunAgentMemoryImprovementResult | undefined + let primaryError: unknown + try { + result = await runAgentMemoryImprovementOwned(options, storage, runDir, lease) + } catch (error) { + primaryError = error + } + let releaseError: unknown + try { + await lease.release() + } catch (error) { + releaseError = error + } + if (primaryError && releaseError) { + throw new AggregateError( + [primaryError, releaseError], + 'memory improvement failed and its controller lease could not be released', + ) + } + if (primaryError) throw primaryError + if (releaseError) throw releaseError + if (!result) throw new Error('memory improvement produced no result') + return result +} + +async function runAgentMemoryImprovementOwned( + options: RunAgentMemoryImprovementOptions, + storage: CampaignStorage, + runDir: string, + lease: OwnedRunLease, +): Promise> { + await lease.assertOwned() + const serializeRaw = options.serializeConfig ?? ((config: TConfig) => canonicalJson(config)) + const parseRaw = options.parseConfig ?? ((surface: string) => JSON.parse(surface) as TConfig) + const serialize = (config: TConfig): string => serializeMemoryConfig(serializeRaw, config) + const parse = (surface: string): TConfig => parseMemoryConfig(parseRaw, surface) + for (const seed of options.seeds) assertMemoryConfigRoundTrip(seed.config, serialize, parse) + if (options.resumable !== false && !options.lineageStore && !storage.append) { + throw new Error('resumable memory improvement requires CampaignStorage.append') + } + assertMemoryImprovementIdentity(options, storage, runDir, serialize) + const costLedger = createRunCostLedger({ + storage, + runDir, + costCeilingUsd: options.maxTotalCostUsd, + }) + const trainScenarios: MemoryConfigScenario[] = options.trainSequences.map((sequence) => ({ + id: sequence.id, + kind: 'agent-memory-config-search', + sequenceId: sequence.id, + })) + const evaluations = new Map>() + + const evaluateSurface = async ( + surface: MutableSurface, + ): Promise<{ score: number; scoreVector: number[] }> => { + await lease.assertOwned() + const text = requireStringSurface(surface) + const config = parse(text) + const canonicalSurface = serialize(config) + const hash = surfaceHash(canonicalSurface) + let pending = evaluations.get(hash) + if (!pending) { + pending = (async () => { + const candidate = await buildCandidate(options, config, hash, `search-${hash}`) + return runAgentMemoryExperiment({ + ...experimentOptions(options, costLedger, storage), + experimentId: `${options.experimentId}:search:${hash}`, + sequences: options.trainSequences, + candidates: [candidate], + runDir: join(runDir, 'search', hash), + costPhase: `memory.search.${hash}`, + }) + })() + evaluations.set(hash, pending) + void pending.catch(() => evaluations.delete(hash)) + } + const result = await pending + const row = result.rows[0] + if (!row || row.cellsFailed > 0) { + throw new Error(`${hash}: memory candidate did not complete every training cell`) + } + return { + score: row.scoreMean, + scoreVector: sequenceScores(result, options.trainSequences, row.candidateId), + } + } + + const lineageStore = + options.lineageStore ?? + (options.resumable === false + ? memLineageStore() + : campaignLineageStore(storage, join(runDir, 'lineage.jsonl'))) + const lineageOptions = { + seeds: options.seeds.map((seed) => ({ + surface: serialize(seed.config), + track: seed.track, + proposer: seed.proposer, + ...(seed.vision !== undefined ? { vision: seed.vision } : {}), + })), + scenarios: trainScenarios, + proposer: withCostContext(options.proposer, costLedger, 'default'), + proposers: options.proposers + ? Object.fromEntries( + Object.entries(options.proposers).map(([name, proposer]) => [ + name, + withCostContext(proposer, costLedger, name), + ]), + ) + : undefined, + scoreSurface: evaluateSurface, + governor: options.governor ? withGovernorCostContext(options.governor, costLedger) : undefined, + budget: { + ...options.budget, + maxNodes: options.seeds.length + options.budget.maxSteps, + }, + store: lineageStore, + populationSize: options.populationSize, + candidateConcurrency: options.candidateConcurrency, + } + const search = await runLineageLoop(lineageOptions) + await lease.assertOwned() + const best = search.best + if (!best) throw new Error('memory improvement produced no measured config') + + const baselineSurface = serialize(options.seeds[0]!.config) + const baselineHash = surfaceHash(baselineSurface) + const winnerConfig = parse(requireStringSurface(best.surface)) + const winnerSurface = serialize(winnerConfig) + const winnerHash = surfaceHash(winnerSurface) + const baselineConfig = parse(baselineSurface) + + let holdout: RunAgentMemoryExperimentResult | undefined + let decision: AgentMemoryPromotionDecision + if (winnerHash === baselineHash) { + const baselineMeasurement = await evaluateSurface(baselineSurface) + decision = { + status: 'no-change', + reasons: ['search did not find a config better than the baseline'], + baselineScore: baselineMeasurement.score, + winnerScore: baselineMeasurement.score, + lift: 0, + criticalDimensions: [], + } + } else { + await lease.assertOwned() + const [baselineCandidate, winnerCandidate] = await Promise.all([ + buildCandidate(options, baselineConfig, baselineHash, 'baseline'), + buildCandidate(options, winnerConfig, winnerHash, 'winner'), + ]) + holdout = await runAgentMemoryExperiment({ + ...experimentOptions(options, costLedger, storage), + experimentId: `${options.experimentId}:holdout`, + sequences: options.holdoutSequences, + candidates: [baselineCandidate, winnerCandidate], + runDir: join(runDir, 'holdout'), + costPhase: 'memory.holdout', + }) + decision = decidePromotion({ + options, + result: holdout, + baselineId: baselineCandidate.id, + winnerId: winnerCandidate.id, + }) + } + + const resultJsonPath = join(runDir, 'memory-improvement-result.json') + const activationId = `memory-activation-${surfaceHash( + canonicalJson({ + experimentId: options.experimentId, + improvementRef: options.improvementRef, + baselineSurfaceHash: baselineHash, + winnerSurfaceHash: winnerHash, + holdoutManifestHash: holdout?.campaign.manifestHash ?? null, + promotionPolicy: normalizedPromotionPolicy(options), + }), + )}` + const activationReceiptDir = join(runDir, 'activations') + const activationReceiptPath = join(activationReceiptDir, `${activationId}.json`) + const activationEligible = decision.status === 'promote' && holdout !== undefined + const alreadyActivated = + activationEligible && + hasCompletedActivation(storage, activationReceiptPath, activationId, winnerHash) + const activation: AgentMemoryActivation = { + id: activationId, + status: !activationEligible + ? 'not-eligible' + : alreadyActivated + ? 'already-activated' + : options.onPromote + ? 'pending' + : 'not-configured', + receiptPath: activationReceiptPath, + } + const result = { + lineage: search.lineage, + baselineConfig, + winnerConfig, + baselineSurface, + winnerSurface, + baselineSurfaceHash: baselineHash, + winnerSurfaceHash: winnerHash, + decision, + activation, + ...(holdout ? { holdout } : {}), + totalCostUsd: costLedger.summary().totalCostUsd, + resultJsonPath, + } satisfies RunAgentMemoryImprovementResult + await lease.assertOwned() + writeMemoryImprovementResult(storage, options.experimentId, result) + if (activationEligible && !alreadyActivated && holdout && options.onPromote) { + await lease.assertOwned() + await options.onPromote({ + activationId, + config: winnerConfig, + decision, + lineage: search.lineage, + holdout, + }) + await lease.assertOwned() + storage.ensureDir(activationReceiptDir) + storage.write( + activationReceiptPath, + `${JSON.stringify( + { + activationId, + experimentId: options.experimentId, + winnerSurfaceHash: winnerHash, + holdoutManifestHash: holdout.campaign.manifestHash, + status: 'activated', + activatedAt: (options.now ?? (() => new Date()))().toISOString(), + }, + null, + 2, + )}\n`, + ) + activation.status = 'activated' + writeMemoryImprovementResult(storage, options.experimentId, result) + } + return result +} + +function withCostContext( + proposer: SurfaceProposer, + costLedger: CostLedgerHandle, + label: string, +): SurfaceProposer { + return { + kind: proposer.kind, + async propose(context) { + return proposer.propose({ + ...context, + costLedger, + costPhase: `memory.proposal.${context.track?.id ?? label}`, + }) + }, + ...(proposer.decide ? { decide: (input) => proposer.decide!(input) } : {}), + } +} + +function withGovernorCostContext( + governor: AgentMemoryGovernor, + costLedger: CostLedgerHandle, +): Governor { + return { + decide(context) { + return governor.decide({ + ...context, + costLedger, + costPhase: 'memory.governor', + }) + }, + } +} + +async function buildCandidate( + options: RunAgentMemoryImprovementOptions, + config: TConfig, + hash: string, + role: string, +): Promise { + const built = await options.createCandidate({ + config, + candidateId: `${role}-${hash}`, + surfaceHash: hash, + }) + return { + ...built, + id: `${role}-${hash}`, + ref: built.ref ?? hash, + } +} + +function experimentOptions( + options: RunAgentMemoryImprovementOptions, + costLedger: ReturnType, + storage: CampaignStorage, +): Pick< + RunAgentMemoryExperimentOptions, + | 'storage' + | 'seed' + | 'reps' + | 'resumable' + | 'maxConcurrency' + | 'dispatchTimeoutMs' + | 'executeStep' + | 'executeStepRef' + | 'onBranchSnapshot' + | 'cleanupBranches' + | 'costLedger' + | 'now' +> { + return { + storage, + seed: options.seed, + reps: options.reps, + resumable: options.resumable, + maxConcurrency: options.sequenceConcurrency, + dispatchTimeoutMs: options.dispatchTimeoutMs, + executeStep: options.executeStep, + executeStepRef: options.executeStepRef, + onBranchSnapshot: options.onBranchSnapshot, + cleanupBranches: options.cleanupBranches ?? true, + costLedger, + now: options.now, + } +} + +interface OwnedRunLease { + assertOwned(): Promise + release(): Promise +} + +const activeCustomStorageRuns = new WeakMap>() + +function assertMemoryImprovementIdentity( + options: RunAgentMemoryImprovementOptions, + storage: CampaignStorage, + runDir: string, + serialize: (config: TConfig) => string, +): void { + const path = join(runDir, 'memory-improvement-manifest.json') + const identity = { + schema: 3, + experimentId: options.experimentId, + improvementRef: options.improvementRef, + proposerKind: options.proposer.kind, + proposerKinds: Object.fromEntries( + Object.entries(options.proposers ?? {}) + .sort(([a], [b]) => a.localeCompare(b)) + .map(([name, proposer]) => [name, proposer.kind]), + ), + populationSize: options.populationSize ?? 4, + budget: { maxSteps: options.budget.maxSteps }, + executeStepRef: options.executeStepRef ?? null, + cleanupBranches: options.cleanupBranches ?? true, + promotionPolicy: normalizedPromotionPolicy(options), + seed: options.seed ?? null, + reps: options.reps ?? 1, + seeds: options.seeds.map((entry) => ({ + config: serialize(entry.config), + track: entry.track, + vision: entry.vision ?? null, + proposer: entry.proposer, + })), + trainSequences: options.trainSequences, + holdoutSequences: options.holdoutSequences, + } + const identityHash = surfaceHash(canonicalJson(identity)) + const stored = storage.read(path) + if (stored === undefined) { + if (storage.exists(path)) throw new Error(`cannot read memory improvement manifest '${path}'`) + if ( + storage.exists(join(runDir, 'lineage.jsonl')) || + storage.exists(join(runDir, 'memory-improvement-result.json')) + ) { + throw new Error(`memory improvement run '${runDir}' has state without an identity manifest`) + } + storage.write(path, `${JSON.stringify({ identityHash, identity }, null, 2)}\n`) + return + } + let manifest: unknown + try { + manifest = JSON.parse(stored) + } catch (error) { + throw new Error(`invalid memory improvement manifest '${path}'`, { cause: error }) + } + if ( + !manifest || + typeof manifest !== 'object' || + (manifest as Record).identityHash !== identityHash || + canonicalJson((manifest as Record).identity) !== canonicalJson(identity) + ) { + throw new Error( + `memory improvement run '${runDir}' does not match its persisted inputs or implementationRef`, + ) + } +} + +async function acquireMemoryImprovementRunLease( + options: RunAgentMemoryImprovementOptions, + storage: CampaignStorage, + runDir: string, +): Promise { + if (options.acquireRunLease) { + const lease = await options.acquireRunLease({ + experimentId: options.experimentId, + runDir, + }) + if (!lease || typeof lease.release !== 'function') { + throw new Error('memory improvement acquireRunLease must return a releasable lease') + } + if (typeof lease.assertOwned !== 'function') { + throw new Error('memory improvement acquireRunLease must return assertOwned') + } + return { + async assertOwned() { + await lease.assertOwned() + }, + async release() { + await lease.release() + }, + } + } + + if (options.storage === undefined) { + const lock = acquireSingleRunLock({ + lockPath: join(runDir, 'memory-improvement.lock'), + releaseOnExit: false, + }) + return { + async assertOwned() {}, + async release() { + lock.release() + }, + } + } + + const active = activeCustomStorageRuns.get(storage) ?? new Set() + if (active.has(runDir)) { + throw new Error(`memory improvement run '${runDir}' already has an active controller`) + } + active.add(runDir) + activeCustomStorageRuns.set(storage, active) + let released = false + return { + async assertOwned() { + if (released || !active.has(runDir)) { + throw new Error(`memory improvement run '${runDir}' controller lease was lost`) + } + }, + async release() { + if (released) return + released = true + active.delete(runDir) + if (active.size === 0) activeCustomStorageRuns.delete(storage) + }, + } +} + +function hasCompletedActivation( + storage: CampaignStorage, + path: string, + activationId: string, + winnerSurfaceHash: string, +): boolean { + const stored = storage.read(path) + if (stored === undefined) { + if (storage.exists(path)) throw new Error(`cannot read activation receipt '${path}'`) + return false + } + let receipt: unknown + try { + receipt = JSON.parse(stored) + } catch (error) { + throw new Error(`invalid activation receipt '${path}'`, { cause: error }) + } + if ( + !receipt || + typeof receipt !== 'object' || + (receipt as Record).activationId !== activationId || + (receipt as Record).winnerSurfaceHash !== winnerSurfaceHash || + (receipt as Record).status !== 'activated' + ) { + throw new Error(`activation receipt '${path}' does not match the measured winner`) + } + return true +} + +function writeMemoryImprovementResult( + storage: CampaignStorage, + experimentId: string, + result: RunAgentMemoryImprovementResult, +): void { + storage.write( + result.resultJsonPath, + `${JSON.stringify( + { + experimentId, + baselineSurfaceHash: result.baselineSurfaceHash, + winnerSurfaceHash: result.winnerSurfaceHash, + baselineSurface: result.baselineSurface, + winnerSurface: result.winnerSurface, + decision: result.decision, + activation: result.activation, + totalCostUsd: result.totalCostUsd, + lineage: result.lineage.toGraph(), + }, + null, + 2, + )}\n`, + ) +} + +function decidePromotion(input: { + options: RunAgentMemoryImprovementOptions + result: RunAgentMemoryExperimentResult + baselineId: string + winnerId: string +}): AgentMemoryPromotionDecision { + const { options, result, baselineId, winnerId } = input + const baselineRow = result.rows.find((row) => row.candidateId === baselineId) + const winnerRow = result.rows.find((row) => row.candidateId === winnerId) + if (!baselineRow || !winnerRow) throw new Error('holdout result is missing a comparison arm') + const paired = pairedArtifacts(result, baselineId, winnerId, (artifact) => artifact.score) + const significance = heldoutSignificance(paired, options.significance) + const tolerance = options.criticalDimensionTolerance ?? 0.05 + const expectedPairCount = options.holdoutSequences.length * (options.reps ?? 1) + const criticalDimensions = (options.criticalDimensions ?? DEFAULT_CRITICAL_DIMENSIONS).map( + (dimension) => { + const dimensionPairs = pairedArtifacts( + result, + baselineId, + winnerId, + (artifact) => artifact.dimensions[dimension], + ) + const comparison = heldoutSignificance(dimensionPairs, { + ...options.significance, + deltaThreshold: 0, + }) + return { + dimension, + n: comparison.n, + measured: comparison.n === expectedPairCount, + meanDelta: comparison.bootstrap.mean, + low: comparison.bootstrap.low, + high: comparison.bootstrap.high, + tolerance, + regressed: comparison.bootstrap.low < -tolerance, + } + }, + ) + const reasons: string[] = [] + if (baselineRow.cellsFailed > 0 || winnerRow.cellsFailed > 0) { + reasons.push('at least one holdout cell failed') + } + if (!significance.significant) { + reasons.push( + significance.fewRuns + ? `only ${significance.n} paired holdout cells; more are required` + : 'holdout lift is not confidently above the promotion threshold', + ) + } + if (winnerRow.scoreMean < (options.minHoldoutScore ?? 0)) { + reasons.push(`winner holdout score ${winnerRow.scoreMean} is below the required minimum`) + } + for (const dimension of criticalDimensions) { + if (!dimension.measured) { + reasons.push( + `critical dimension ${dimension.dimension} was measured on ${dimension.n}/${expectedPairCount} paired holdout cells`, + ) + } else if (dimension.regressed) { + reasons.push(`${dimension.dimension} may regress beyond ${tolerance}`) + } + } + return { + status: reasons.length === 0 ? 'promote' : 'hold', + reasons, + baselineScore: baselineRow.scoreMean, + winnerScore: winnerRow.scoreMean, + lift: winnerRow.scoreMean - baselineRow.scoreMean, + significance, + criticalDimensions, + } +} + +function normalizedPromotionPolicy( + options: RunAgentMemoryImprovementOptions, +): Record { + return { + significance: { + deltaThreshold: options.significance?.deltaThreshold ?? 0, + minProductiveRuns: options.significance?.minProductiveRuns ?? 3, + confidence: options.significance?.confidence ?? 0.95, + resamples: options.significance?.resamples ?? 2000, + seed: options.significance?.seed ?? 1337, + statistic: options.significance?.statistic ?? 'mean', + }, + criticalDimensions: [...(options.criticalDimensions ?? DEFAULT_CRITICAL_DIMENSIONS)], + criticalDimensionTolerance: options.criticalDimensionTolerance ?? 0.05, + minHoldoutScore: options.minHoldoutScore ?? 0, + } +} + +function pairedArtifacts( + result: RunAgentMemoryExperimentResult, + baselineId: string, + winnerId: string, + select: (artifact: AgentMemorySequenceArtifact) => number | undefined, +): PairedHoldout { + const baseline = artifactValues(result, baselineId, select) + const winner = artifactValues(result, winnerId, select) + const keys = [...baseline.keys()].filter((key) => winner.has(key)).sort() + return { + before: keys.map((key) => baseline.get(key)!), + after: keys.map((key) => winner.get(key)!), + cellIds: keys, + } +} + +function artifactValues( + result: RunAgentMemoryExperimentResult, + candidateId: string, + select: (artifact: AgentMemorySequenceArtifact) => number | undefined, +): Map { + const values = new Map() + for (const cell of result.campaign.cells) { + if (cell.error || cell.artifact.candidateId !== candidateId) continue + const value = select(cell.artifact) + if (value === undefined || !Number.isFinite(value)) continue + values.set(`${cell.artifact.sequenceId}:${cell.rep}`, value) + } + return values +} + +function sequenceScores( + result: RunAgentMemoryExperimentResult, + sequences: readonly AgentMemorySequence[], + candidateId: string, +): number[] { + const bySequence = new Map() + for (const cell of result.campaign.cells) { + if (cell.error || cell.artifact.candidateId !== candidateId) continue + const bucket = bySequence.get(cell.artifact.sequenceId) ?? [] + bucket.push(cell.artifact.score) + bySequence.set(cell.artifact.sequenceId, bucket) + } + return sequences.map((sequence) => mean(bySequence.get(sequence.id) ?? [])) +} + +function requireStringSurface(surface: MutableSurface): string { + if (typeof surface !== 'string' || surface.trim().length === 0) { + throw new Error('memory config proposer must return a JSON string surface') + } + return surface +} + +function serializeMemoryConfig( + serialize: (config: TConfig) => string, + config: TConfig, +): string { + let surface: unknown + try { + surface = serialize(config) + } catch (error) { + throw new Error('memory config serializer failed', { cause: error }) + } + if (typeof surface !== 'string' || surface.trim().length === 0) { + throw new Error('memory config serializer must return a non-empty string') + } + return surface +} + +function parseMemoryConfig(parse: (surface: string) => TConfig, surface: string): TConfig { + try { + const config = parse(surface) + if (config === undefined) throw new Error('parser returned undefined') + return config + } catch (error) { + throw new Error('memory config surface could not be parsed', { cause: error }) + } +} + +function assertMemoryConfigRoundTrip( + config: TConfig, + serialize: (config: TConfig) => string, + parse: (surface: string) => TConfig, +): void { + const first = serialize(config) + const second = serialize(parse(first)) + if (first !== second) { + throw new Error('memory config serializer and parser must round-trip seed configs exactly') + } +} + +function memorySequenceFingerprint(sequence: AgentMemorySequence): string { + const { id: _id, split: _split, tags: _tags, ...content } = sequence + return surfaceHash(canonicalJson(content)) +} + +function assertMemoryImprovementOptions( + options: RunAgentMemoryImprovementOptions, +): void { + for (const [name, value] of [ + ['experimentId', options.experimentId], + ['runDir', options.runDir], + ] as const) { + if (typeof value !== 'string' || !value.trim()) { + throw new Error(`memory improvement ${name} must be a non-empty string`) + } + } + if (typeof options.proposer?.kind !== 'string' || !options.proposer.kind.trim()) { + throw new Error('memory improvement proposer.kind must be a non-empty string') + } + if (typeof options.proposer?.propose !== 'function') { + throw new Error('memory improvement proposer.propose must be a function') + } + if (options.governor !== undefined && typeof options.governor.decide !== 'function') { + throw new Error('memory improvement governor.decide must be a function') + } + for (const [name, proposer] of Object.entries(options.proposers ?? {})) { + if (!name.trim()) throw new Error('memory improvement proposer labels must be non-empty') + if ( + !proposer || + typeof proposer.kind !== 'string' || + !proposer.kind.trim() || + typeof proposer.propose !== 'function' + ) { + throw new Error(`memory improvement proposer '${name}' is invalid`) + } + } + if (options.serializeConfig !== undefined && typeof options.serializeConfig !== 'function') { + throw new Error('memory improvement serializeConfig must be a function') + } + if (options.parseConfig !== undefined && typeof options.parseConfig !== 'function') { + throw new Error('memory improvement parseConfig must be a function') + } + if (!Number.isSafeInteger(options.budget.maxSteps) || options.budget.maxSteps < 0) { + throw new Error('memory improvement budget.maxSteps must be a non-negative safe integer') + } + for (const [name, value] of [ + ['populationSize', options.populationSize], + ['candidateConcurrency', options.candidateConcurrency], + ['sequenceConcurrency', options.sequenceConcurrency], + ['reps', options.reps], + ] as const) { + if (value !== undefined && (!Number.isSafeInteger(value) || value <= 0)) { + throw new Error(`memory improvement ${name} must be a positive safe integer`) + } + } + if ( + options.maxTotalCostUsd !== undefined && + (!Number.isFinite(options.maxTotalCostUsd) || options.maxTotalCostUsd < 0) + ) { + throw new Error('memory improvement maxTotalCostUsd must be a non-negative finite number') + } + const tolerance = options.criticalDimensionTolerance + if (tolerance !== undefined && (!Number.isFinite(tolerance) || tolerance < 0 || tolerance > 1)) { + throw new Error('memory improvement criticalDimensionTolerance must be between 0 and 1') + } + const minimum = options.minHoldoutScore + if (minimum !== undefined && (!Number.isFinite(minimum) || minimum < 0 || minimum > 1)) { + throw new Error('memory improvement minHoldoutScore must be between 0 and 1') + } + for (const seed of options.seeds) { + if ( + typeof seed.track !== 'string' || + !seed.track.trim() || + typeof seed.proposer !== 'string' || + !seed.proposer.trim() + ) { + throw new Error('memory improvement seeds require non-empty track and proposer values') + } + if (seed.vision !== undefined && (typeof seed.vision !== 'string' || !seed.vision.trim())) { + throw new Error('memory improvement seed vision must be a non-empty string when provided') + } + } + const dimensions = options.criticalDimensions ?? DEFAULT_CRITICAL_DIMENSIONS + if (dimensions.some((dimension) => typeof dimension !== 'string' || !dimension.trim())) { + throw new Error('memory improvement criticalDimensions must contain non-empty names') + } + if (new Set(dimensions).size !== dimensions.length) { + throw new Error('memory improvement criticalDimensions must be unique') + } + assertDistinctSequenceContent(options.trainSequences, 'train') + assertDistinctSequenceContent(options.holdoutSequences, 'holdout') +} + +function assertDistinctSequenceContent( + sequences: readonly AgentMemorySequence[], + split: string, +): void { + const idsByFingerprint = new Map() + for (const sequence of sequences) { + const fingerprint = memorySequenceFingerprint(sequence) + const prior = idsByFingerprint.get(fingerprint) + if (prior) { + throw new Error( + `memory improvement ${split} histories duplicate content: ${prior}/${sequence.id}`, + ) + } + idsByFingerprint.set(fingerprint, sequence.id) + } +} + +function mean(values: readonly number[]): number { + return values.length === 0 ? 0 : values.reduce((sum, value) => sum + value, 0) / values.length +} diff --git a/src/memory/index.ts b/src/memory/index.ts index da6fffb..92b698c 100644 --- a/src/memory/index.ts +++ b/src/memory/index.ts @@ -1,5 +1,10 @@ export * from './adapter' +export * from './branch' +export * from './experiment' +export * from './graphiti' export * from './holdout' +export * from './improvement' +export * from './mem0' export * from './neo4j' export * from './schemas' export * from './source-record' diff --git a/src/memory/mem0.ts b/src/memory/mem0.ts new file mode 100644 index 0000000..257f401 --- /dev/null +++ b/src/memory/mem0.ts @@ -0,0 +1,376 @@ +import { canonicalJson } from '@tangle-network/agent-eval' +import { stableId } from '../ids' +import { defaultGetMemoryContext } from './adapter' +import { memoryWriteResultToSourceRecord } from './source-record' +import type { + AgentMemoryAdapter, + AgentMemoryHit, + AgentMemoryKind, + AgentMemoryScope, + AgentMemorySearchOptions, + AgentMemoryWriteInput, +} from './types' + +export type Mem0ClientMode = 'hosted' | 'oss' + +export interface Mem0ClientLike { + add( + messages: Array<{ role: string; content: string }>, + options?: Record, + ): Promise + search(query: string, options?: Record): Promise + getAll?(options?: Record): Promise + delete?(memoryId: string, options?: Record): Promise + deleteAll?(options?: Record): Promise +} + +export interface Mem0MemoryAdapterOptions { + client: Mem0ClientLike + mode: Mem0ClientMode + id?: string + appId?: string + infer?: boolean + rerank?: boolean + latestOnly?: boolean + defaultScope?: AgentMemoryScope +} + +/** Connects both the hosted Mem0 client and the open-source `Memory` class. */ +export function createMem0MemoryAdapter(options: Mem0MemoryAdapterOptions): AgentMemoryAdapter { + assertMem0Options(options) + const id = options.id ?? `mem0-${options.mode}` + + const adapter: AgentMemoryAdapter = { + id, + branchIsolation: { mode: 'scoped' }, + async search(query, searchOptions = {}) { + assertMem0SearchOptions(searchOptions, id) + if (searchOptions.limit === 0) return [] + const scope = mergeScopes(options.defaultScope, searchOptions.scope) + const raw = await options.client.search(query, { + filters: mem0Filters(scope, options.appId), + topK: searchOptions.limit, + threshold: searchOptions.minScore, + ...(options.rerank !== undefined ? { rerank: options.rerank } : {}), + ...(options.mode === 'hosted' && options.latestOnly !== undefined + ? { latestOnly: options.latestOnly } + : {}), + }) + return normalizeMem0Hits(raw, id, searchOptions) + }, + async getContext(query, searchOptions = {}) { + return defaultGetMemoryContext(adapter, query, searchOptions) + }, + async write(input) { + const scope = mergeScopes(options.defaultScope, input.scope) + const scopeMetadata = mem0ScopeMetadata(scope, options.appId) + const metadata = compactRecord({ + memoryKind: input.kind, + memoryTitle: input.title, + entityName: input.entityName, + entityType: input.entityType, + category: input.category, + predicate: input.predicate, + subject: input.subject, + object: input.object, + confidence: input.confidence, + originalRole: input.role, + ...input.metadata, + ...scopeMetadata, + }) + const entity = mem0Entity(scope, options.appId) + const raw = await options.client.add([{ role: mem0Role(input.role), content: input.text }], { + ...entity, + metadata, + infer: options.infer ?? true, + ...(options.mode === 'oss' ? { filters: mem0Filters(scope, options.appId) } : {}), + }) + const items = extractMem0Items(raw) + const mutations = items.filter((candidate) => mem0Event(candidate) !== 'NOOP') + const item = mutations[0] ?? items[0] + const itemId = + stringField(item, ['id']) ?? + input.id ?? + stableId( + 'mem', + canonicalJson( + compactRecord({ adapterId: id, kind: input.kind, text: input.text, scope }), + ), + ) + const event = mem0Event(item) + const events = [...new Set(mutations.map(mem0Event).filter(Boolean))] + const result = { + accepted: mutations.length > 0, + id: itemId, + uri: `memory://${id}/${encodeURIComponent(itemId)}`, + kind: input.kind, + metadata: { + provider: 'mem0', + mode: options.mode, + ...(event ? { event } : {}), + ...(events.length > 1 ? { events } : {}), + }, + } as const + return { + ...result, + sourceRecord: memoryWriteResultToSourceRecord(result, input.text, { scope }), + } + }, + async clear(scope) { + const mergedScope = mergeScopes(options.defaultScope, scope) + if (options.client.getAll && options.client.delete) { + let deleted = 0 + for (let batch = 0; batch < 100; batch += 1) { + const raw = await options.client.getAll( + options.mode === 'hosted' + ? { + filters: mem0Filters(mergedScope, options.appId), + page: 1, + pageSize: 100, + latestOnly: false, + showExpired: true, + } + : { + filters: mem0Filters(mergedScope, options.appId), + topK: 100, + showExpired: true, + }, + ) + const items = extractMem0Items(raw) + if (items.length === 0) return + const rawIds = items.map((item) => stringField(item, ['id'])) + if (!rawIds.every((memoryId): memoryId is string => memoryId !== undefined)) { + throw new Error(`${id}: Mem0 returned memories without ids during scoped clear`) + } + const ids = [...new Set(rawIds)] + for (const memoryId of ids) await options.client.delete(memoryId) + deleted += ids.length + } + throw new Error(`${id}: refused to clear more than ${deleted} Mem0 memories in one call`) + } + if (!options.client.deleteAll || !canDeleteAllExactly(mergedScope)) { + throw new Error( + `${id}: exact scoped clear requires Mem0 getAll and delete for tenant, team, session, tag, or nested run scopes`, + ) + } + await options.client.deleteAll(mem0Entity(mergedScope, options.appId)) + }, + } + return adapter +} + +function assertMem0Options(options: Mem0MemoryAdapterOptions): void { + if (options.id !== undefined && (typeof options.id !== 'string' || !options.id.trim())) { + throw new Error('Mem0 id must be a non-empty string') + } + if (options.appId !== undefined && (typeof options.appId !== 'string' || !options.appId.trim())) { + throw new Error('Mem0 appId must be a non-empty string') + } +} + +function assertMem0SearchOptions(options: AgentMemorySearchOptions, adapterId: string): void { + if (options.limit !== undefined && (!Number.isSafeInteger(options.limit) || options.limit < 0)) { + throw new Error(`${adapterId}: search limit must be a non-negative safe integer`) + } + if (options.minScore !== undefined && !Number.isFinite(options.minScore)) { + throw new Error(`${adapterId}: minScore must be finite`) + } +} + +function normalizeMem0Hits( + raw: unknown, + adapterId: string, + options: AgentMemorySearchOptions, +): AgentMemoryHit[] { + const limit = options.limit ?? Number.POSITIVE_INFINITY + const kinds = options.kinds ? new Set(options.kinds) : null + return extractMem0Items(raw) + .map((item): AgentMemoryHit | undefined => { + const text = + stringField(item, ['memory', 'text', 'content']) ?? + stringField(recordField(item, 'data'), ['memory', 'text', 'content']) + if (!text) return undefined + const metadata = recordField(item, 'metadata') + const kind = mem0Kind(metadata?.memoryKind ?? item?.memoryType) + if (kinds && !kinds.has(kind)) return undefined + const score = numberField(item, ['rerankScore', 'rerank_score', 'score']) + if (options.minScore !== undefined && (score === undefined || score < options.minScore)) { + return undefined + } + const memoryId = + stringField(item, ['id']) ?? + stableId('mem', canonicalJson(compactRecord({ adapterId, text, metadata, item }))) + const normalizedScore = score !== undefined && score >= 0 && score <= 1 ? score : undefined + return { + id: memoryId, + uri: `memory://${adapterId}/${encodeURIComponent(memoryId)}`, + kind, + text, + ...(stringField(metadata, ['memoryTitle']) ? { title: String(metadata!.memoryTitle) } : {}), + ...(score !== undefined ? { score } : {}), + ...(normalizedScore !== undefined ? { normalizedScore } : {}), + ...(dateField(item, ['createdAt', 'created_at']) + ? { createdAt: dateField(item, ['createdAt', 'created_at']) } + : {}), + ...(dateField(item, ['expirationDate', 'expiration_date']) + ? { validUntil: dateField(item, ['expirationDate', 'expiration_date']) } + : {}), + metadata: compactRecord({ + provider: 'mem0', + ...metadata, + categories: item?.categories, + owner: item?.owner, + runId: item?.runId, + agentId: item?.agentId, + userId: item?.userId, + }), + } + }) + .filter((hit): hit is AgentMemoryHit => hit !== undefined) + .slice(0, limit) +} + +function extractMem0Items(raw: unknown): Array> { + if (Array.isArray(raw)) return raw.filter(isRecord) + if (!isRecord(raw)) return [] + for (const key of ['results', 'memories', 'data']) { + if (Array.isArray(raw[key])) return raw[key].filter(isRecord) + } + return [raw] +} + +function mem0Event(item: Record | undefined): string | undefined { + return stringField(item, ['event'])?.toUpperCase() +} + +function mem0Entity(scope: AgentMemoryScope, appId?: string): Record { + return compactStringRecord({ + userId: scope.userId, + agentId: scope.agentId, + appId, + runId: scope.namespace ?? scope.runId, + }) +} + +function mem0Filters(scope: AgentMemoryScope, appId?: string): Record { + return mem0ScopeMetadata(scope, appId) +} + +function mem0ScopeMetadata(scope: AgentMemoryScope, appId?: string): Record { + const metadata = compactStringRecord({ + user_id: scope.userId, + agent_id: scope.agentId, + run_id: scope.namespace ?? scope.runId, + logical_run_id: scope.namespace ? scope.runId : undefined, + app_id: appId, + tenant_id: scope.tenantId, + team_id: scope.teamId, + session_id: scope.sessionId, + }) + for (const [key, value] of Object.entries(scope.tags ?? {})) { + metadata[`tag_${key}`] = value + } + return metadata +} + +function canDeleteAllExactly(scope: AgentMemoryScope): boolean { + return ( + scope.tenantId === undefined && + scope.teamId === undefined && + scope.sessionId === undefined && + (!scope.tags || Object.keys(scope.tags).length === 0) && + !(scope.namespace && scope.runId) + ) +} + +function mem0Role(role: AgentMemoryWriteInput['role']): 'user' | 'assistant' { + return role === 'assistant' || role === 'tool' ? 'assistant' : 'user' +} + +function mem0Kind(value: unknown): AgentMemoryKind { + switch (value) { + case 'message': + case 'entity': + case 'fact': + case 'preference': + case 'observation': + case 'reasoning-trace': + return value + default: + return 'fact' + } +} + +function mergeScopes(base?: AgentMemoryScope, extra?: AgentMemoryScope): AgentMemoryScope { + return { + ...(base ?? {}), + ...(extra ?? {}), + tags: { ...(base?.tags ?? {}), ...(extra?.tags ?? {}) }, + } +} + +function compactRecord(input: Record): Record { + return Object.fromEntries(Object.entries(input).filter(([, value]) => value !== undefined)) +} + +function compactStringRecord(input: Record): Record { + return Object.fromEntries( + Object.entries(input).filter((entry): entry is [string, string] => entry[1] !== undefined), + ) +} + +function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value) +} + +function recordField( + value: Record | undefined, + key: string, +): Record | undefined { + const field = value?.[key] + return isRecord(field) ? field : undefined +} + +function stringField( + value: Record | undefined, + keys: readonly string[], +): string | undefined { + for (const key of keys) { + const field = value?.[key] + if (typeof field === 'string' && field.length > 0) return field + } + return undefined +} + +function numberField( + value: Record | undefined, + keys: readonly string[], +): number | undefined { + for (const key of keys) { + const field = value?.[key] + if (typeof field === 'number' && Number.isFinite(field)) return field + } + return undefined +} + +function dateField( + value: Record | undefined, + keys: readonly string[], +): string | undefined { + for (const key of keys) { + const field = value?.[key] + if (typeof field === 'string' && field.length > 0) return field + if (field instanceof Date && !Number.isNaN(field.valueOf())) return field.toISOString() + } + return undefined +} + +/** Stable identity useful for dispatch/cache keys without exposing client credentials. */ +export function mem0MemoryAdapterIdentity( + options: Pick< + Mem0MemoryAdapterOptions, + 'mode' | 'id' | 'appId' | 'infer' | 'rerank' | 'latestOnly' | 'defaultScope' + >, +): string { + return stableId('mem0', canonicalJson(compactRecord(options))) +} diff --git a/src/memory/neo4j.ts b/src/memory/neo4j.ts index a6e73db..619e82a 100644 --- a/src/memory/neo4j.ts +++ b/src/memory/neo4j.ts @@ -1,3 +1,4 @@ +import { canonicalJson } from '@tangle-network/agent-eval' import { stableId } from '../ids' import { defaultGetMemoryContext } from './adapter' import { emitRetrievalHoldoutBypass } from './holdout' @@ -12,218 +13,260 @@ import type { } from './types' export interface Neo4jAgentMemoryAdapterOptions { - client: Record + client: object + /** Match the transport passed to the official MemoryClient. */ + transport: 'rest' | 'bridge' + /** Use Neo4j's whole-conversation context instead of query-based search. */ + contextMode?: 'search' | 'native' id?: string + /** Assert that the client owns disposable external state for this exact branch. */ + branchId?: string } export function createNeo4jAgentMemoryAdapter( options: Neo4jAgentMemoryAdapterOptions, ): AgentMemoryAdapter { - const client = options.client + assertNeo4jOptions(options) + const client = options.client as Record const id = options.id ?? 'neo4j-agent-memory' - return { + const adapter: AgentMemoryAdapter = { id, + branchIsolation: options.branchId + ? { mode: 'instance', branchId: options.branchId, supportsLogicalScopes: false } + : { + mode: 'unsupported', + reason: 'create a separate MemoryClient namespace per branch and pass branchId', + }, async search(query, searchOptions = {}) { - const sdkHits = await searchNeo4jMemory(client, query, searchOptions, id) - if (sdkHits.length > 0) return sdkHits - - const result = await callOptional( - client, - ['search', 'memory_search'], - [query, neo4jOptions(searchOptions)], - ) - if (result !== undefined) return normalizeHits(result, searchOptions, id) - - const context = await callOptional( - client, - ['getContext', 'get_context'], - [query, neo4jOptions(searchOptions)], - ) - if (context !== undefined) return normalizeHits(context, searchOptions, id) - return [] + assertNeo4jSearchOptions(searchOptions, id) + return searchNeo4jMemory(client, query, searchOptions, id, options.transport) }, async getContext(query, searchOptions = {}) { + assertNeo4jSearchOptions(searchOptions, id) + assertNeo4jSearchKinds(searchOptions.kinds, options.transport, id) const shortTerm = nested(client, ['shortTerm', 'short_term'], {}) const sessionId = searchOptions.scope?.sessionId - if (sessionId) { - const conversationContext = await callOptional( - shortTerm, - ['getContext', 'get_context'], - [sessionId], - ) - if (conversationContext !== undefined) { - const hits = normalizeConversationContextHits(conversationContext, searchOptions, id) - const text = - textFromConversationContext(conversationContext) ?? - (typeof conversationContext === 'string' - ? conversationContext - : hits.length > 0 - ? renderHits(hits) - : '') - if (searchOptions.holdout) { - emitRetrievalHoldoutBypass( - hits, - searchOptions.holdout, - holdoutBypassContext(query, searchOptions), - 'short-term-context', - ) - } - return { - query, - text, - hits, - sourceRecords: hits.map((hit) => - memoryHitToSourceRecord(hit, { scope: searchOptions.scope }), - ), - metadata: { adapter: id, rawContext: conversationContext }, - } - } + if (options.contextMode === 'native' && options.transport !== 'rest') { + throw new Error(`${id}: Neo4j native context requires transport="rest"`) } - - const result = await callOptional( - client, - ['getContext', 'get_context'], - [query, neo4jOptions(searchOptions)], - ) - if (result === undefined) { - return defaultGetMemoryContext( - { - search: async () => { - const sdkHits = await searchNeo4jMemory(client, query, searchOptions, id) - if (sdkHits.length > 0) return sdkHits - const searchResult = await callOptional( - client, - ['search', 'memory_search'], - [query, neo4jOptions(searchOptions)], - ) - return normalizeHits(searchResult, searchOptions, id) - }, - }, - query, - searchOptions, + if (options.contextMode === 'native' && !sessionId) { + throw new Error(`${id}: Neo4j native context requires scope.sessionId`) + } + if ( + options.contextMode === 'native' && + searchOptions.kinds?.some((kind) => kind !== 'message' && kind !== 'observation') + ) { + throw new Error( + `${id}: Neo4j native context only returns message and observation memory kinds`, ) } - if (typeof result === 'string') { - const hit: AgentMemoryHit = { - id: stableId('mem', `${id}:${query}:${result}`), - uri: `memory://${id}/context/${stableId('ctx', query)}`, - kind: 'fact', - text: result, - title: 'Memory context', - normalizedScore: 1, - } + if (options.contextMode === 'native' && options.transport === 'rest' && sessionId) { + const conversationContext = await callRequired(shortTerm, ['getContext'], [sessionId]) + const hits = normalizeConversationContextHits(conversationContext, searchOptions, id) + const text = renderHits(hits) if (searchOptions.holdout) { emitRetrievalHoldoutBypass( - [hit], + hits, searchOptions.holdout, holdoutBypassContext(query, searchOptions), - 'raw-string-context', + 'short-term-context', ) } return { query, - text: result, - hits: [hit], - sourceRecords: [memoryHitToSourceRecord(hit, { scope: searchOptions.scope })], - metadata: { adapter: id }, + text, + hits, + sourceRecords: hits.map((hit) => + memoryHitToSourceRecord(hit, { scope: searchOptions.scope }), + ), + metadata: { adapter: id, rawContext: conversationContext }, } } - const hits = normalizeHits(result, searchOptions, id) - if (hits.length > 0) - return defaultGetMemoryContext({ search: async () => hits }, query, searchOptions) - return defaultGetMemoryContext({ search: async () => [] }, query, searchOptions) + + return defaultGetMemoryContext(adapter, query, searchOptions) }, async write(input) { - const result = await writeNeo4jMemory(client, input, id) + const result = await writeNeo4jMemory(client, input, id, options.transport) return { ...result, sourceRecord: memoryWriteResultToSourceRecord(result, input.text, { scope: input.scope }), } }, + async close() { + await callOptional(client, ['close'], []) + }, + } + return adapter +} + +function assertNeo4jOptions(options: Neo4jAgentMemoryAdapterOptions): void { + if (typeof options.client !== 'object' || options.client === null) { + throw new Error('Neo4j agent-memory client must be an object') + } + if (options.id !== undefined && (typeof options.id !== 'string' || !options.id.trim())) { + throw new Error('Neo4j agent-memory id must be a non-empty string') + } + if ( + options.branchId !== undefined && + (typeof options.branchId !== 'string' || !options.branchId.trim()) + ) { + throw new Error('Neo4j agent-memory branchId must be a non-empty string') + } + if (options.contextMode !== undefined && !['search', 'native'].includes(options.contextMode)) { + throw new Error('Neo4j agent-memory contextMode must be search or native') + } +} + +function assertNeo4jSearchOptions(options: AgentMemorySearchOptions, adapterId: string): void { + if (options.limit !== undefined && (!Number.isSafeInteger(options.limit) || options.limit < 0)) { + throw new Error(`${adapterId}: search limit must be a non-negative safe integer`) + } + if (options.minScore !== undefined && !Number.isFinite(options.minScore)) { + throw new Error(`${adapterId}: minScore must be finite`) + } +} + +function assertNeo4jSearchKinds( + kinds: AgentMemorySearchOptions['kinds'], + transport: 'rest' | 'bridge', + adapterId: string, +): void { + if (!kinds?.length) return + const supported = new Set( + transport === 'rest' + ? ['message', 'observation', 'entity'] + : ['message', 'observation', 'entity', 'preference', 'reasoning-trace'], + ) + const unsupported = [...new Set(kinds.filter((kind) => !supported.has(kind)))] + if (unsupported.length > 0) { + throw new Error( + `${adapterId}: Neo4j ${transport} cannot search memory kinds: ${unsupported.join(', ')}`, + ) } } +function assertNeo4jBridgeCapability( + transport: 'rest' | 'bridge', + kind: AgentMemoryWriteInput['kind'], + adapterId: string, +): void { + if (transport === 'bridge') return + throw new Error( + `${adapterId}: Neo4j REST cannot write ${kind}; use transport="bridge" or map it to an entity/message`, + ) +} + async function writeNeo4jMemory( client: Record, input: AgentMemoryWriteInput, adapterId: string, + transport: 'rest' | 'bridge', ): Promise { const scope = input.scope ?? {} const sessionId = scope.sessionId ?? input.metadata?.sessionId let result: unknown - if (input.kind === 'message') { + if (input.kind === 'message' || input.kind === 'observation') { + if (transport === 'rest' && typeof sessionId !== 'string') { + throw new Error( + `${adapterId}: Neo4j REST message writes require scope.sessionId as the conversation id`, + ) + } const shortTerm = nested(client, ['shortTerm', 'short_term'], client) - result = await callOptional( + const metadata = { ...input.metadata, agentKnowledgeKind: input.kind } + result = await callRequired( shortTerm, ['addMessage'], [ sessionId, - input.role ?? 'user', + neo4jMessageRole(input.role), input.text, - { - metadata: input.metadata, - conversationId: sessionId, - userId: scope.userId, - }, + { metadata, conversationId: sessionId }, ], ) - if (result === undefined) { - result = await callRequired( - shortTerm, - ['add_message'], - [ - { - session_id: sessionId, - sessionId, - role: input.role ?? 'user', - content: input.text, - user_identifier: scope.userId, - userIdentifier: scope.userId, - metadata: input.metadata, - }, - ], - ) - } } else if (input.kind === 'entity') { const longTerm = nested(client, ['longTerm', 'long_term'], client) - result = await callOptional( + result = await callRequired( longTerm, ['addEntity'], [ input.entityName ?? input.title ?? input.text, - input.entityType ?? 'ENTITY', + input.entityType ?? 'custom', neo4jEntityOptions(input), ], ) - if (result === undefined) { - result = await callRequired( - longTerm, - ['add_entity'], - [ - input.entityName ?? input.title ?? input.text, - input.entityType ?? 'ENTITY', - neo4jWriteOptions(input), - ], - ) - } } else if (input.kind === 'preference') { + assertNeo4jBridgeCapability(transport, input.kind, adapterId) const longTerm = nested(client, ['longTerm', 'long_term'], client) - result = await callOptional( + result = await callRequired( longTerm, ['addPreference'], [input.category ?? 'general', input.text, neo4jPreferenceOptions(input)], ) - if (result === undefined) { + } else if (input.kind === 'reasoning-trace') { + const reasoning = nested(client, ['reasoning'], client) + const conversationId = sessionId ?? scope.runId + if (transport === 'rest') { + if (typeof conversationId !== 'string') { + throw new Error( + `${adapterId}: Neo4j REST reasoning writes require scope.sessionId or scope.runId`, + ) + } result = await callRequired( - longTerm, - ['add_preference'], - [input.category ?? 'general', input.text, neo4jWriteOptions(input)], + reasoning, + ['recordStep'], + [ + { + conversationId, + reasoning: input.text, + actionTaken: stringMetadata(input.metadata, 'action') ?? input.title ?? 'memory', + result: + stringMetadata(input.metadata, 'result') ?? + stringMetadata(input.metadata, 'observation') ?? + stringMetadata(input.metadata, 'outcome'), + }, + ], + ) + } else { + const trace = await callRequired( + reasoning, + ['startTrace'], + [ + conversationId ?? stableId('session', input.text), + stringMetadata(input.metadata, 'task') ?? input.title ?? input.text, + ], + ) + const traceId = idFromResult(trace) + if (!traceId) throw new Error(`${adapterId}: Neo4j startTrace returned no trace id`) + await callRequired( + reasoning, + ['addStep'], + [ + traceId, + { + thought: input.text, + observation: stringMetadata(input.metadata, 'observation'), + action: stringMetadata(input.metadata, 'action'), + }, + ], + ) + result = await callRequired( + reasoning, + ['completeTrace'], + [ + traceId, + { + outcome: stringMetadata(input.metadata, 'outcome'), + success: + typeof input.metadata?.success === 'boolean' ? input.metadata.success : undefined, + }, + ], ) } } else { + assertNeo4jBridgeCapability(transport, input.kind, adapterId) result = await callRequired( nested(client, ['longTerm', 'long_term'], client), - ['addFact', 'add_fact'], + ['addFact'], [ input.subject ?? input.title ?? input.kind, input.predicate ?? 'states', @@ -232,13 +275,19 @@ async function writeNeo4jMemory( ) } - const id = idFromResult(result) ?? input.id ?? stableId('mem', `${input.kind}:${input.text}`) + const id = + idFromResult(result) ?? + input.id ?? + stableId( + 'mem', + canonicalJson({ adapterId, kind: input.kind, text: input.text, scope: input.scope ?? {} }), + ) return { accepted: true, id, uri: `memory://${adapterId}/${encodeURIComponent(id)}`, kind: input.kind, - metadata: { rawResult: result }, + metadata: { provider: 'neo4j-agent-memory', transport }, } } @@ -247,52 +296,64 @@ async function searchNeo4jMemory( query: string, options: AgentMemorySearchOptions, adapterId: string, + transport: 'rest' | 'bridge', ): Promise { + const limit = options.limit ?? 10 + if (limit === 0) return [] + assertNeo4jSearchKinds(options.kinds, transport, adapterId) const searches: Promise[] = [] const kinds = options.kinds const includeKind = (kind: AgentMemoryHit['kind']) => !kinds?.length || kinds.includes(kind) const shortTerm = nested(client, ['shortTerm', 'short_term'], {}) const longTerm = nested(client, ['longTerm', 'long_term'], {}) const reasoning = nested(client, ['reasoning'], {}) + const wantsMessages = includeKind('message') || includeKind('observation') + const hasConversation = typeof options.scope?.sessionId === 'string' - if (includeKind('message')) { + if ( + transport === 'rest' && + wantsMessages && + !hasConversation && + options.kinds?.some((kind) => kind === 'message' || kind === 'observation') + ) { + throw new Error( + `${adapterId}: Neo4j REST message search requires scope.sessionId as the conversation id`, + ) + } + + if (wantsMessages && (transport === 'bridge' || hasConversation)) { + const messageKinds = options.kinds?.filter( + (kind) => kind === 'message' || kind === 'observation', + ) ?? ['message', 'observation'] searches.push( - callOptional( - shortTerm, - ['searchMessages', 'search_messages'], - [query, searchMessagesOptions(options)], - ).then((result) => normalizeHits(result, { ...options, kinds: ['message'] }, adapterId)), + callRequired(shortTerm, ['searchMessages'], [query, searchMessagesOptions(options)]).then( + (result) => normalizeHits(result, { ...options, kinds: messageKinds }, adapterId), + ), ) } if (includeKind('entity')) { searches.push( - callOptional( - longTerm, - ['searchEntities', 'search_entities'], - [query, searchEntitiesOptions(options)], - ).then((result) => normalizeHits(result, { ...options, kinds: ['entity'] }, adapterId)), + callRequired(longTerm, ['searchEntities'], [query, searchEntitiesOptions(options)]).then( + (result) => normalizeHits(result, { ...options, kinds: ['entity'] }, adapterId), + ), ) } - if (includeKind('preference')) { + if (transport === 'bridge' && includeKind('preference')) { searches.push( - callOptional( + callRequired( longTerm, - ['searchPreferences', 'search_preferences'], + ['searchPreferences'], [query, searchPreferencesOptions(options)], ).then((result) => normalizeHits(result, { ...options, kinds: ['preference'] }, adapterId)), ) } - if (includeKind('reasoning-trace')) { + if (transport === 'bridge' && includeKind('reasoning-trace')) { searches.push( - callOptional( - reasoning, - ['getSimilarTraces', 'get_similar_traces'], - [query, similarTracesOptions(options)], - ).then((result) => - normalizeHits(result, { ...options, kinds: ['reasoning-trace'] }, adapterId), + callRequired(reasoning, ['getSimilarTraces'], [query, similarTracesOptions(options)]).then( + (result) => normalizeHits(result, { ...options, kinds: ['reasoning-trace'] }, adapterId), ), ) } @@ -300,7 +361,7 @@ async function searchNeo4jMemory( const hits = (await Promise.all(searches)).flat() return hits .sort((a, b) => (b.normalizedScore ?? b.score ?? 0) - (a.normalizedScore ?? a.score ?? 0)) - .slice(0, options.limit) + .slice(0, limit) } // Mirrors defaultGetMemoryContext's holdout call context so bypass events join the same log stream. @@ -315,31 +376,12 @@ function holdoutBypassContext( } } -function neo4jOptions(options: AgentMemorySearchOptions): Record { - return { - limit: options.limit, - k: options.limit, - min_score: options.minScore, - minScore: options.minScore, - user_identifier: options.scope?.userId, - userIdentifier: options.scope?.userId, - session_id: options.scope?.sessionId, - sessionId: options.scope?.sessionId, - namespace: options.scope?.namespace, - tenant_id: options.scope?.tenantId, - tenantId: options.scope?.tenantId, - kinds: options.kinds, - metadata: options.metadata, - } -} - function searchMessagesOptions(options: AgentMemorySearchOptions): Record { return { limit: options.limit, sessionId: options.scope?.sessionId, conversationId: options.scope?.sessionId, threshold: options.minScore, - metadata: options.metadata, } } @@ -360,23 +402,10 @@ function searchPreferencesOptions(options: AgentMemorySearchOptions): Record { return { limit: options.limit, - sessionId: options.scope?.sessionId, successOnly: options.metadata?.successOnly, } } -function neo4jWriteOptions(input: AgentMemoryWriteInput): Record { - return { - user_identifier: input.scope?.userId, - userIdentifier: input.scope?.userId, - session_id: input.scope?.sessionId, - sessionId: input.scope?.sessionId, - namespace: input.scope?.namespace, - confidence: input.confidence, - metadata: input.metadata, - } -} - function neo4jEntityOptions(input: AgentMemoryWriteInput): Record { return { description: input.metadata?.description ?? input.title, @@ -406,8 +435,10 @@ async function callRequired( names: string[], args: unknown[], ): Promise { - const result = await callOptional(target, names, args) - if (result !== undefined) return result + for (const name of names) { + const fn = target[name] + if (typeof fn === 'function') return await fn.apply(target, args) + } throw new Error(`Neo4j agent-memory client is missing method ${names.join(' or ')}`) } @@ -438,7 +469,7 @@ function normalizeHits( : (hit.normalizedScore ?? hit.score ?? 0) >= options.minScore!, ) .filter((hit) => (options.kinds?.length ? options.kinds.includes(hit.kind) : true)) - .slice(0, options.limit) + .slice(0, options.limit ?? 10) } function normalizeHit(value: unknown, index: number, adapterId: string): AgentMemoryHit | null { @@ -453,7 +484,11 @@ function normalizeHit(value: unknown, index: number, adapterId: string): AgentMe } const obj = record(value) if (!obj) return null - const kind = memoryKind(stringField(obj, ['kind', 'type', 'label']), obj) + const metadata = record(obj.metadata) ?? {} + const kind = memoryKind( + stringField(metadata, ['agentKnowledgeKind']) ?? stringField(obj, ['kind', 'type', 'label']), + obj, + ) const text = textFromHitObject(obj, kind) if (!text) return null const id = stringField(obj, ['id', 'memoryId', 'uuid']) ?? stableId('mem', text) @@ -536,17 +571,6 @@ function textFromHitObject( return undefined } -function textFromConversationContext(value: unknown): string | undefined { - const obj = record(value) - if (!obj) return undefined - const parts = [ - ...rawHitsFrom(obj.reflections).map((hit) => normalizeContextPart(hit)), - ...rawHitsFrom(obj.observations).map((hit) => normalizeContextPart(hit)), - ...rawHitsFrom(obj.recentMessages).map((hit) => normalizeContextPart(hit)), - ].filter(Boolean) - return parts.length > 0 ? parts.join('\n\n') : undefined -} - function normalizeConversationContextHits( value: unknown, options: AgentMemorySearchOptions, @@ -574,14 +598,6 @@ function tagContextHit( : { kind, title, content: value } } -function normalizeContextPart(value: unknown): string | undefined { - if (typeof value === 'string') return value - const obj = record(value) - return obj - ? textFromHitObject(obj, memoryKind(stringField(obj, ['kind', 'type', 'label']), obj)) - : undefined -} - function renderHits(hits: AgentMemoryHit[]): string { return hits.map((hit) => hit.text).join('\n\n') } @@ -612,3 +628,15 @@ function numberField(obj: Record, names: string[]): number | un } return undefined } + +function neo4jMessageRole(role: AgentMemoryWriteInput['role']): 'system' | 'user' | 'assistant' { + return role === 'tool' ? 'assistant' : (role ?? 'user') +} + +function stringMetadata( + metadata: Record | undefined, + key: string, +): string | undefined { + const value = metadata?.[key] + return typeof value === 'string' && value.length > 0 ? value : undefined +} diff --git a/src/memory/schemas.ts b/src/memory/schemas.ts index 6f6e791..b6e5da4 100644 --- a/src/memory/schemas.ts +++ b/src/memory/schemas.ts @@ -12,6 +12,9 @@ export const AgentMemoryKindSchema = z.enum([ export const AgentMemoryScopeSchema = z.object({ tenantId: z.string().optional(), userId: z.string().optional(), + agentId: z.string().optional(), + teamId: z.string().optional(), + runId: z.string().optional(), sessionId: z.string().optional(), namespace: z.string().optional(), tags: z.record(z.string(), z.string()).optional(), diff --git a/src/memory/types.ts b/src/memory/types.ts index a538d1f..8d8b46f 100644 --- a/src/memory/types.ts +++ b/src/memory/types.ts @@ -12,6 +12,9 @@ export type AgentMemoryKind = export interface AgentMemoryScope { tenantId?: string userId?: string + agentId?: string + teamId?: string + runId?: string sessionId?: string namespace?: string tags?: Record @@ -79,10 +82,24 @@ export interface AgentMemoryWriteResult { metadata?: Record } +export type AgentMemoryBranchIsolation = + | { mode: 'scoped' } + | { + mode: 'instance' + branchId: string + /** True only when the dedicated instance also enforces every logical scope. */ + supportsLogicalScopes?: boolean + } + | { mode: 'unsupported'; reason: string } + export interface AgentMemoryAdapter { readonly id: string + /** How this adapter prevents candidate branches from reading each other's state. */ + readonly branchIsolation?: AgentMemoryBranchIsolation search(query: string, options?: AgentMemorySearchOptions): Promise getContext(query: string, options?: AgentMemorySearchOptions): Promise write(input: AgentMemoryWriteInput): Promise + clear?(scope?: AgentMemoryScope): Promise flush?(): Promise + close?(): Promise } diff --git a/tests/benchmarks.test.ts b/tests/benchmarks.test.ts index 6df91a8..c38ded3 100644 --- a/tests/benchmarks.test.ts +++ b/tests/benchmarks.test.ts @@ -307,4 +307,32 @@ describe('knowledge benchmark adapters', () => { expect(storage.read(result.rankingMarkdownPath)).toContain('| 1 | in-memory |') expect(storage.read(result.rows[0]!.reportJsonPath)).toContain('"memory_stale_safe"') }) + + it('rejects every unsafe candidate id before creating any adapter', async () => { + let creates = 0 + await expect( + runMemoryAdapterBenchmark({ + cases: buildFirstPartyMemoryLifecycleBenchmarkCases().slice(0, 1), + runDir: '/runs/invalid-memory-candidate', + storage: inMemoryCampaignStorage(), + candidates: [ + { + id: 'valid', + createAdapter() { + creates += 1 + return createInMemoryBenchmarkAdapter() + }, + }, + { + id: '../invalid', + createAdapter() { + creates += 1 + return createInMemoryBenchmarkAdapter() + }, + }, + ], + }), + ).rejects.toThrow('must be a safe directory segment') + expect(creates).toBe(0) + }) }) diff --git a/tests/memory-adapter.test.ts b/tests/memory-adapter.test.ts index 5df869a..2ab308c 100644 --- a/tests/memory-adapter.test.ts +++ b/tests/memory-adapter.test.ts @@ -48,29 +48,22 @@ describe('memory adapters', () => { expect(similarTraceArgs[1].successOnly).toBe(true) }) - it('wraps a Neo4j Agent Memory-like client for search and context', async () => { + it('renders bridge preference search results as memory context', async () => { const client = { - async search(query: string, options: Record) { - return [ - { - id: 'pref-1', - type: 'preference', - text: `${query}: likes concise answers`, - score: 0.8, - normalizedScore: 1, - userIdentifier: options.userIdentifier, - }, - ] + longTerm: { + async searchPreferences(query: string) { + return [{ id: 'pref-1', category: 'writing', preference: `${query}: concise answers` }] + }, }, } - const adapter = createNeo4jAgentMemoryAdapter({ client }) + const adapter = createNeo4jAgentMemoryAdapter({ client, transport: 'bridge' }) const hits = await adapter.search('writing style', { - scope: { userId: 'user-1' }, + kinds: ['preference'], limit: 3, }) const context = await adapter.getContext('writing style', { - scope: { userId: 'user-1' }, + kinds: ['preference'], limit: 3, }) @@ -78,10 +71,9 @@ describe('memory adapters', () => { expect(hits[0]).toMatchObject({ id: 'pref-1', kind: 'preference', - text: 'writing style: likes concise answers', - normalizedScore: 1, + text: 'writing style: concise answers', }) - expect(context.text).toContain('likes concise answers') + expect(context.text).toContain('concise answers') expect(context.sourceRecords[0]?.uri).toBe('memory://neo4j-agent-memory/pref-1') }) @@ -113,7 +105,7 @@ describe('memory adapters', () => { }, }, } - const adapter = createNeo4jAgentMemoryAdapter({ client }) + const adapter = createNeo4jAgentMemoryAdapter({ client, transport: 'bridge' }) const hits = await adapter.search('project preferences', { scope: { sessionId: 'session-1' }, @@ -149,7 +141,11 @@ describe('memory adapters', () => { }, }, } - const adapter = createNeo4jAgentMemoryAdapter({ client }) + const adapter = createNeo4jAgentMemoryAdapter({ + client, + transport: 'rest', + contextMode: 'native', + }) const context = await adapter.getContext('style', { scope: { sessionId: 'conversation-1' } }) @@ -160,6 +156,21 @@ describe('memory adapters', () => { expect(context.sourceRecords[0]?.metadata?.source).toBe('agent-memory') }) + it('keeps provider-native context explicit when requested kinds require query search', async () => { + const adapter = createNeo4jAgentMemoryAdapter({ + client: { shortTerm: { async getContext() {} } }, + transport: 'rest', + contextMode: 'native', + }) + + await expect( + adapter.getContext('company', { + scope: { sessionId: 'conversation-1' }, + kinds: ['entity'], + }), + ).rejects.toThrow('native context only returns message and observation') + }) + it('delegates message writes using the published TypeScript SDK positional signature', async () => { const calls: unknown[][] = [] const client = { @@ -170,7 +181,7 @@ describe('memory adapters', () => { }, }, } - const adapter = createNeo4jAgentMemoryAdapter({ client }) + const adapter = createNeo4jAgentMemoryAdapter({ client, transport: 'rest' }) const result = await adapter.write({ kind: 'message', @@ -185,13 +196,12 @@ describe('memory adapters', () => { expect(calls[0]?.[2]).toBe('I will keep updates short.') expect(calls[0]?.[3]).toMatchObject({ conversationId: 'session-1', - userId: 'user-1', metadata: { source: 'test' }, }) expect(result.uri).toBe('memory://neo4j-agent-memory/msg-1') }) - it('falls back to bridge-style snake_case message writes', async () => { + it('rejects clients that do not expose the official addMessage method', async () => { const calls: unknown[][] = [] const client = { short_term: { @@ -201,22 +211,17 @@ describe('memory adapters', () => { }, }, } - const adapter = createNeo4jAgentMemoryAdapter({ client }) - - const result = await adapter.write({ - kind: 'message', - role: 'user', - text: 'Remember this bridge message.', - scope: { sessionId: 'session-1', userId: 'user-1' }, - }) + const adapter = createNeo4jAgentMemoryAdapter({ client, transport: 'bridge' }) - expect(calls[0]?.[0]).toMatchObject({ - session_id: 'session-1', - role: 'user', - content: 'Remember this bridge message.', - user_identifier: 'user-1', - }) - expect(result.id).toBe('msg-bridge') + await expect( + adapter.write({ + kind: 'message', + role: 'user', + text: 'Remember this bridge message.', + scope: { sessionId: 'session-1' }, + }), + ).rejects.toThrow('missing method addMessage') + expect(calls).toEqual([]) }) it('uses narrow hosted SDK options for entity and preference writes', async () => { @@ -233,7 +238,7 @@ describe('memory adapters', () => { }, }, } - const adapter = createNeo4jAgentMemoryAdapter({ client }) + const adapter = createNeo4jAgentMemoryAdapter({ client, transport: 'bridge' }) await adapter.write({ kind: 'entity', @@ -275,7 +280,7 @@ describe('memory adapters', () => { }, }, } - const adapter = createNeo4jAgentMemoryAdapter({ client }) + const adapter = createNeo4jAgentMemoryAdapter({ client, transport: 'bridge' }) const result = await adapter.write({ kind: 'preference', @@ -309,7 +314,7 @@ describe('memory adapters', () => { }, }, } - const adapter = createNeo4jAgentMemoryAdapter({ client }) + const adapter = createNeo4jAgentMemoryAdapter({ client, transport: 'bridge' }) await expect( adapter.write({ @@ -321,15 +326,24 @@ describe('memory adapters', () => { expect(calls).toEqual(['addPreference']) }) - it('uses the configured adapter id for fallback memory URIs', async () => { + it('uses the configured adapter id for memory URIs', async () => { const client = { - async getContext() { - return 'Use the private project namespace.' + shortTerm: { + async searchMessages() { + return [{ id: 'message-1', role: 'user', content: 'Use the private project namespace.' }] + }, }, } - const adapter = createNeo4jAgentMemoryAdapter({ client, id: 'neo4j-private' }) + const adapter = createNeo4jAgentMemoryAdapter({ + client, + transport: 'rest', + id: 'neo4j-private', + }) - const context = await adapter.getContext('project namespace') + const context = await adapter.getContext('project namespace', { + scope: { sessionId: 'conversation-1' }, + kinds: ['message'], + }) expect(context.hits[0]?.uri).toMatch(/^memory:\/\/neo4j-private\//) expect(context.sourceRecords[0]?.uri).toBe(context.hits[0]?.uri) diff --git a/tests/memory-holdout.test.ts b/tests/memory-holdout.test.ts index c6f6756..b0a0f7a 100644 --- a/tests/memory-holdout.test.ts +++ b/tests/memory-holdout.test.ts @@ -279,7 +279,11 @@ describe('retrieval holdout: adapter bypass paths still log the call', () => { }, }, } - const adapter = createNeo4jAgentMemoryAdapter({ client }) + const adapter = createNeo4jAgentMemoryAdapter({ + client, + transport: 'rest', + contextMode: 'native', + }) const context = await adapter.getContext('style', { scope: { sessionId: 'conv-1', tags: { taskId: 'task-3' } }, @@ -308,38 +312,6 @@ describe('retrieval holdout: adapter bypass paths still log the call', () => { expect(event.deliveredIds).toEqual(['obs-1', 'msg-1']) expect(event.queryHash).toBe(sha256('style').slice(0, 16)) }) - - it('emits a raw-string-context bypass event for string getContext results', async () => { - const { config, events } = collectingConfig({ epsilon: 1, watchlist: ['w-1'], rng: () => 0 }) - const client = { - async getContext() { - return 'Use the private project namespace.' - }, - } - const adapter = createNeo4jAgentMemoryAdapter({ client, id: 'neo4j-private' }) - - const context = await adapter.getContext('project namespace', { holdout: config }) - - expect(context.text).toBe('Use the private project namespace.') - expect(context.hits).toHaveLength(1) - expect(events).toHaveLength(1) - const event = events[0]! - expect(event.bypassReason).toBe('raw-string-context') - expect(event.holdoutEligible).toBe(false) - expect(event.droppedId).toBeNull() - expect(event.eligible).toEqual([ - { - id: context.hits[0]!.id, - rank: 1, - score: 1, - kind: 'fact', - contentHash: sha256('Use the private project namespace.').slice(0, 16), - }, - ]) - expect(event.deliveredIds).toEqual([context.hits[0]!.id]) - expect(event.sessionId).toBeUndefined() - expect(event.sessionIdHash).toBeUndefined() - }) }) describe('retrieval holdout: value-keyed session registry', () => { diff --git a/tests/memory-systems.test.ts b/tests/memory-systems.test.ts new file mode 100644 index 0000000..5071213 --- /dev/null +++ b/tests/memory-systems.test.ts @@ -0,0 +1,2306 @@ +import type { MemoryClient as Neo4jMemoryClient } from '@neo4j-labs/agent-memory' +import { inMemoryCampaignStorage, type SurfaceProposer } from '@tangle-network/agent-eval/campaign' +import type { MemoryClient } from 'mem0ai' +import type { Memory as OssMemory } from 'mem0ai/oss' +import { describe, expect, it } from 'vitest' +import { + type AgentMemoryAdapter, + type AgentMemoryHit, + type AgentMemoryScope, + buildAgentMemorySequencesFromBenchmarkCases, + createAgentMemoryBranch, + createGraphitiMemoryAdapter, + createMem0MemoryAdapter, + createNeo4jAgentMemoryAdapter, + type GraphitiMcpClientLike, + graphitiMemoryAdapterIdentity, + type Mem0ClientLike, + mem0MemoryAdapterIdentity, + type RunAgentMemoryImprovementOptions, + runAgentMemoryExperiment, + runAgentMemoryImprovement, +} from '../src/memory/index' + +describe('Mem0 adapter', () => { + it('is type-compatible with both current Mem0 clients', () => { + const hosted = null as unknown as MemoryClient + const oss = null as unknown as OssMemory + + expect(createMem0MemoryAdapter({ client: hosted, mode: 'hosted' }).id).toBe('mem0-hosted') + expect(createMem0MemoryAdapter({ client: oss, mode: 'oss' }).id).toBe('mem0-oss') + }) + + it('writes and searches with the same provider scope', async () => { + const calls: Array<{ method: string; options?: Record }> = [] + const client: Mem0ClientLike = { + async add(_messages, options) { + calls.push({ method: 'add', options }) + return [{ id: 'memory-1', event: 'ADD' }] + }, + async search(_query, options) { + calls.push({ method: 'search', options }) + return { + results: [ + { + id: 'memory-1', + memory: 'Use concise status updates.', + score: 0.91, + metadata: { + memoryKind: 'preference', + memoryTitle: 'Writing style', + eventId: 'event-1', + }, + }, + ], + } + }, + } + const adapter = createMem0MemoryAdapter({ client, mode: 'hosted', rerank: true }) + const scope = { + tenantId: 'tenant-1', + userId: 'user-1', + agentId: 'agent-1', + teamId: 'team-1', + namespace: 'branch-1/private', + sessionId: 'session-1', + tags: { task: 'support' }, + } + + await adapter.write({ + id: 'event-1', + kind: 'preference', + title: 'Writing style', + text: 'Use concise status updates.', + scope, + metadata: { run_id: 'attempted-override' }, + }) + const hits = await adapter.search('status style', { scope, limit: 3 }) + + const add = calls[0]?.options + const search = calls[1]?.options + expect(add).toMatchObject({ + userId: 'user-1', + agentId: 'agent-1', + runId: 'branch-1/private', + infer: true, + metadata: { + tenant_id: 'tenant-1', + user_id: 'user-1', + agent_id: 'agent-1', + team_id: 'team-1', + run_id: 'branch-1/private', + session_id: 'session-1', + tag_task: 'support', + }, + }) + expect(search).toMatchObject({ + filters: { + tenant_id: 'tenant-1', + user_id: 'user-1', + agent_id: 'agent-1', + team_id: 'team-1', + run_id: 'branch-1/private', + session_id: 'session-1', + tag_task: 'support', + }, + topK: 3, + rerank: true, + }) + expect(hits[0]).toMatchObject({ + id: 'memory-1', + kind: 'preference', + text: 'Use concise status updates.', + score: 0.91, + normalizedScore: 0.91, + metadata: { eventId: 'event-1' }, + }) + }) + + it('does not journal an empty provider result as an accepted write', async () => { + const adapter = createMem0MemoryAdapter({ + mode: 'hosted', + client: { + async add() { + return [] + }, + async search() { + return { results: [] } + }, + }, + }) + + await expect( + adapter.write({ kind: 'fact', text: 'No provider mutation' }), + ).resolves.toMatchObject({ accepted: false }) + }) + + it('journals and replays Mem0 deletion mutations', async () => { + const parentCalls: string[] = [] + const childCalls: string[] = [] + const adapter = (calls: string[]) => + createMem0MemoryAdapter({ + mode: 'hosted', + client: { + async add(messages) { + const text = messages[0]!.content + calls.push(text) + return [ + { + id: text.startsWith('Forget') ? 'memory-old' : 'memory-new', + event: text.startsWith('Forget') ? 'DELETE' : 'ADD', + }, + ] + }, + async search() { + return { results: [] } + }, + }, + }) + const parent = createAgentMemoryBranch({ + adapter: adapter(parentCalls), + branchId: 'mem0-parent', + baseScope: { agentId: 'agent-1' }, + }) + + await parent.write({ kind: 'fact', text: 'Remember the old launch date.' }) + const deletion = await parent.write({ kind: 'fact', text: 'Forget the old launch date.' }) + const snapshot = await parent.snapshot() + await parent.fork({ branchId: 'mem0-child', adapter: adapter(childCalls) }) + + expect(deletion).toMatchObject({ accepted: true, metadata: { event: 'DELETE' } }) + expect(snapshot.journal.map((entry) => entry.input.text)).toEqual([ + 'Remember the old launch date.', + 'Forget the old launch date.', + ]) + expect(childCalls).toEqual(parentCalls) + }) + + it('includes the default scope in the Mem0 adapter identity', () => { + const base = { mode: 'hosted' as const, id: 'mem0' } + expect(mem0MemoryAdapterIdentity({ ...base, defaultScope: { tenantId: 'tenant-a' } })).not.toBe( + mem0MemoryAdapterIdentity({ ...base, defaultScope: { tenantId: 'tenant-b' } }), + ) + }) + + it('passes OSS entity filters on add', async () => { + let addOptions: Record | undefined + const adapter = createMem0MemoryAdapter({ + mode: 'oss', + client: { + async add(_messages, options) { + addOptions = options + return { results: [{ id: 'oss-1', memory: 'Remembered' }] } + }, + async search() { + return { results: [] } + }, + }, + }) + + await adapter.write({ + kind: 'fact', + text: 'Remembered', + scope: { userId: 'user-1', agentId: 'agent-1', namespace: 'run-1' }, + }) + + expect(addOptions).toMatchObject({ + userId: 'user-1', + agentId: 'agent-1', + runId: 'run-1', + filters: { user_id: 'user-1', agent_id: 'agent-1', run_id: 'run-1' }, + }) + }) + + it('deletes only memories matching a complete scoped filter', async () => { + const rows = new Set(['memory-1', 'memory-2']) + const getAllOptions: Record[] = [] + let deleteAllCalls = 0 + const adapter = createMem0MemoryAdapter({ + mode: 'hosted', + appId: 'support-app', + client: { + async add() { + return [] + }, + async search() { + return { results: [] } + }, + async getAll(options) { + getAllOptions.push(options ?? {}) + return { results: [...rows].map((id) => ({ id, memory: id })) } + }, + async delete(memoryId) { + rows.delete(memoryId) + return { message: 'deleted' } + }, + async deleteAll() { + deleteAllCalls += 1 + return { message: 'deleted all' } + }, + }, + }) + + await adapter.clear?.({ + tenantId: 'tenant-1', + userId: 'user-1', + namespace: 'physical-branch', + runId: 'logical-run', + sessionId: 'session-1', + tags: { visibility: 'private' }, + }) + + expect(rows.size).toBe(0) + expect(deleteAllCalls).toBe(0) + expect(getAllOptions[0]).toMatchObject({ + filters: { + app_id: 'support-app', + tenant_id: 'tenant-1', + user_id: 'user-1', + run_id: 'physical-branch', + logical_run_id: 'logical-run', + session_id: 'session-1', + tag_visibility: 'private', + }, + page: 1, + pageSize: 100, + showExpired: true, + }) + }) +}) + +describe('Graphiti adapter', () => { + it('uses official MCP request shapes and rejoins fact provenance to episodes', async () => { + const requests: Array<{ name: string; arguments?: Record }> = [] + const episodes = new Map>() + const client: GraphitiMcpClientLike = { + async callTool(request) { + requests.push(request) + const args = request.arguments ?? {} + if (request.name === 'add_memory') { + episodes.set(String(args.uuid), { + uuid: args.uuid, + name: args.name, + content: args.episode_body, + source_description: args.source_description, + group_id: args.group_id, + }) + return { structuredContent: { message: 'queued' } } + } + if (request.name === 'get_episodes') { + return { + content: [{ type: 'text', text: JSON.stringify({ episodes: [...episodes.values()] }) }], + } + } + if (request.name === 'search_memory_facts') { + const episode = [...episodes.values()][0]! + return { + structuredContent: { + facts: [ + { + uuid: 'fact-1', + name: 'PREFERS', + fact: 'The user prefers concise status updates.', + group_id: episode.group_id, + episodes: [episode.uuid], + }, + ], + }, + } + } + if (request.name === 'search_nodes') { + return { structuredContent: { nodes: [] } } + } + throw new Error(`unexpected tool ${request.name}`) + }, + } + const adapter = createGraphitiMemoryAdapter({ + client, + search: ['facts'], + pollIntervalMs: 1, + ingestionTimeoutMs: 50, + }) + + const write = await adapter.write({ + id: 'event-1', + kind: 'message', + role: 'user', + text: 'I prefer concise status updates.', + scope: { userId: 'user-1', namespace: 'branch-1' }, + metadata: { actorId: 'user', timestamp: '2026-07-17T12:00:00Z' }, + }) + const hits = await adapter.search('status preference', { + scope: { userId: 'user-1', namespace: 'branch-1' }, + limit: 5, + }) + + expect(requests[0]).toMatchObject({ + name: 'add_memory', + arguments: { + episode_body: 'I prefer concise status updates.', + source: 'message', + reference_time: '2026-07-17T12:00:00Z', + }, + }) + expect(String(requests[0]?.arguments?.group_id)).toMatch(/^ak_[a-f0-9]{32}$/) + expect(String(requests[0]?.arguments?.uuid)).toMatch( + /^[a-f0-9]{8}-[a-f0-9]{4}-5[a-f0-9]{3}-a[a-f0-9]{3}-[a-f0-9]{12}$/, + ) + expect(write.metadata).toMatchObject({ consistency: 'visible', queued: false }) + expect(hits[0]).toMatchObject({ + id: 'fact-1', + kind: 'fact', + text: 'The user prefers concise status updates.', + metadata: { eventId: 'event-1', actorId: 'user' }, + }) + }) + + it('surfaces Graphiti tool errors', async () => { + const adapter = createGraphitiMemoryAdapter({ + client: { + async callTool() { + return { structuredContent: { error: 'database unavailable' } } + }, + }, + }) + + await expect(adapter.search('anything')).rejects.toThrow('database unavailable') + }) + + it('expands the episode scan when Graphiti truncates a long group', async () => { + const episodes = Array.from({ length: 150 }, (_, index) => ({ + uuid: `older-${index.toString().padStart(3, '0')}`, + })) + const limits: number[] = [] + const adapter = createGraphitiMemoryAdapter({ + pollIntervalMs: 1, + ingestionTimeoutMs: 50, + client: { + async callTool(request) { + if (request.name === 'add_memory') { + episodes.push({ uuid: String(request.arguments?.uuid) }) + return { structuredContent: { message: 'queued' } } + } + if (request.name === 'get_episodes') { + const limit = Number(request.arguments?.max_episodes) + limits.push(limit) + return { structuredContent: { episodes: episodes.slice(0, limit) } } + } + throw new Error(`unexpected tool ${request.name}`) + }, + }, + }) + + await adapter.write({ kind: 'fact', text: 'Visible beyond the first page.' }) + + expect(limits).toEqual([100, 200]) + }) + + it('keeps polling a full Graphiti group while the new episode becomes visible', async () => { + const older = Array.from({ length: 100 }, (_, index) => ({ uuid: `older-${index}` })) + let pendingUuid = '' + let visible = false + let polls = 0 + const adapter = createGraphitiMemoryAdapter({ + episodeScanLimit: 100, + pollIntervalMs: 1, + ingestionTimeoutMs: 50, + client: { + async callTool(request) { + if (request.name === 'add_memory') { + pendingUuid = String(request.arguments?.uuid) + setTimeout(() => { + visible = true + }, 5) + return { structuredContent: { message: 'queued' } } + } + if (request.name === 'get_episodes') { + polls += 1 + return { + structuredContent: { + episodes: visible ? [...older.slice(1), { uuid: pendingUuid }] : older, + }, + } + } + throw new Error(`unexpected tool ${request.name}`) + }, + }, + }) + + await expect( + adapter.write({ kind: 'fact', text: 'Eventually visible.' }), + ).resolves.toMatchObject({ accepted: true }) + expect(polls).toBeGreaterThan(1) + }) + + it('fails explicitly when visibility cannot be proved within the episode scan limit', async () => { + const episodes = Array.from({ length: 100 }, (_, index) => ({ uuid: `older-${index}` })) + const adapter = createGraphitiMemoryAdapter({ + episodeScanLimit: 100, + pollIntervalMs: 1, + ingestionTimeoutMs: 50, + client: { + async callTool(request) { + if (request.name === 'add_memory') { + episodes.push({ uuid: String(request.arguments?.uuid) }) + return { structuredContent: { message: 'queued' } } + } + if (request.name === 'get_episodes') { + return { structuredContent: { episodes: episodes.slice(0, 100) } } + } + throw new Error(`unexpected tool ${request.name}`) + }, + }, + }) + + await expect(adapter.write({ kind: 'fact', text: 'Outside the scan window.' })).rejects.toThrow( + 'episodeScanLimit=100', + ) + }) + + it('supports documented tool aliases and does not invent missing relevance scores', async () => { + const names: string[] = [] + const adapter = createGraphitiMemoryAdapter({ + search: ['facts'], + toolNames: { searchFacts: 'search_facts' }, + client: { + async callTool(request) { + names.push(request.name) + return { + structuredContent: { + facts: [{ uuid: 'fact-1', fact: 'An unscored fact', episodes: [] }], + }, + } + }, + }, + }) + + const hits = await adapter.search('fact', { minScore: 0.5 }) + + expect(names).toEqual(['search_facts']) + expect(hits).toEqual([]) + }) + + it('isolates run, session, and tag scopes into different Graphiti groups', async () => { + const groupIds: string[] = [] + const adapter = createGraphitiMemoryAdapter({ + search: ['facts'], + client: { + async callTool(request) { + if (request.name === 'search_memory_facts') { + groupIds.push(String((request.arguments?.group_ids as string[])[0])) + return { structuredContent: { facts: [] } } + } + return { structuredContent: { episodes: [] } } + }, + }, + }) + + await adapter.search('query', { + scope: { namespace: 'same', runId: 'run-a', sessionId: 'session', tags: { arm: 'a' } }, + }) + await adapter.search('query', { + scope: { namespace: 'same', runId: 'run-b', sessionId: 'session', tags: { arm: 'b' } }, + }) + + expect(new Set(groupIds).size).toBe(2) + }) + + it('keeps episode ids stable when independent groups execute in a different order', async () => { + const run = async (scopes: AgentMemoryScope[]) => { + const ids = new Map() + const adapter = createGraphitiMemoryAdapter({ + consistency: 'queued', + client: { + async callTool(request) { + if (request.name === 'add_memory') { + ids.set(String(request.arguments?.group_id), String(request.arguments?.uuid)) + } + return { structuredContent: { message: 'queued' } } + }, + }, + }) + for (const scope of scopes) { + await adapter.write({ kind: 'fact', text: 'same fact', scope }) + } + return ids + } + const first = await run([{ namespace: 'a' }, { namespace: 'b' }]) + const reversed = await run([{ namespace: 'b' }, { namespace: 'a' }]) + + expect(first).toEqual(reversed) + }) + + it('uses a new episode id when the content of one source event changes', async () => { + const episodeIds: string[] = [] + const adapter = createGraphitiMemoryAdapter({ + consistency: 'queued', + client: { + async callTool(request) { + if (request.name === 'add_memory') { + episodeIds.push(String(request.arguments?.uuid)) + } + return { structuredContent: { message: 'queued' } } + }, + }, + }) + + await adapter.write({ id: 'source-1', kind: 'fact', text: 'Launch is Friday.' }) + await adapter.write({ id: 'source-1', kind: 'fact', text: 'Launch is Monday.' }) + + expect(episodeIds).toHaveLength(2) + expect(episodeIds[0]).not.toBe(episodeIds[1]) + }) + + it('uses Graphiti clear_graph for exact group cleanup', async () => { + const requests: Array<{ name: string; arguments?: Record }> = [] + const adapter = createGraphitiMemoryAdapter({ + client: { + async callTool(request) { + requests.push(request) + return { structuredContent: { message: 'Graph cleared successfully' } } + }, + }, + }) + + await adapter.clear?.({ userId: 'user-1', namespace: 'candidate-1' }) + + expect(requests).toHaveLength(1) + expect(requests[0]).toMatchObject({ + name: 'clear_graph', + arguments: { group_ids: [expect.stringMatching(/^ak_[a-f0-9]{32}$/)] }, + }) + }) + + it('rejects queued ingestion for resumable branch experiments', () => { + const adapter = createGraphitiMemoryAdapter({ + consistency: 'queued', + client: { + async callTool() { + return { structuredContent: { message: 'queued' } } + }, + }, + }) + + expect(() => createAgentMemoryBranch({ adapter, branchId: 'candidate-1' })).toThrow( + 'queued Graphiti writes can become visible after a process restart', + ) + }) + + it('includes the default scope in the Graphiti adapter identity', () => { + expect( + graphitiMemoryAdapterIdentity({ id: 'graphiti', defaultScope: { namespace: 'one' } }), + ).not.toBe( + graphitiMemoryAdapterIdentity({ id: 'graphiti', defaultScope: { namespace: 'two' } }), + ) + }) +}) + +describe('Neo4j Agent Memory adapter', () => { + it('is type-compatible with the current official client', () => { + const client = {} as unknown as Neo4jMemoryClient + expect( + createNeo4jAgentMemoryAdapter({ client, transport: 'rest', branchId: 'branch-1' }).id, + ).toBe('neo4j-agent-memory') + }) + + it('stores observations as searchable messages without retrying a successful void write', async () => { + let camelWrites = 0 + let snakeWrites = 0 + let role: unknown + let closes = 0 + const adapter = createNeo4jAgentMemoryAdapter({ + transport: 'rest', + branchId: 'branch-1', + client: { + shortTerm: { + async addMessage(_sessionId: unknown, inputRole: unknown) { + camelWrites += 1 + role = inputRole + return undefined + }, + async add_message() { + snakeWrites += 1 + return { id: 'duplicate' } + }, + async searchMessages() { + return [ + { + id: 'observation-1', + role: 'assistant', + content: 'The deployment completed.', + metadata: { agentKnowledgeKind: 'observation' }, + }, + ] + }, + }, + async close() { + closes += 1 + }, + }, + }) + + const write = await adapter.write({ + kind: 'observation', + role: 'tool', + text: 'The deployment completed.', + scope: { sessionId: 'session-1' }, + }) + const hits = await adapter.search('deployment', { + kinds: ['observation'], + scope: { sessionId: 'session-1' }, + }) + await adapter.close?.() + + expect(write.accepted).toBe(true) + expect(camelWrites).toBe(1) + expect(snakeWrites).toBe(0) + expect(role).toBe('assistant') + expect(hits).toMatchObject([{ id: 'observation-1', kind: 'observation' }]) + expect(closes).toBe(1) + }) + + it('fails clearly when REST is asked for bridge-only facts and preferences', async () => { + const adapter = createNeo4jAgentMemoryAdapter({ + transport: 'rest', + client: { longTerm: {} }, + }) + + await expect(adapter.write({ kind: 'fact', text: 'A fact' })).rejects.toThrow( + 'Neo4j REST cannot write fact', + ) + await expect(adapter.search('preference', { kinds: ['preference'] })).rejects.toThrow( + 'Neo4j rest cannot search memory kinds: preference', + ) + }) + + it('rejects an incompatible Neo4j client instead of reporting an empty search', async () => { + const adapter = createNeo4jAgentMemoryAdapter({ + client: { longTerm: {} }, + transport: 'rest', + }) + + await expect(adapter.search('company', { kinds: ['entity'] })).rejects.toThrow( + 'missing method searchEntities', + ) + }) + + it('maps hosted reasoning memories to recordStep', async () => { + let recorded: Record | undefined + const adapter = createNeo4jAgentMemoryAdapter({ + transport: 'rest', + client: { + reasoning: { + async recordStep(input: Record) { + recorded = input + return { id: 'step-1', ...input } + }, + }, + }, + }) + + const result = await adapter.write({ + kind: 'reasoning-trace', + text: 'Check the deployment status.', + title: 'status-check', + scope: { sessionId: 'conversation-1' }, + metadata: { action: 'query_status', result: 'healthy' }, + }) + + expect(recorded).toEqual({ + conversationId: 'conversation-1', + reasoning: 'Check the deployment status.', + actionTaken: 'query_status', + result: 'healthy', + }) + expect(result).toMatchObject({ accepted: true, id: 'step-1', kind: 'reasoning-trace' }) + }) + + it('uses bridge preference APIs only in bridge mode', async () => { + const adapter = createNeo4jAgentMemoryAdapter({ + transport: 'bridge', + client: { + longTerm: { + async addPreference(category: string, preference: string) { + return { id: 'preference-1', category, preference } + }, + async searchPreferences() { + return [{ id: 'preference-1', category: 'style', preference: 'Be concise.' }] + }, + }, + }, + }) + + await expect( + adapter.write({ kind: 'preference', category: 'style', text: 'Be concise.' }), + ).resolves.toMatchObject({ id: 'preference-1' }) + await expect(adapter.search('concise', { kinds: ['preference'] })).resolves.toMatchObject([ + { id: 'preference-1', kind: 'preference', text: 'Be concise.' }, + ]) + }) +}) + +describe('memory branches', () => { + it('rejects adapters that do not declare how branches are isolated', () => { + const { branchIsolation: _branchIsolation, ...legacy } = createScopedTestAdapter('legacy') + + expect(() => + createAgentMemoryBranch({ + adapter: legacy, + branchId: 'candidate-a', + }), + ).toThrow(/adapter must declare branchIsolation/) + }) + + it('rejects Neo4j clients that were not isolated for the exact branch', () => { + const unscoped = createNeo4jAgentMemoryAdapter({ client: {}, transport: 'rest' }) + expect(() => createAgentMemoryBranch({ adapter: unscoped, branchId: 'candidate-a' })).toThrow( + 'create a separate MemoryClient namespace per branch', + ) + + const scoped = createNeo4jAgentMemoryAdapter({ + client: {}, + transport: 'rest', + branchId: 'candidate-a', + }) + expect( + createAgentMemoryBranch({ + adapter: scoped, + branchId: 'candidate-a', + policy: { read: ['shared'], write: 'shared' }, + }).branchId, + ).toBe('candidate-a') + expect(() => createAgentMemoryBranch({ adapter: scoped, branchId: 'candidate-a' })).toThrow( + 'only shared memory policy', + ) + expect(() => createAgentMemoryBranch({ adapter: scoped, branchId: 'candidate-b' })).toThrow( + "adapter instance belongs to branch 'candidate-a'", + ) + }) + + it('isolates branches and private agents while allowing team sharing', async () => { + const storage = createScopedTestAdapter('scoped') + const alpha = createAgentMemoryBranch({ + adapter: storage, + branchId: 'alpha', + policy: { read: ['private'], write: 'private' }, + baseScope: { tenantId: 'tenant', userId: 'user' }, + }) + const beta = createAgentMemoryBranch({ + adapter: storage, + branchId: 'beta', + policy: { read: ['private'], write: 'private' }, + baseScope: { tenantId: 'tenant', userId: 'user' }, + }) + + await alpha.write({ + kind: 'fact', + text: 'alpha private', + scope: { agentId: 'agent-a', teamId: 'team-1' }, + }) + await beta.write({ + kind: 'fact', + text: 'beta private', + scope: { agentId: 'agent-a', teamId: 'team-1' }, + }) + + expect((await alpha.search('private', { scope: { agentId: 'agent-a' } })).map(hitText)).toEqual( + ['alpha private'], + ) + expect((await alpha.search('private', { scope: { agentId: 'agent-b' } })).map(hitText)).toEqual( + [], + ) + expect((await beta.search('private', { scope: { agentId: 'agent-a' } })).map(hitText)).toEqual([ + 'beta private', + ]) + + const team = createAgentMemoryBranch({ + adapter: storage, + branchId: 'team-branch', + policy: { read: ['team'], write: 'team' }, + baseScope: { tenantId: 'tenant', userId: 'user' }, + }) + await team.write({ + kind: 'observation', + text: 'shared with the team', + scope: { agentId: 'agent-a', teamId: 'team-1' }, + }) + const teamHits = await team.search('team', { + scope: { agentId: 'agent-b', teamId: 'team-1' }, + }) + expect(teamHits.map(hitText)).toEqual(['shared with the team']) + }) + + it('serializes writes per actor and permits independent actors in parallel', async () => { + let active = 0 + let maxActive = 0 + const starts: string[] = [] + const storage = createScopedTestAdapter('ordered', async (scope, text) => { + starts.push(`${scope.agentId}:${text}`) + active += 1 + maxActive = Math.max(maxActive, active) + await new Promise((resolve) => setTimeout(resolve, 10)) + active -= 1 + }) + const branch = createAgentMemoryBranch({ + adapter: storage, + branchId: 'parallel', + policy: { read: ['private'], write: 'private' }, + }) + + await Promise.all([ + branch.write({ kind: 'message', text: 'a1', scope: { agentId: 'a' } }), + branch.write({ kind: 'message', text: 'a2', scope: { agentId: 'a' } }), + branch.write({ kind: 'message', text: 'b1', scope: { agentId: 'b' } }), + ]) + + expect(maxActive).toBe(2) + expect(starts.indexOf('a:a1')).toBeLessThan(starts.indexOf('a:a2')) + expect((await branch.snapshot()).journal.map((entry) => entry.input.text)).toEqual([ + 'a1', + 'a2', + 'b1', + ]) + }) + + it('preserves one actor ordering across sessions and tags', async () => { + const completed: string[] = [] + const storage = createScopedTestAdapter('actor-order', async (_scope, text) => { + if (text === 'first') await new Promise((resolve) => setTimeout(resolve, 20)) + completed.push(text) + }) + const branch = createAgentMemoryBranch({ + adapter: storage, + branchId: 'actor-order', + baseScope: { tenantId: 'tenant', agentId: 'agent-a' }, + }) + + await Promise.all([ + branch.write({ + kind: 'fact', + text: 'first', + scope: { sessionId: 'session-1', tags: { task: 'one' } }, + }), + branch.write({ + kind: 'fact', + text: 'second', + scope: { sessionId: 'session-2', tags: { task: 'two' } }, + }), + ]) + + expect(completed).toEqual(['first', 'second']) + }) + + it('takes a point-in-time snapshot while later writes wait at the boundary', async () => { + let releaseFirst!: () => void + const firstMayFinish = new Promise((resolve) => { + releaseFirst = resolve + }) + let reportFirstStarted!: () => void + const firstStarted = new Promise((resolve) => { + reportFirstStarted = resolve + }) + const providerStarts: string[] = [] + const storage = createScopedTestAdapter('snapshot-boundary', async (_scope, text) => { + providerStarts.push(text) + if (text === 'first') { + reportFirstStarted() + await firstMayFinish + } + }) + const branch = createAgentMemoryBranch({ + adapter: storage, + branchId: 'snapshot-boundary', + baseScope: { agentId: 'worker' }, + }) + + const firstWrite = branch.write({ kind: 'fact', text: 'first' }) + await firstStarted + const snapshotPromise = branch.snapshot() + const secondWrite = branch.write({ kind: 'fact', text: 'second' }) + await Promise.resolve() + + expect(providerStarts).toEqual(['first']) + releaseFirst() + await firstWrite + const snapshot = await snapshotPromise + await secondWrite + + expect(snapshot.journal.map((entry) => entry.input.text)).toEqual(['first']) + expect(providerStarts).toEqual(['first', 'second']) + }) + + it('does not clear storage while a preceding read is still using it', async () => { + let finishRead!: () => void + const readMayFinish = new Promise((resolve) => { + finishRead = resolve + }) + let reportReadStarted!: () => void + const readStarted = new Promise((resolve) => { + reportReadStarted = resolve + }) + let clearStarted = false + const adapter: AgentMemoryAdapter = { + id: 'read-clear-boundary', + branchIsolation: { mode: 'scoped' }, + async search() { + reportReadStarted() + await readMayFinish + return [] + }, + async getContext(query) { + return { query, text: '', hits: [], sourceRecords: [] } + }, + async write(input) { + return { accepted: true, id: 'write', uri: 'memory://write', kind: input.kind } + }, + async clear() { + clearStarted = true + }, + } + const branch = createAgentMemoryBranch({ + adapter, + branchId: 'read-clear-boundary', + baseScope: { agentId: 'worker' }, + }) + + const read = branch.search('active read') + await readStarted + const clear = branch.clear?.() + await Promise.resolve() + expect(clearStarted).toBe(false) + + finishRead() + await read + await clear + expect(clearStarted).toBe(true) + }) + + it('clears a touched scope after the provider commits and then throws', async () => { + let dirty = false + const clearedScopes: AgentMemoryScope[] = [] + const adapter: AgentMemoryAdapter = { + id: 'commit-then-error', + branchIsolation: { mode: 'scoped' }, + async search() { + return [] + }, + async getContext(query) { + return { query, text: '', hits: [], sourceRecords: [] } + }, + async write() { + dirty = true + throw new Error('connection lost after commit') + }, + async clear(scope) { + clearedScopes.push(scope ?? {}) + dirty = false + }, + } + const branch = createAgentMemoryBranch({ + adapter, + branchId: 'commit-then-error', + baseScope: { tenantId: 'tenant' }, + }) + + await expect( + branch.write({ kind: 'fact', text: 'partial', scope: { agentId: 'worker' } }), + ).rejects.toThrow('connection lost after commit') + await branch.clear?.() + + expect(dirty).toBe(false) + expect(clearedScopes).toHaveLength(1) + expect(clearedScopes[0]).toMatchObject({ tenantId: 'tenant', agentId: 'worker' }) + }) + + it('rejects non-durable journal data before it can corrupt resume state', async () => { + let providerWrites = 0 + const adapter = createScopedTestAdapter('durable-journal', async () => { + providerWrites += 1 + }) + const branch = createAgentMemoryBranch({ + adapter, + branchId: 'durable-journal', + baseScope: { agentId: 'worker' }, + }) + const cyclic: Record = {} + cyclic.self = cyclic + + await expect( + branch.write({ kind: 'fact', text: 'bad metadata', metadata: cyclic }), + ).rejects.toThrow('must not contain cycles') + expect(providerWrites).toBe(0) + expect((await branch.snapshot()).journal).toEqual([]) + }) + + it('returns snapshots detached from live branch state', async () => { + const adapter = createScopedTestAdapter('detached-snapshot') + const branch = createAgentMemoryBranch({ + adapter, + branchId: 'detached-snapshot', + baseScope: { tenantId: 'tenant-original', agentId: 'agent-a' }, + }) + await branch.write({ + kind: 'fact', + text: 'original text', + metadata: { nested: { status: 'original' } }, + }) + + const exposed = await branch.snapshot() + exposed.baseScope.tenantId = 'tenant-mutated' + const nested = exposed.journal[0]!.input.metadata!.nested as Record + nested.status = 'mutated' + const current = await branch.snapshot() + + expect(current.baseScope.tenantId).toBe('tenant-original') + expect(current.journal[0]?.input.metadata).toEqual({ nested: { status: 'original' } }) + expect(await branch.search('original')).toHaveLength(1) + }) + + it('closes a shared adapter once after every branch releases it', async () => { + let closes = 0 + const base = createScopedTestAdapter('shared-lifecycle') + const adapter: AgentMemoryAdapter = { + ...base, + async close() { + closes += 1 + }, + } + const first = createAgentMemoryBranch({ + adapter, + branchId: 'first', + baseScope: { agentId: 'agent-1' }, + }) + const second = createAgentMemoryBranch({ + adapter, + branchId: 'second', + baseScope: { agentId: 'agent-2' }, + }) + + await first.close?.() + await first.close?.() + expect(closes).toBe(0) + await second.close?.() + await second.close?.() + + expect(closes).toBe(1) + await expect(first.search('after close')).rejects.toThrow('closed branch') + }) + + it('forks by replaying accepted writes into another adapter and resumes exactly', async () => { + const parentStorage = createScopedTestAdapter('parent') + const childStorage = createScopedTestAdapter('child') + const parent = createAgentMemoryBranch({ + adapter: parentStorage, + branchId: 'parent', + policy: { read: ['private'], write: 'private' }, + baseScope: { tenantId: 'tenant', agentId: 'agent-a' }, + }) + await parent.write({ kind: 'fact', text: 'seed fact' }) + + await expect(parent.fork({ branchId: 'parent' })).rejects.toThrow( + 'child branchId must differ from its parent', + ) + expect((await parent.search('fact')).map(hitText)).toEqual(['seed fact']) + + const child = await parent.fork({ branchId: 'child', adapter: childStorage }) + await child.write({ kind: 'fact', text: 'child only' }) + + expect((await parent.search('fact')).map(hitText)).toEqual(['seed fact']) + expect((await child.search('fact')).map(hitText)).toEqual(['seed fact', 'child only']) + const snapshot = await child.snapshot() + const resumed = createAgentMemoryBranch({ + adapter: childStorage, + branchId: 'child', + snapshot, + }) + snapshot.baseScope.tenantId = 'mutated-after-resume' + expect((await resumed.search('fact')).map(hitText)).toEqual(['seed fact', 'child only']) + expect((await resumed.snapshot()).baseScope.tenantId).toBe('tenant') + expect((await resumed.snapshot()).digest).toBe(snapshot.digest) + + await expect( + Promise.resolve().then(() => + createAgentMemoryBranch({ + adapter: childStorage, + branchId: 'child', + snapshot: { ...snapshot, digest: 'sha256:bad' }, + }), + ), + ).rejects.toThrow('digest mismatch') + }) +}) + +describe('agent memory experiments', () => { + it('runs existing ordered memory benchmark cases without reshaping the dataset', async () => { + const sequences = buildAgentMemorySequencesFromBenchmarkCases([ + { + id: 'launch-update', + family: 'longmemeval', + taskKind: 'memory-temporal', + split: 'holdout', + events: [ + { id: 'old', actorId: 'pm', text: 'Launch is April 3.' }, + { id: 'current', actorId: 'pm', text: 'Launch moved to April 17.' }, + ], + prompt: 'When is launch?', + requiredFacts: [{ id: 'current-date', anyOf: ['April 17'] }], + forbiddenFacts: [{ id: 'old-date', anyOf: ['April 3'], obsolete: true }], + expectedEventIds: ['current'], + expectedActorIds: ['pm'], + }, + ]) + const result = await runAgentMemoryExperiment({ + experimentId: 'benchmark-conversion', + sequences, + candidates: [ + { + id: 'literal-memory', + createAdapter: () => createScopedTestAdapter('literal-memory'), + }, + ], + runDir: '/runs/benchmark-conversion', + storage: inMemoryCampaignStorage(), + }) + + expect(sequences[0]?.steps.map((step) => step.id)).toEqual([ + 'event:old', + 'event:current', + 'probe', + ]) + expect(result.rows[0]).toMatchObject({ + scoreMean: 0.75, + totalSequences: 1, + totalProbes: 1, + cellsFailed: 0, + }) + expect(result.rows[0]?.dimensions).toMatchObject({ memory_stale_safe: 0 }) + }) + + it('keeps each history ordered while comparing candidate branches in parallel', async () => { + const storage = inMemoryCampaignStorage() + const snapshots: string[] = [] + const stepOrder = new Map() + let active = 0 + let maxActive = 0 + let release: (() => void) | undefined + const twoActive = new Promise((resolve) => { + release = resolve + }) + const sequence = (id: string) => ({ + id, + family: 'first-party' as const, + split: 'holdout' as const, + steps: [ + { + id: 'research', + scope: { agentId: 'researcher', teamId: 'team-1' }, + writes: [ + { + id: `${id}-event`, + kind: 'fact' as const, + text: `${id} launch date is Friday`, + metadata: { eventId: `${id}-event`, actorId: 'researcher' }, + }, + ], + }, + { + id: 'delivery', + scope: { agentId: 'builder', teamId: 'team-1' }, + probes: [ + { + id: 'launch-date', + query: `${id} launch date`, + requiredFacts: [{ id: 'current', anyOf: [`${id} launch date is Friday`] }], + expectedEventIds: [`${id}-event`], + expectedActorIds: ['researcher'], + }, + ], + }, + ], + }) + + const result = await runAgentMemoryExperiment({ + experimentId: 'team-sharing-vs-private', + sequences: [sequence('alpha'), sequence('beta')], + candidates: [ + { + id: 'private', + policy: { read: ['private'], write: 'private' }, + createAdapter: ({ branchId }) => createScopedTestAdapter(`private:${branchId}`), + }, + { + id: 'team', + policy: { read: ['team'], write: 'team' }, + createAdapter: ({ branchId }) => createScopedTestAdapter(`team:${branchId}`), + }, + ], + runDir: '/runs/team-sharing-vs-private', + storage, + maxConcurrency: 4, + cleanupBranches: true, + executeStepRef: 'test-runtime/v1', + executeStep: async ({ memory, step }) => { + const order = stepOrder.get(memory.branchId) ?? [] + order.push(step.id) + stepOrder.set(memory.branchId, order) + if (step.id === 'research') { + active += 1 + maxActive = Math.max(maxActive, active) + if (active === 2) release?.() + await twoActive + active -= 1 + } + }, + onBranchSnapshot: ({ snapshot }) => { + snapshots.push(snapshot.digest) + }, + }) + + expect(maxActive).toBeGreaterThanOrEqual(2) + expect(result.rows[0]).toMatchObject({ + rank: 1, + candidateId: 'team', + scoreMean: 1, + passRate: 1, + totalSequences: 2, + totalCells: 2, + totalProbes: 2, + cellsFailed: 0, + }) + expect(result.rows[1]?.candidateId).toBe('private') + expect(result.rows[1]?.scoreMean).toBeLessThan(0.3) + expect(result.campaign.cells).toHaveLength(4) + expect(snapshots).toHaveLength(4) + expect([...stepOrder.values()].every((steps) => steps.join(',') === 'research,delivery')).toBe( + true, + ) + expect(storage.read(result.rankingJsonPath)).toContain('"candidateId": "team"') + expect(storage.read(result.rankingMarkdownPath)).toContain('| 1 | team |') + }) + + it('uses distinct external branch ids for distinct run directories', async () => { + const branchIds: string[] = [] + const sequence = { + id: 'branch-id', + family: 'first-party' as const, + steps: [ + { + id: 'probe', + scope: { agentId: 'worker' }, + probes: [{ id: 'state', query: 'state', referenceAnswer: 'state' }], + }, + ], + } + const run = (runDir: string) => + runAgentMemoryExperiment({ + experimentId: 'same-experiment', + sequences: [sequence], + candidates: [ + { + id: 'memory', + createAdapter({ branchId }) { + branchIds.push(branchId) + return createScopedTestAdapter(branchId) + }, + }, + ], + runDir, + storage: inMemoryCampaignStorage(), + }) + + await run('/runs/branch-id-a') + await run('/runs/branch-id-b') + + expect(branchIds).toHaveLength(2) + expect(branchIds[0]).not.toBe(branchIds[1]) + }) + + it('uses a stable external branch id across worker paths when experimentRunId matches', async () => { + const branchIds: string[] = [] + const run = (runDir: string) => + runAgentMemoryExperiment({ + experimentId: 'distributed-experiment', + experimentRunId: 'distributed-run-17', + sequences: [ + { + id: 'shared-history', + family: 'first-party', + steps: [ + { + id: 'probe', + scope: { agentId: 'worker' }, + probes: [{ id: 'state', query: 'state', referenceAnswer: 'state' }], + }, + ], + }, + ], + candidates: [ + { + id: 'memory', + createAdapter({ branchId }) { + branchIds.push(branchId) + return createScopedTestAdapter(branchId) + }, + }, + ], + runDir, + storage: inMemoryCampaignStorage(), + }) + + await run('/worker-a/run') + await run('/worker-b/run') + + expect(branchIds).toHaveLength(2) + expect(branchIds[0]).toBe(branchIds[1]) + }) + + it('waits for timed-out provider work to finish cleanup before returning', async () => { + let releaseWrite!: () => void + const writeMayFinish = new Promise((resolve) => { + releaseWrite = resolve + }) + let reportWriteStarted!: () => void + const writeStarted = new Promise((resolve) => { + reportWriteStarted = resolve + }) + const rows: AgentMemoryHit[] = [] + let clears = 0 + const adapter: AgentMemoryAdapter = { + id: 'slow-provider', + branchIsolation: { mode: 'scoped' }, + async search() { + return [...rows] + }, + async getContext(query) { + return { query, text: rows.map(hitText).join('\n'), hits: [...rows], sourceRecords: [] } + }, + async write(input) { + reportWriteStarted() + await writeMayFinish + const hit = { + id: 'late-write', + uri: 'memory://slow-provider/late-write', + kind: input.kind, + text: input.text, + } + rows.push(hit) + return { accepted: true, id: hit.id, uri: hit.uri, kind: hit.kind } + }, + async clear() { + clears += 1 + rows.length = 0 + }, + } + const run = runAgentMemoryExperiment({ + experimentId: 'timeout-cleanup', + sequences: [ + { + id: 'slow-history', + family: 'first-party', + steps: [ + { + id: 'write', + scope: { agentId: 'worker' }, + writes: [{ kind: 'fact', text: 'late fact' }], + probes: [{ id: 'fact', query: 'fact', referenceAnswer: 'late fact' }], + }, + ], + }, + ], + candidates: [{ id: 'slow', createAdapter: () => adapter }], + runDir: '/runs/timeout-cleanup', + storage: inMemoryCampaignStorage(), + dispatchTimeoutMs: 5, + }) + await writeStarted + let settled = false + void run.then( + () => { + settled = true + }, + () => { + settled = true + }, + ) + await new Promise((resolve) => setTimeout(resolve, 15)) + + expect(settled).toBe(false) + releaseWrite() + const result = await run + + expect(result.rows[0]).toMatchObject({ cellsFailed: 1 }) + expect(result.campaign.cells[0]?.error).toContain('dispatch exceeded 5ms') + expect(clears).toBeGreaterThanOrEqual(2) + expect(rows).toEqual([]) + }) + + it('fails when cleanup after a timed-out provider write fails', async () => { + let releaseWrite!: () => void + const writeMayFinish = new Promise((resolve) => { + releaseWrite = resolve + }) + let reportWriteStarted!: () => void + const writeStarted = new Promise((resolve) => { + reportWriteStarted = resolve + }) + let clears = 0 + const adapter: AgentMemoryAdapter = { + id: 'cleanup-failure', + branchIsolation: { mode: 'scoped' }, + async search() { + return [] + }, + async getContext(query) { + return { query, text: '', hits: [], sourceRecords: [] } + }, + async write(input) { + reportWriteStarted() + await writeMayFinish + return { + accepted: true, + id: 'late-write', + uri: 'memory://cleanup-failure/late-write', + kind: input.kind, + } + }, + async clear() { + clears += 1 + if (clears > 1) throw new Error('provider cleanup unavailable') + }, + } + const run = runAgentMemoryExperiment({ + experimentId: 'timeout-cleanup-failure', + sequences: [ + { + id: 'slow-history', + family: 'first-party', + steps: [ + { + id: 'write', + scope: { agentId: 'worker' }, + writes: [{ kind: 'fact', text: 'late fact' }], + probes: [{ id: 'fact', query: 'fact', referenceAnswer: 'late fact' }], + }, + ], + }, + ], + candidates: [{ id: 'slow', createAdapter: () => adapter }], + runDir: '/runs/timeout-cleanup-failure', + storage: inMemoryCampaignStorage(), + dispatchTimeoutMs: 5, + }) + await writeStarted + await new Promise((resolve) => setTimeout(resolve, 10)) + releaseWrite() + + await expect(run).rejects.toThrow('memory experiment cleanup failed after dispatch') + }) + + it('uses the same random seed for every candidate on one history and repetition', async () => { + const seeds = new Map() + const sequence = { + id: 'paired-history', + family: 'first-party' as const, + steps: [ + { + id: 'probe', + scope: { agentId: 'worker' }, + probes: [{ id: 'state', query: 'state', referenceAnswer: 'state' }], + }, + ], + } + await runAgentMemoryExperiment({ + experimentId: 'paired-seeds', + sequences: [sequence], + candidates: ['a', 'b'].map((candidateId) => ({ + id: candidateId, + createAdapter: ({ sequence: candidateSequence, rep, seed }) => { + seeds.set(`${candidateId}:${candidateSequence.id}:${rep}`, seed) + return createScopedTestAdapter(`${candidateId}:${rep}`) + }, + })), + reps: 2, + runDir: '/runs/paired-seeds', + storage: inMemoryCampaignStorage(), + maxConcurrency: 4, + }) + + expect(seeds.get('a:paired-history:0')).toBe(seeds.get('b:paired-history:0')) + expect(seeds.get('a:paired-history:1')).toBe(seeds.get('b:paired-history:1')) + expect(seeds.get('a:paired-history:0')).not.toBe(seeds.get('a:paired-history:1')) + }) + + it('clears accepted writes and disposes the adapter after a failed step', async () => { + let clears = 0 + let closes = 0 + let disposals = 0 + const result = await runAgentMemoryExperiment({ + experimentId: 'failed-step-cleanup', + sequences: [ + { + id: 'failure', + family: 'first-party', + steps: [ + { + id: 'write-then-fail', + scope: { agentId: 'worker' }, + writes: [{ kind: 'fact', text: 'partial state' }], + probes: [{ id: 'state', query: 'partial state', referenceAnswer: 'partial state' }], + }, + ], + }, + ], + candidates: [ + { + id: 'scoped', + createAdapter() { + const base = createScopedTestAdapter('failure') + return { + ...base, + async clear(scope) { + clears += 1 + await base.clear?.(scope) + if (clears > 1) throw new Error('provider clear failed') + }, + async close() { + closes += 1 + }, + } + }, + async disposeAdapter() { + disposals += 1 + }, + }, + ], + runDir: '/runs/failed-step-cleanup', + storage: inMemoryCampaignStorage(), + cleanupBranches: true, + executeStepRef: 'failing-worker/v1', + async executeStep() { + throw new Error('worker failed') + }, + }) + + expect(result.rows[0]).toMatchObject({ cellsFailed: 1 }) + expect(result.campaign.cells[0]?.error).toContain( + 'memory branch cleanup failed after: worker failed', + ) + expect(clears).toBe(2) + expect(closes).toBe(1) + expect(disposals).toBe(1) + }) + + it('cleans an unjournaled provider side effect and ranks failed histories below complete ones', async () => { + let dirtyRows = 0 + let clears = 0 + const result = await runAgentMemoryExperiment({ + experimentId: 'provider-side-effect-cleanup', + sequences: [ + { + id: 'one', + family: 'first-party', + steps: [ + { + id: 'remember', + scope: { agentId: 'worker' }, + writes: [{ kind: 'fact', text: 'Launch is Friday.' }], + probes: [{ id: 'launch', query: 'launch', referenceAnswer: 'Launch is Friday.' }], + }, + ], + }, + ], + candidates: [ + { + id: 'side-effect-then-error', + createAdapter: () => ({ + id: 'side-effect-then-error', + branchIsolation: { mode: 'scoped' }, + async search() { + return [] + }, + async getContext(query) { + return { query, text: '', hits: [], sourceRecords: [] } + }, + async write() { + dirtyRows += 1 + throw new Error('provider disconnected after commit') + }, + async clear() { + clears += 1 + dirtyRows = 0 + }, + }), + }, + { + id: 'complete', + createAdapter: () => createScopedTestAdapter('complete'), + }, + ], + runDir: '/runs/provider-side-effect-cleanup', + storage: inMemoryCampaignStorage(), + cleanupBranches: true, + maxConcurrency: 2, + }) + + expect(dirtyRows).toBe(0) + expect(clears).toBe(2) + expect(result.rows[0]).toMatchObject({ + candidateId: 'complete', + cellsFailed: 0, + scoreMean: 1, + }) + expect(result.rows[1]).toMatchObject({ + candidateId: 'side-effect-then-error', + cellsFailed: 1, + scoreMean: 0, + passRate: 0, + }) + }) + + it('closes and disposes an adapter when branch validation fails', async () => { + let closes = 0 + let disposals = 0 + const result = await runAgentMemoryExperiment({ + experimentId: 'invalid-branch-cleanup', + sequences: [ + { + id: 'one', + family: 'first-party', + steps: [ + { + id: 'probe', + scope: { agentId: 'worker' }, + probes: [{ id: 'state', query: 'state', referenceAnswer: 'state' }], + }, + ], + }, + ], + candidates: [ + { + id: 'legacy', + createAdapter() { + const { branchIsolation: _branchIsolation, ...legacy } = + createScopedTestAdapter('legacy') + return { + ...legacy, + async close() { + closes += 1 + }, + } + }, + async disposeAdapter() { + disposals += 1 + }, + }, + ], + runDir: '/runs/invalid-branch-cleanup', + storage: inMemoryCampaignStorage(), + cleanupBranches: false, + }) + + expect(result.rows[0]).toMatchObject({ cellsFailed: 1 }) + expect(result.campaign.cells[0]?.error).toContain('adapter must declare branchIsolation') + expect(closes).toBe(1) + expect(disposals).toBe(1) + }) + + it('fails closed when cleanup is requested for an adapter without scoped clear', async () => { + let disposals = 0 + const result = await runAgentMemoryExperiment({ + experimentId: 'unsupported-cleanup', + sequences: [ + { + id: 'one', + family: 'first-party', + steps: [ + { + id: 'probe', + scope: { agentId: 'worker' }, + probes: [{ id: 'state', query: 'state', referenceAnswer: 'state' }], + }, + ], + }, + ], + candidates: [ + { + id: 'no-clear', + createAdapter() { + const { clear: _clear, ...adapter } = createScopedTestAdapter('no-clear') + return adapter + }, + async disposeAdapter() { + disposals += 1 + }, + }, + ], + runDir: '/runs/unsupported-cleanup', + storage: inMemoryCampaignStorage(), + cleanupBranches: true, + }) + + expect(result.rows[0]).toMatchObject({ cellsFailed: 1 }) + expect(result.campaign.cells[0]?.error).toContain('requires an adapter with scoped clear') + expect(disposals).toBe(1) + }) + + it('rejects dynamic writes whose scope cannot be cleaned after a crash', async () => { + let providerWrites = 0 + const result = await runAgentMemoryExperiment({ + experimentId: 'undeclared-dynamic-scope', + sequences: [ + { + id: 'one', + family: 'first-party', + steps: [ + { + id: 'agent-step', + scope: { agentId: 'declared' }, + probes: [{ id: 'state', query: 'state', referenceAnswer: 'state' }], + }, + ], + }, + ], + candidates: [ + { + id: 'scoped', + createAdapter: () => + createScopedTestAdapter('scoped', async () => { + providerWrites += 1 + }), + }, + ], + runDir: '/runs/undeclared-dynamic-scope', + storage: inMemoryCampaignStorage(), + cleanupBranches: true, + executeStepRef: 'dynamic-scope-test/v1', + executeStep: async ({ memory }) => { + await memory.write({ + kind: 'fact', + text: 'must not reach the provider', + scope: { agentId: 'undeclared' }, + }) + }, + }) + + expect(result.rows[0]).toMatchObject({ cellsFailed: 1 }) + expect(result.campaign.cells[0]?.error).toContain( + 'write scope was not declared in the experiment sequence cleanupScopes', + ) + expect(providerWrites).toBe(0) + }) + + it('requires an external-state disposer when scoped cleanup is disabled', async () => { + let creates = 0 + const result = await runAgentMemoryExperiment({ + experimentId: 'missing-disposer', + sequences: [ + { + id: 'one', + family: 'first-party', + steps: [ + { + id: 'probe', + scope: { agentId: 'worker' }, + probes: [{ id: 'state', query: 'state', referenceAnswer: 'state' }], + }, + ], + }, + ], + candidates: [ + { + id: 'persistent', + createAdapter() { + creates += 1 + return createScopedTestAdapter('persistent') + }, + }, + ], + runDir: '/runs/missing-disposer', + storage: inMemoryCampaignStorage(), + cleanupBranches: false, + }) + + expect(creates).toBe(0) + expect(result.rows[0]).toMatchObject({ cellsFailed: 1, scoreMean: 0 }) + expect(result.campaign.cells[0]?.error).toContain('requires disposeAdapter') + }) + + it('rejects probes that cannot distinguish a useful memory system', async () => { + await expect( + runAgentMemoryExperiment({ + experimentId: 'no-target', + sequences: [ + { + id: 'one', + family: 'first-party', + steps: [ + { + id: 'probe', + scope: { agentId: 'worker' }, + probes: [{ id: 'state', query: 'state' }], + }, + ], + }, + ], + candidates: [{ id: 'memory', createAdapter: () => createScopedTestAdapter('memory') }], + runDir: '/runs/no-target', + storage: inMemoryCampaignStorage(), + }), + ).rejects.toThrow('has no measurable target') + }) +}) + +describe('agent memory improvement', () => { + it('searches isolated configs and activates only a fresh holdout win', async () => { + type Config = { visibility: 'private' | 'team' | 'shared' } + const storage = inMemoryCampaignStorage() + const promoted: Config[] = [] + const activationIds: string[] = [] + const contenderIds = new Set() + const activeContenderCalls = new Map() + let maxConcurrentConfigs = 0 + let reportContendersActive: (() => void) | undefined + const contendersActive = new Promise((resolve) => { + reportContendersActive = resolve + }) + let releaseContenders: (() => void) | undefined + const continueContenders = new Promise((resolve) => { + releaseContenders = resolve + }) + let proposalCalls = 0 + const proposer: SurfaceProposer = { + kind: 'team-sharing-proposer', + async propose() { + proposalCalls += 1 + return [ + { + surface: JSON.stringify({ visibility: 'team' }), + label: 'share within the team', + rationale: "the second agent needs the first agent's accepted fact", + }, + { + surface: JSON.stringify({ visibility: 'shared' }), + label: 'share globally', + rationale: 'compare a broader sharing policy under the same histories', + }, + ] + }, + } + + const options: RunAgentMemoryImprovementOptions = { + experimentId: 'improve-team-memory', + trainSequences: [ + improvementSequence('train-a', 'train'), + improvementSequence('train-b', 'train'), + ], + holdoutSequences: [ + improvementSequence('holdout-a', 'holdout'), + improvementSequence('holdout-b', 'holdout'), + ], + seeds: [ + { + config: { visibility: 'private' }, + track: 'baseline', + proposer: 'seed', + }, + ], + proposer, + improvementRef: 'team-memory-policy/v1', + budget: { maxSteps: 1 }, + populationSize: 2, + candidateConcurrency: 2, + sequenceConcurrency: 4, + runDir: '/runs/improve-team-memory', + storage, + significance: { minProductiveRuns: 2, resamples: 200, seed: 7 }, + createCandidate: ({ config, candidateId }) => { + if (config.visibility !== 'private') contenderIds.add(candidateId) + return { + label: config.visibility, + policy: { read: [config.visibility], write: config.visibility }, + createAdapter: ({ branchId }) => createScopedTestAdapter(`${candidateId}:${branchId}`), + } + }, + executeStepRef: 'parallel-config-proof/v1', + executeStep: async ({ candidateId, step }) => { + if (step.id !== 'research' || !contenderIds.has(candidateId)) return + activeContenderCalls.set(candidateId, (activeContenderCalls.get(candidateId) ?? 0) + 1) + maxConcurrentConfigs = Math.max(maxConcurrentConfigs, activeContenderCalls.size) + if (activeContenderCalls.size === 2) reportContendersActive?.() + try { + await continueContenders + } finally { + const remaining = (activeContenderCalls.get(candidateId) ?? 1) - 1 + if (remaining === 0) activeContenderCalls.delete(candidateId) + else activeContenderCalls.set(candidateId, remaining) + } + }, + onPromote: async ({ activationId, config }) => { + activationIds.push(activationId) + promoted.push(config) + }, + } + const firstRun = runAgentMemoryImprovement(options) + await contendersActive + await expect(runAgentMemoryImprovement(options)).rejects.toThrow('active controller') + releaseContenders?.() + const result = await firstRun + + expect(result.decision).toMatchObject({ + status: 'promote', + reasons: [], + baselineScore: 0, + winnerScore: 1, + lift: 1, + }) + expect(result.decision.significance).toMatchObject({ n: 2, significant: true }) + expect(result.winnerConfig).toEqual({ visibility: 'team' }) + expect(result.holdout?.campaign.cells).toHaveLength(4) + expect(maxConcurrentConfigs).toBe(2) + expect(promoted).toEqual([{ visibility: 'team' }]) + expect(result.activation).toMatchObject({ status: 'activated' }) + expect(storage.read(result.resultJsonPath)).toContain('"status": "promote"') + + const resumed = await runAgentMemoryImprovement(options) + expect(proposalCalls).toBe(1) + expect(promoted).toEqual([{ visibility: 'team' }]) + expect(activationIds).toEqual([result.activation.id]) + expect(resumed.activation).toEqual({ + ...result.activation, + status: 'already-activated', + }) + const resumedWithoutCallback = await runAgentMemoryImprovement({ + ...options, + onPromote: undefined, + }) + expect(resumedWithoutCallback.activation.status).toBe('already-activated') + await expect( + runAgentMemoryImprovement({ ...options, budget: { maxSteps: 2 } }), + ).rejects.toThrow('does not match its persisted inputs or implementationRef') + await expect(runAgentMemoryImprovement({ ...options, minHoldoutScore: 0.99 })).rejects.toThrow( + 'does not match its persisted inputs or implementationRef', + ) + await expect( + runAgentMemoryImprovement({ ...options, improvementRef: 'team-memory-policy/v2' }), + ).rejects.toThrow('does not match its persisted inputs or implementationRef') + }) + + it('routes independent tracks to their named proposers with track context', async () => { + type Config = { visibility: 'private' | 'team' } + const trackContexts: Array<{ + id?: string + operation?: string + vision?: string + generation: number + costPhase?: string + hasCostLedger: boolean + }> = [] + let governorCostPhase: string | undefined + let governorHasCostLedger = false + const trackProposer: SurfaceProposer = { + kind: 'team-memory-researcher', + async propose(context) { + trackContexts.push({ + id: context.track?.id, + operation: context.track?.operation, + vision: context.track?.vision, + generation: context.generation, + costPhase: context.costPhase, + hasCostLedger: context.costLedger !== undefined, + }) + return [JSON.stringify({ visibility: 'team' })] + }, + } + + await runAgentMemoryImprovement({ + experimentId: 'named-track-proposers', + trainSequences: [improvementSequence('track-train', 'train')], + holdoutSequences: [improvementSequence('track-holdout', 'holdout')], + seeds: [ + { config: { visibility: 'private' }, track: 'baseline', proposer: 'baseline' }, + { + config: { visibility: 'private' }, + track: 'sharing-research', + proposer: 'team-memory-researcher', + vision: 'test whether team memory transfers accepted facts', + }, + ], + proposer: { + kind: 'unexpected-fallback', + async propose() { + throw new Error('named track should not use the fallback proposer') + }, + }, + proposers: { 'team-memory-researcher': trackProposer }, + governor: { + decide(context) { + governorCostPhase = context.costPhase + governorHasCostLedger = context.costLedger !== undefined + return { op: 'extend', track: 'sharing-research' } + }, + }, + improvementRef: 'named-track-proposers/v1', + budget: { maxSteps: 1 }, + populationSize: 1, + runDir: '/runs/named-track-proposers', + storage: inMemoryCampaignStorage(), + createCandidate: ({ config, candidateId }) => ({ + policy: { read: [config.visibility], write: config.visibility }, + createAdapter: ({ branchId }) => createScopedTestAdapter(`${candidateId}:${branchId}`), + }), + }) + + expect(trackContexts).toEqual([ + { + id: 'sharing-research', + operation: 'extend', + vision: 'test whether team memory transfers accepted facts', + generation: 1, + costPhase: 'memory.proposal.sharing-research', + hasCostLedger: true, + }, + ]) + expect(governorCostPhase).toBe('memory.governor') + expect(governorHasCostLedger).toBe(true) + }) + + it('holds a winner when a configured critical dimension is missing', async () => { + type Config = { visibility: 'private' | 'team' } + const result = await runAgentMemoryImprovement({ + experimentId: 'missing-critical-dimension', + trainSequences: [improvementSequence('critical-train', 'train')], + holdoutSequences: [ + improvementSequence('critical-holdout-a', 'holdout'), + improvementSequence('critical-holdout-b', 'holdout'), + improvementSequence('critical-holdout-c', 'holdout'), + ], + seeds: [{ config: { visibility: 'private' }, track: 'baseline', proposer: 'sharing' }], + proposer: { + kind: 'sharing', + async propose() { + return [JSON.stringify({ visibility: 'team' })] + }, + }, + improvementRef: 'missing-critical-dimension/v1', + budget: { maxSteps: 1 }, + populationSize: 1, + runDir: '/runs/missing-critical-dimension', + storage: inMemoryCampaignStorage(), + significance: { minProductiveRuns: 1, resamples: 100, seed: 9 }, + criticalDimensions: ['missing_safety'], + createCandidate: ({ config, candidateId }) => ({ + policy: { read: [config.visibility], write: config.visibility }, + createAdapter: ({ branchId }) => createScopedTestAdapter(`${candidateId}:${branchId}`), + }), + }) + + expect(result.decision.status).toBe('hold') + expect(result.decision.criticalDimensions).toEqual([ + expect.objectContaining({ dimension: 'missing_safety', n: 0, measured: false }), + ]) + expect(result.decision.reasons).toContain( + 'critical dimension missing_safety was measured on 0/3 paired holdout cells', + ) + }) + + it('stops before a proposer call would exceed the run-wide cost limit', async () => { + let proposerExecuted = false + + await expect( + runAgentMemoryImprovement({ + experimentId: 'proposer-cost-limit', + trainSequences: [improvementSequence('cost-train', 'train')], + holdoutSequences: [improvementSequence('cost-holdout', 'holdout')], + seeds: [ + { + config: { visibility: 'private' as const }, + track: 'baseline', + proposer: 'costed', + }, + ], + proposer: { + kind: 'costed', + async propose(context) { + if (!context.costLedger) throw new Error('missing run cost ledger') + const paid = await context.costLedger.runPaidCall({ + actor: 'memory-config-proposer', + channel: 'agent', + phase: context.costPhase, + model: 'fixture-model', + maximumCharge: { externallyEnforcedMaximumUsd: 0.06 }, + execute: async () => { + proposerExecuted = true + return JSON.stringify({ visibility: 'team' }) + }, + receipt: () => ({ + model: 'fixture-model', + inputTokens: 0, + outputTokens: 0, + usageUnknown: true, + actualCostUsd: 0.06, + }), + }) + if (!paid.succeeded) throw paid.error + return [paid.value] + }, + }, + improvementRef: 'proposer-cost-limit/v1', + budget: { maxSteps: 1 }, + maxTotalCostUsd: 0.05, + runDir: '/runs/proposer-cost-limit', + storage: inMemoryCampaignStorage(), + createCandidate: ({ config, candidateId }) => ({ + policy: { read: [config.visibility], write: config.visibility }, + createAdapter: ({ branchId }) => createScopedTestAdapter(`${candidateId}:${branchId}`), + }), + }), + ).rejects.toThrow('would exceed ceiling 0.05') + expect(proposerExecuted).toBe(false) + }) + + it('rejects train and holdout histories with the same id', async () => { + const sequence = improvementSequence('duplicate', 'train') + await expect( + runAgentMemoryImprovement({ + experimentId: 'overlap', + trainSequences: [sequence], + holdoutSequences: [{ ...sequence, split: 'holdout' }], + seeds: [{ config: {}, track: 'baseline', proposer: 'seed' }], + proposer: { + kind: 'unused', + async propose() { + return [] + }, + }, + improvementRef: 'overlap-test/v1', + budget: { maxSteps: 1 }, + runDir: '/runs/overlap', + storage: inMemoryCampaignStorage(), + createCandidate: () => ({ + createAdapter: () => createScopedTestAdapter('unused'), + }), + }), + ).rejects.toThrow('train/holdout overlap: duplicate') + }) + + it('rejects a holdout history copied under a different id', async () => { + const train = improvementSequence('train-original', 'train') + await expect( + runAgentMemoryImprovement({ + experimentId: 'renamed-overlap', + trainSequences: [train], + holdoutSequences: [{ ...train, id: 'renamed-holdout', split: 'holdout' }], + seeds: [{ config: {}, track: 'baseline', proposer: 'seed' }], + proposer: { + kind: 'unused', + async propose() { + return [] + }, + }, + improvementRef: 'renamed-overlap/v1', + budget: { maxSteps: 0 }, + runDir: '/runs/renamed-overlap', + storage: inMemoryCampaignStorage(), + createCandidate: () => ({ + createAdapter: () => createScopedTestAdapter('unused'), + }), + }), + ).rejects.toThrow('histories duplicate content') + }) +}) + +function improvementSequence(id: string, split: 'train' | 'holdout') { + return { + id, + family: 'first-party' as const, + split, + steps: [ + { + id: 'research', + scope: { agentId: 'researcher', teamId: 'team-1' }, + writes: [ + { + id: `${id}-event`, + kind: 'fact' as const, + text: `${id} launch date is Friday`, + metadata: { eventId: `${id}-event`, actorId: 'researcher' }, + }, + ], + }, + { + id: 'delivery', + scope: { agentId: 'builder', teamId: 'team-1' }, + probes: [ + { + id: 'launch-date', + query: `${id} launch date`, + requiredFacts: [{ id: 'current', anyOf: [`${id} launch date is Friday`] }], + expectedEventIds: [`${id}-event`], + expectedActorIds: ['researcher'], + }, + ], + }, + ], + } +} + +function hitText(hit: AgentMemoryHit): string { + return hit.text +} + +function createScopedTestAdapter( + id: string, + beforeWrite?: (scope: AgentMemoryScope, text: string) => Promise, +): AgentMemoryAdapter { + const rows: Array<{ scope: AgentMemoryScope; hit: AgentMemoryHit }> = [] + let sequence = 0 + return { + id, + branchIsolation: { mode: 'scoped' }, + async search(_query, options = {}) { + return rows.filter((row) => sameScope(row.scope, options.scope)).map((row) => row.hit) + }, + async getContext(query, options = {}) { + const hits = await this.search(query, options) + return { query, hits, text: hits.map(hitText).join('\n'), sourceRecords: [] } + }, + async write(input) { + const scope = input.scope ?? {} + await beforeWrite?.(scope, input.text) + sequence += 1 + const memoryId = input.id ?? `${id}-${sequence}` + const hit: AgentMemoryHit = { + id: memoryId, + uri: `memory://${id}/${memoryId}`, + kind: input.kind, + text: input.text, + metadata: input.metadata, + } + rows.push({ scope, hit }) + return { accepted: true, id: memoryId, uri: hit.uri, kind: input.kind } + }, + async clear(scope) { + for (let index = rows.length - 1; index >= 0; index -= 1) { + if (sameScope(rows[index]!.scope, scope)) rows.splice(index, 1) + } + }, + } +} + +function sameScope(left: AgentMemoryScope, right: AgentMemoryScope = {}): boolean { + for (const key of [ + 'tenantId', + 'userId', + 'agentId', + 'teamId', + 'runId', + 'sessionId', + 'namespace', + ] as const) { + if (right[key] !== undefined && left[key] !== right[key]) return false + } + for (const [key, value] of Object.entries(right.tags ?? {})) { + if (left.tags?.[key] !== value) return false + } + return true +} From 85639b729be2714720ef99e633baff4ebae1120a Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Fri, 17 Jul 2026 21:10:38 -0600 Subject: [PATCH 2/4] fix(memory): harden provider recovery and cost controls --- README.md | 121 ++- src/benchmarks/index.ts | 785 ++++++++++++++++-- src/memory/attempt-log.ts | 74 ++ src/memory/branch.ts | 214 +++-- src/memory/experiment.ts | 723 +++++++++++++++-- src/memory/graphiti.ts | 44 +- src/memory/improvement.ts | 202 ++--- src/memory/index.ts | 2 + src/memory/lifecycle.ts | 81 ++ src/memory/mem0.ts | 341 +++++++- src/memory/neo4j.ts | 6 +- src/memory/rank.ts | 62 ++ src/memory/run-control.ts | 93 +++ src/memory/types.ts | 9 +- tests/benchmarks.test.ts | 337 +++++++- tests/memory-systems.test.ts | 1483 ++++++++++++++++++++++++++++++++-- 16 files changed, 4099 insertions(+), 478 deletions(-) create mode 100644 src/memory/attempt-log.ts create mode 100644 src/memory/lifecycle.ts create mode 100644 src/memory/rank.ts create mode 100644 src/memory/run-control.ts diff --git a/README.md b/README.md index 3952d25..46c8347 100644 --- a/README.md +++ b/README.md @@ -151,7 +151,7 @@ The `/memory` subpath exports an optional memory adapter contract. Use it to bridge episodic or graph-native memory systems into the same source-grounded readiness/eval machinery without making `agent-knowledge` own the database. -## Benchmark Harness +## Benchmark Runner Use `runKnowledgeBenchmarkSuite()` when the product goal is to run a fixed RAG/KB benchmark pack and get one report across retrieval, answer quality, hallucination, and candidate-KB improvement cases. The module also exports `INDUSTRY_RAG_BENCHMARKS`, a compact manifest for BEIR, MTEB retrieval, MS MARCO, TREC DL, MIRACL, LoTTE, BRIGHT, CRAG, HotpotQA, KILT, RAGTruth, FaithBench, and first-party KB-improvement tasks. @@ -175,12 +175,14 @@ import { await runKnowledgeBenchmarkSuite({ cases: buildIndustryRagBenchmarkSmokeCases(), runDir: '.agent-knowledge/benchmark-runs/industry-smoke', + respondRef: 'industry-rag-smoke:v1', respond: respondToIndustryRagBenchmarkSmokeCase, }) await runKnowledgeBenchmarkSuite({ cases: buildIndustryMemoryBenchmarkSmokeCases(), runDir: '.agent-knowledge/benchmark-runs/memory-smoke', + respondRef: 'industry-memory-smoke:v1', respond: ({ case: testCase }) => { if (!isKnowledgeMemoryBenchmarkCase(testCase)) return { answer: '' } return respondToIndustryMemoryBenchmarkSmokeCase({ case: testCase }) @@ -191,9 +193,10 @@ const memoryRanking = await runMemoryAdapterBenchmark({ cases: buildFirstPartyMemoryLifecycleBenchmarkCases(), runDir: '.agent-knowledge/benchmark-runs/memory-ranking', candidates: [ - { id: 'no-memory', createAdapter: () => createNoopMemoryBenchmarkAdapter() }, + { id: 'no-memory', ref: 'no-memory:v1', createAdapter: () => createNoopMemoryBenchmarkAdapter() }, { id: 'in-memory', + ref: 'in-memory:v1', createAdapter: () => createInMemoryBenchmarkAdapter(), searchLimit: 1, }, @@ -214,6 +217,7 @@ const cases = buildRetrievalBenchmarkCasesFromQrels({ const result = await runKnowledgeBenchmarkSuite({ cases, runDir: '.agent-knowledge/benchmark-runs/beir-nfcorpus-smoke', + respondRef: 'product-retriever:v3', respond: async ({ case: testCase }) => { if (testCase.taskKind !== 'retrieval') return { hits: [] } const hits = await retrieveFromYourKb(testCase.query) @@ -224,10 +228,19 @@ const result = await runKnowledgeBenchmarkSuite({ console.log(result.report.score.mean) ``` -`runMemoryAdapterBenchmark()` is for simple adapters whose request scope fully isolates each case. -Use `runAgentMemoryExperiment()` below for external providers, retries, resume, parallel candidates, and branch cleanup. +`runMemoryAdapterBenchmark()` is for adapters that enforce every supplied scope and support exact scoped deletion. +Every cell records its fresh provider namespace before the first write and deletes it after measurement, so concurrent repetitions and retries cannot share memory. +On restart, unfinished scopes are deleted before any new benchmark cell runs. +The candidate factory receives `purpose: 'execute' | 'recovery'`; both paths must reconnect to the same provider and adapter identity. +The execute factory only constructs the adapter; normal provider work belongs in adapter methods so it runs inside per-case cost accounting. +Use `runAgentMemoryExperiment()` below when an agent profile needs ordered multi-step histories, branch snapshots, runtime callbacks, or automatic configuration improvement. +Set `cleanupTimeoutMs` to bound each provider cleanup and close operation; the default is 180 seconds. +Set `costUsdPerCase` for normal provider work and `recoveryCostUsdPerAttempt` when reconnecting and deleting an interrupted case can add provider charges. +One `costCeiling` covers every candidate and recovery call in the comparison, and `totalCostUsd` reports the complete comparison cost. +The default filesystem storage uses an OS lock; custom storage requires `controllerMode: 'process-local'` or a distributed `acquireRunLease` implementation. Billable responders must execute paid work through `context.cost.runPaidCall()`; `costUsd` on an artifact is display-only. +Change `respondRef` whenever the model, prompt, retrieval, tools, or runtime behavior changes; resumable runs reject a missing reference instead of reusing ambiguous cached rows. Use `buildRetrievalBenchmarkCasesFromQrels()` for qrels-backed retrieval datasets. The smoke pack proves that every declared benchmark family is wired through the runner; full BEIR, MTEB, MS MARCO, TREC DL, MIRACL, LoTTE, BRIGHT, CRAG, HotpotQA, KILT, RAGTruth, and FaithBench runs should pass real dataset rows through the same case shapes. Use `KnowledgeAnswerBenchmarkCase` for CRAG/HotpotQA/KILT-style answer checks and RAGTruth/FaithBench-style hallucination checks by encoding required claims, forbidden claims, and expected source IDs. @@ -439,9 +452,26 @@ import { createNeo4jAgentMemoryAdapter, } from '@tangle-network/agent-knowledge/memory' -const mem0 = createMem0MemoryAdapter({ +const mem0ApiKey = process.env.MEM0_API_KEY +if (!mem0ApiKey) throw new Error('MEM0_API_KEY is required') + +const mem0Hosted = createMem0MemoryAdapter({ client: mem0Client, - mode: 'hosted', // use 'oss' with mem0ai/oss Memory + mode: 'hosted', + consistency: 'visible', + async getEvent(eventId) { + const response = await fetch( + `https://api.mem0.ai/v1/event/${encodeURIComponent(eventId)}/`, + { headers: { Authorization: `Token ${mem0ApiKey}` } }, + ) + if (!response.ok) throw new Error(`Mem0 event request failed: ${response.status}`) + return response.json() + }, +}) + +const mem0Oss = createMem0MemoryAdapter({ + client: mem0OssClient, + mode: 'oss', }) const graphiti = createGraphitiMemoryAdapter({ @@ -459,8 +489,20 @@ The Graphiti defaults match the current official server tools: `add_memory`, `se Set `toolNames` explicitly when your server lists different names. The adapter never retries a write under another name because that could enqueue it twice. Visible writes expand the episode scan when Graphiti truncates a long group and fail explicitly at `episodeScanLimit` instead of reporting a false success. +Hosted Mem0 also defaults to visible writes and polls the official event endpoint until the add succeeds or fails. +The current JavaScript SDK does not expose that endpoint, so hosted mode accepts the small `getEvent` callback shown above. + +Each adapter uses the same `search`, `getContext`, and `write` method shape, but provider capabilities remain explicit. -Each adapter supports the same `search`, `getContext`, and `write` calls. +| Adapter | Searchable through this adapter | Writable through this adapter | +| --- | --- | --- | +| Mem0 hosted or OSS | messages, observations, facts, entities, preferences, reasoning traces | all six kinds | +| Graphiti MCP | extracted facts and entities | all six kinds as episodes | +| Neo4j hosted REST | messages, observations, entities | messages, observations, entities, conversation reasoning steps | +| Neo4j bridge | messages, observations, entities, preferences, similar reasoning traces | all six kinds | + +Neo4j bridge facts are graph writes. +The official `@neo4j-labs/agent-memory` 0.4 API has no corresponding fact-search method, so fact-only retrieval experiments should use another provider or a custom Neo4j query adapter. Mem0 and Graphiti apply tenant, user, agent, team, run, session, namespace, and tag scopes per request. Neo4j Agent Memory experiments require disposable external state per branch because the SDK cannot clear every memory kind atomically. Pass `branchId` only after the caller has provisioned that isolation. @@ -472,9 +514,16 @@ const neo4jInstances = new WeakMap() const neo4jCandidate = { id: 'neo4j', + ref: 'neo4j-agent-memory:0.4.0:rest', policy: { read: ['shared'], write: 'shared' }, - async createAdapter({ branchId }: { branchId: string }) { - const client = await provisionDedicatedNeo4jMemory({ branchId, transport: 'rest' }) + async createAdapter({ branchId, purpose }: { + branchId: string + purpose: 'execute' | 'recovery' + }) { + const client = purpose === 'recovery' + ? await openDedicatedNeo4jMemory({ branchId, transport: 'rest' }) + : await provisionDedicatedNeo4jMemory({ branchId, transport: 'rest' }) + if (!client) return null const adapter = createNeo4jAgentMemoryAdapter({ client, transport: 'rest', branchId }) neo4jInstances.set(adapter, branchId) return adapter @@ -487,19 +536,20 @@ const neo4jCandidate = { } ``` -Hosted REST supports messages, entities, and conversation reasoning steps. -The bridge transport also supports facts, preferences, and reusable reasoning traces. -The adapter requires `transport: 'rest' | 'bridge'` and rejects unsupported memory kinds before making a provider call. +The adapter requires `transport: 'rest' | 'bridge'` and rejects unsupported searches or writes before making a provider call. `contextMode: 'search'` is the default and preserves query-based behavior across providers. Set `contextMode: 'native'` with hosted REST to use Neo4j's whole-conversation context for message and observation memory. Use a branch when agents or candidate configurations must not share state accidentally: ```ts -import { createAgentMemoryBranch } from '@tangle-network/agent-knowledge/memory' +import { + createAgentMemoryBranch, + createMem0MemoryAdapter, +} from '@tangle-network/agent-knowledge/memory' const memory = createAgentMemoryBranch({ - adapter: mem0, + adapter: mem0Oss, branchId: 'candidate-17', policy: { read: ['private', 'team'], write: 'team' }, baseScope: { tenantId: 'acme', teamId: 'research' }, @@ -517,8 +567,14 @@ const context = await memory.getContext('When is launch?', { }) const snapshot = await memory.snapshot() +await memory.close?.() +const reopenedMem0Oss = createMem0MemoryAdapter({ + client: mem0OssClient, + mode: 'oss', + id: snapshot.adapterId, +}) const resumed = createAgentMemoryBranch({ - adapter: mem0, + adapter: reopenedMem0Oss, branchId: 'candidate-17', snapshot, }) @@ -573,9 +629,13 @@ const result = await runAgentMemoryImprovement({ `candidateConcurrency` controls how many configurations are measured at once. `sequenceConcurrency` controls how many independent histories run at once. +Every experiment candidate must set `ref` to a version or commit that changes whenever its provider configuration or adapter behavior changes. The first seed is always the deployed baseline. Each seed starts an independent track with its own objective and optional proposer, and a branch inherits its parent track's proposer unless the governor chooses another one. -Set each candidate's `externalCostUsdPerSequence` to a conservative memory-provider charge; model calls inside `executeStep` should use `context.cost.runPaidCall()` so `maxTotalCostUsd` counts both sources before starting more work. +Set each candidate's `externalCostUsdPerSequence` to a conservative memory-provider charge; it is counted once provider adapter creation is attempted because a rejected factory may already have contacted the provider. +Set `externalRecoveryCostUsdPerAttempt` when reconnecting and deleting an interrupted history adds a separate provider charge. +Recovery uses the same durable run-wide cost account as candidate execution, proposers, profile steps, and the governor, so a resumed run cannot exceed `maxTotalCostUsd` by treating cleanup as free work. +Model calls inside `executeStep` should use `context.cost.runPaidCall()` so `maxTotalCostUsd` counts both sources before starting more work. The same budget is supplied to named proposers as `context.costLedger` and to the governor as `context.costLedger`; the standard `agent-eval` proposers charge their model calls there. Steps inside one history remain ordered, and writes from one agent remain ordered while independent agents can write concurrently. Runtime callbacks may write only to scopes declared on a step, write, probe, or `sequence.cleanupScopes`; this lets an interrupted retry clear every actor and session it could have touched. @@ -583,23 +643,40 @@ Search never changes the live configuration. Only `onPromote` can activate the fresh-history winner. The callback receives a stable `activationId`; use it as the idempotency key when activation updates an external service. The default filesystem path permits one controller and many parallel workers, persists candidate history, and resumes without adding more than `maxSteps` candidate nodes. -Distributed controllers using custom storage provide `acquireRunLease`; independent workers still use `candidateConcurrency` and `sequenceConcurrency`. -Workers that execute one experiment from different local paths must share an explicit `experimentRunId`; otherwise the resolved `runDir` deliberately creates a different provider branch namespace. -Timed-out histories finish branch cleanup before `runAgentMemoryExperiment()` returns, and a cleanup failure fails the run. +Custom storage must set `controllerMode: 'process-local'` when all controllers share one process, or provide `acquireRunLease` for distributed controllers. +Independent workers still use `candidateConcurrency` and `sequenceConcurrency`. +Every measured cell receives a fresh provider branch ID for that execution attempt. +This prevents a late asynchronous write from a crashed worker from entering a retried cell. +`experimentRunId` keeps the logical experiment identity stable across workers without reusing physical branch state. +Timed-out histories enter bounded branch cleanup before `runAgentMemoryExperiment()` returns. +The run waits for cleanup to succeed or for `cleanupTimeoutMs` to expire, and either cleanup failure leaves the attempt available for recovery. +The runner appends a `started` event before adapter creation and a `cleaned` event only after provider cleanup and disposal succeed. +On resume it recovers every unfinished attempt before starting new cells. +Independent unfinished attempts recover up to `maxConcurrency` at once, and no new cell starts until every cleanup succeeds. +`createAdapter` receives `purpose: 'recovery'` during that pass so a dedicated-instance candidate can reconnect to the existing branch instead of provisioning another one. +It may return `null` during recovery when the recorded attempt never created external state. +Hosted Mem0 and Graphiti recovery waits for their configured ingestion timeout before deletion so an accepted asynchronous write can become visible first. +The default filesystem storage survives process restarts and uses an OS lock. +Custom durable storage must implement atomic `append`; process-local custom storage opts in with `controllerMode: 'process-local'`, and distributed controllers provide `acquireRunLease`. The persisted run identity includes `budget.maxSteps`; start a new `runDir` to enlarge a completed search instead of mutating its history. Promotion is blocked when a configured critical dimension is missing or incomplete on either fresh-history arm. -Improvement runs clear each scoped branch before and after a measured history by default, including recovery after an interrupted cell. +Improvement runs clear each scoped branch before and after a measured history by default, including timeout and thrown-error cleanup. Set `cleanupBranches: false` only when the candidate creates and disposes a physically isolated provider instance per cell. The current Neo4j SDK has no complete all-memory clear operation, so a Neo4j improvement run must use disposable per-cell service instances with `cleanupBranches: false` and `disposeAdapter`; a persistent shared workspace is rejected because failed retries could read partial state. +If recovery cannot confirm deletion, the resumed run stops before launching new provider work and leaves the attempt active for another retry. The package stays independent of `agent-runtime`. The `executeStep` callback is where a runtime profile, coding agent, research agent, or deterministic worker uses the branch. This keeps provider and measurement code reusable while the profile owns what agents do and what they choose to remember. The Neo4j adapter accepts the real `@neo4j-labs/agent-memory` client and rejects unsafe branch reuse unless `branchId` identifies caller-provisioned isolated state. -The Mem0 adapter typechecks against `mem0ai` hosted and open-source clients. +The Mem0 adapter typechecks against `mem0ai` hosted and open-source clients and waits for the hosted event result by default. The Graphiti adapter calls the official MCP tools and waits for queued episodes to become visible before evaluating them by default. -`consistency: 'queued'` is available for direct ingestion, but branch experiments reject it because a queued write can appear after a crashed process has already cleaned the branch. +Hosted Mem0 and Graphiti visible writes are safe only in `lifetime: 'attempt'` branches because their writes may outlive a worker process. +`runAgentMemoryExperiment()` creates those fresh attempt branches automatically. +Attempt snapshots are audit and fork inputs, not restart points; use `forkAgentMemoryBranchSnapshot()` to replay one into a fresh branch. +Mem0 OSS and caller-provisioned disposable instances can use resumable branches. +For hosted Mem0 and Graphiti, `consistency: 'queued'` is available only for direct ingestion and is rejected by branch execution. ## Research Loop diff --git a/src/benchmarks/index.ts b/src/benchmarks/index.ts index d324e9c..0fd3879 100644 --- a/src/benchmarks/index.ts +++ b/src/benchmarks/index.ts @@ -1,14 +1,34 @@ +import { randomUUID } from 'node:crypto' import { join } from 'node:path' +import { canonicalJson } from '@tangle-network/agent-eval' import { type CampaignResult, type CampaignStorage, + type CostLedgerHandle, + createRunCostLedger, type DispatchContext, fsCampaignStorage, type JudgeConfig, type RunCampaignOptions, + resolveRunDir, runCampaign, type Scenario, } from '@tangle-network/agent-eval/campaign' +import { stableId } from '../ids' +import { appendAttemptJournalEvent, readActiveAttemptJournal } from '../memory/attempt-log' +import { + createMemoryExecutionPool, + memoryRecoveryDelayMs, + resolveMemoryCleanupTimeoutMs, + runBoundedMemoryLifecycle, + sleepForMemoryRecovery, +} from '../memory/lifecycle' +import { + type AgentMemoryAcquireRunLease, + type AgentMemoryControllerMode, + acquireAgentMemoryRunLease, + type OwnedAgentMemoryRunLease, +} from '../memory/run-control' import { memoryHitToSourceRecord, memoryWriteResultToSourceRecord } from '../memory/source-record' import type { AgentMemoryAdapter, @@ -24,6 +44,9 @@ import { scoreRetrievalArtifact, } from '../retrieval-eval' +const KNOWLEDGE_BENCHMARK_IMPLEMENTATION_REF = 'agent-knowledge:benchmark-suite:v2' +const MEMORY_ADAPTER_BENCHMARK_IMPLEMENTATION_REF = 'agent-knowledge:memory-adapter-benchmark:v2' + export type KnowledgeBenchmarkTaskKind = | 'retrieval' | 'rag-answer' @@ -170,6 +193,8 @@ export interface KnowledgeBenchmarkEvaluation { score: number passed: boolean dimensions: Record + /** Dimensions for which this case declared an actual target. */ + applicableDimensions?: readonly string[] notes: string raw: Record } @@ -191,6 +216,8 @@ export type KnowledgeBenchmarkResponder export interface RunKnowledgeBenchmarkSuiteOptions { cases: readonly KnowledgeBenchmarkCase[] respond: KnowledgeBenchmarkResponder + /** Versioned identity for the model, prompt, retrieval, and runtime behavior. */ + respondRef?: string runDir: string splits?: readonly KnowledgeBenchmarkSplit[] repo?: string @@ -198,6 +225,9 @@ export interface RunKnowledgeBenchmarkSuiteOptions AgentMemoryAdapter | Promise + /** Construction must not start billable execute work; adapter methods run inside per-case accounting. */ + createAdapter: (input: { + purpose: 'execute' | 'recovery' + }) => AgentMemoryAdapter | Promise searchLimit?: number costUsdPerCase?: number + /** Conservative extra provider charge for recovering one interrupted case. */ + recoveryCostUsdPerAttempt?: number scope?: AgentMemoryScope } @@ -261,10 +298,18 @@ export interface RunMemoryAdapterBenchmarkOptions { reps?: number resumable?: boolean costCeiling?: number + /** Shared with nested benchmark suites so the dollar limit applies to the whole comparison. */ + costLedger?: CostLedgerHandle + costPhase?: string maxConcurrency?: number dispatchTimeoutMs?: number + cleanupTimeoutMs?: number expectUsage?: 'assert' | 'warn' | 'off' now?: () => Date + /** Required with custom storage when all controllers are confined to one process. */ + controllerMode?: AgentMemoryControllerMode + /** Required for distributed controllers that share custom storage. */ + acquireRunLease?: AgentMemoryAcquireRunLease } export interface MemoryAdapterBenchmarkRankingRow { @@ -285,8 +330,26 @@ export interface MemoryAdapterBenchmarkRankingRow { export interface RunMemoryAdapterBenchmarkResult { rows: readonly MemoryAdapterBenchmarkRankingRow[] + totalCostUsd: number rankingJsonPath: string rankingMarkdownPath: string + attemptLogPath: string +} + +class MemoryAdapterBenchmarkCleanupError extends AggregateError {} + +interface MemoryAdapterBenchmarkAttemptEvent { + schema: 1 + status: 'started' | 'cleaned' + attemptId: string + candidateId: string + candidateRef: string + adapterId: string + caseId: string + cellId: string + scope: AgentMemoryScope + recordedAt: string + recovery: boolean } export interface KnowledgeRetrievalBenchmarkQuery { @@ -837,14 +900,20 @@ export function buildFirstPartyMemoryLifecycleBenchmarkCases(): KnowledgeMemoryB ] } -export function createMemoryAdapterBenchmarkResponder(options: { +function createMemoryAdapterBenchmarkResponder(options: { adapter: AgentMemoryAdapter candidateId: string + candidateRef: string + storage: CampaignStorage + attemptLogPath: string + lease: OwnedAgentMemoryRunLease + cleanupTimeoutMs: number searchLimit?: number scope?: AgentMemoryScope costUsdPerCase?: number now?: () => Date }): KnowledgeBenchmarkResponder { + assertScopedMemoryBenchmarkAdapter(options.adapter) return async ({ case: testCase, context: dispatchContext }) => { if (!isKnowledgeMemoryBenchmarkCase(testCase)) { return { answer: '', metadata: { candidateId: options.candidateId, skipped: true } } @@ -855,50 +924,143 @@ export function createMemoryAdapterBenchmarkResponder(options: { } const execute = async (): Promise => { + dispatchContext.signal.throwIfAborted() + await options.lease.assertOwned() const startedAt = Date.now() - const scope = benchmarkMemoryScope(options.candidateId, testCase, options.scope) - for (const event of testCase.events) { - await options.adapter.write({ - id: event.id, - kind: 'message', - text: event.text, - role: event.actorId === 'user' ? 'user' : 'assistant', - title: `${testCase.id}:${event.id}`, + const attemptId = randomUUID() + const scope = benchmarkMemoryScope( + options.candidateId, + testCase, + dispatchContext.cellId, + attemptId, + options.scope, + ) + const attempt: MemoryAdapterBenchmarkAttemptEvent = { + schema: 1, + status: 'started', + attemptId, + candidateId: options.candidateId, + candidateRef: options.candidateRef, + adapterId: options.adapter.id, + caseId: testCase.id, + cellId: dispatchContext.cellId, + scope, + recordedAt: (options.now ?? (() => new Date()))().toISOString(), + recovery: false, + } + appendMemoryBenchmarkAttemptEvent(options.storage, options.attemptLogPath, attempt) + let artifact: KnowledgeBenchmarkArtifact | undefined + let primaryError: unknown + try { + for (const event of testCase.events) { + dispatchContext.signal.throwIfAborted() + await options.lease.assertOwned() + await options.adapter.write({ + id: event.id, + kind: 'message', + text: event.text, + role: event.actorId === 'user' ? 'user' : 'assistant', + title: `${testCase.id}:${event.id}`, + scope, + metadata: compactObject({ + benchmarkCaseId: testCase.id, + benchmarkCellId: dispatchContext.cellId, + benchmarkAttemptId: attemptId, + eventId: event.id, + actorId: event.actorId, + sessionId: event.sessionId, + timestamp: event.timestamp, + ...event.metadata, + }) as Record, + }) + dispatchContext.signal.throwIfAborted() + await options.lease.assertOwned() + } + await options.adapter.flush?.() + dispatchContext.signal.throwIfAborted() + await options.lease.assertOwned() + + const adapterContext = await options.adapter.getContext(testCase.prompt, { scope, - metadata: compactObject({ + limit: options.searchLimit ?? 1, + metadata: { benchmarkCaseId: testCase.id, - eventId: event.id, - actorId: event.actorId, - sessionId: event.sessionId, - timestamp: event.timestamp, - ...event.metadata, - }) as Record, + benchmarkCellId: dispatchContext.cellId, + benchmarkAttemptId: attemptId, + candidateId: options.candidateId, + }, }) + dispatchContext.signal.throwIfAborted() + await options.lease.assertOwned() + const hits = adapterContext.hits + artifact = { + answer: adapterContext.text, + rememberedFacts: hits.map((hit) => hit.text), + citedEventIds: unique(hits.map(memoryEventId).filter((id): id is string => Boolean(id))), + usedMemoryIds: hits.map((hit) => hit.id), + actorIds: unique(hits.map(memoryActorId).filter((id): id is string => Boolean(id))), + costUsd, + durationMs: Math.max(0, Date.now() - startedAt), + metadata: { + candidateId: options.candidateId, + adapterId: options.adapter.id, + hitCount: hits.length, + }, + } + } catch (error) { + primaryError = error } - const adapterContext = await options.adapter.getContext(testCase.prompt, { - scope, - limit: options.searchLimit ?? 1, - metadata: { - benchmarkCaseId: testCase.id, - candidateId: options.candidateId, - }, - }) - const hits = adapterContext.hits - return { - answer: adapterContext.text, - rememberedFacts: hits.map((hit) => hit.text), - citedEventIds: unique(hits.map(memoryEventId).filter((id): id is string => Boolean(id))), - usedMemoryIds: hits.map((hit) => hit.id), - actorIds: unique(hits.map(memoryActorId).filter((id): id is string => Boolean(id))), - costUsd, - durationMs: Math.max(0, Date.now() - startedAt), - metadata: { - candidateId: options.candidateId, - adapterId: options.adapter.id, - hitCount: hits.length, - }, + const cleanupErrors: unknown[] = [] + let cleanupOwned = true + try { + await options.lease.assertOwned() + } catch (error) { + cleanupOwned = false + cleanupErrors.push(error) + } + if (cleanupOwned) { + try { + await runBoundedMemoryLifecycle({ + operation: `${options.candidateId}: benchmark attempt flush`, + timeoutMs: options.cleanupTimeoutMs, + run: () => options.adapter.flush?.(), + }) + } catch (error) { + cleanupErrors.push(error) + } + try { + await runBoundedMemoryLifecycle({ + operation: `${options.candidateId}: benchmark attempt cleanup`, + timeoutMs: options.cleanupTimeoutMs, + run: () => options.adapter.clear!(scope), + }) + } catch (error) { + cleanupErrors.push(error) + } + } + if (cleanupOwned && cleanupErrors.length === 0) { + try { + appendMemoryBenchmarkAttemptEvent(options.storage, options.attemptLogPath, { + ...attempt, + status: 'cleaned', + recordedAt: (options.now ?? (() => new Date()))().toISOString(), + }) + } catch (error) { + cleanupErrors.push(error) + } + } + if (cleanupErrors.length > 0) { + const errors = [...(primaryError ? [primaryError] : []), ...cleanupErrors] + throw new MemoryAdapterBenchmarkCleanupError( + errors, + `${options.candidateId}: memory benchmark attempt cleanup failed`, + ) } + if (primaryError) throw primaryError + if (!artifact) + throw new Error(`${options.candidateId}: memory benchmark produced no artifact`) + return artifact } if (costUsd === 0) return execute() @@ -932,40 +1094,169 @@ export async function runMemoryAdapterBenchmark( 'memory adapter candidate id', ) for (const candidate of options.candidates) { + assertNonEmptyBenchmarkString(candidate.ref, `memory adapter candidate ${candidate.id} ref`) if (!/^[A-Za-z0-9][A-Za-z0-9._-]*$/.test(candidate.id)) { throw new Error( `memory adapter candidate id '${candidate.id}' must be a safe directory segment`, ) } + if ( + candidate.recoveryCostUsdPerAttempt !== undefined && + (!Number.isFinite(candidate.recoveryCostUsdPerAttempt) || + candidate.recoveryCostUsdPerAttempt < 0) + ) { + throw new Error( + `${candidate.id}: recoveryCostUsdPerAttempt must be a non-negative finite number`, + ) + } } const storage = options.storage ?? fsCampaignStorage() + if (!storage.append) { + throw new Error('memory adapter benchmark requires CampaignStorage.append') + } + const runDir = resolveRunDir(options.runDir, options.repo) + const cleanupTimeoutMs = resolveMemoryCleanupTimeoutMs( + options.cleanupTimeoutMs, + 'memory adapter benchmark', + ) + storage.ensureDir(runDir) + const lease = await acquireAgentMemoryRunLease({ + experimentId: `memory-adapter-benchmark:${runDir}`, + runDir, + storage, + customStorage: options.storage !== undefined, + lockFileName: 'memory-adapter-benchmark.lock', + label: 'memory adapter benchmark', + controllerMode: options.controllerMode, + acquireRunLease: options.acquireRunLease, + }) + let result: RunMemoryAdapterBenchmarkResult | undefined + let primaryError: unknown + try { + result = await runOwnedMemoryAdapterBenchmark(options, storage, runDir, lease, cleanupTimeoutMs) + } catch (error) { + primaryError = error + } + let releaseError: unknown + try { + await lease.release() + } catch (error) { + releaseError = error + } + if (primaryError && releaseError) { + throw new AggregateError( + [primaryError, releaseError], + 'memory adapter benchmark failed and its controller lease could not be released', + ) + } + if (primaryError) throw primaryError + if (releaseError) throw releaseError + if (!result) throw new Error('memory adapter benchmark produced no result') + return result +} + +async function runOwnedMemoryAdapterBenchmark( + options: RunMemoryAdapterBenchmarkOptions, + storage: CampaignStorage, + runDir: string, + lease: OwnedAgentMemoryRunLease, + cleanupTimeoutMs: number, +): Promise { + const maxConcurrency = options.maxConcurrency ?? 2 + if (!Number.isSafeInteger(maxConcurrency) || maxConcurrency <= 0) { + throw new Error('memory adapter benchmark maxConcurrency must be a positive safe integer') + } + const costLedger = + options.costLedger ?? + createRunCostLedger({ + storage, + runDir, + costCeilingUsd: options.costCeiling, + }) + if (options.costCeiling !== undefined && costLedger.costCeilingUsd !== options.costCeiling) { + throw new Error( + 'memory adapter benchmark costCeiling must match the shared cost ledger ceiling', + ) + } + const attemptLogPath = join(runDir, 'memory-adapter-attempts.jsonl') + await recoverMemoryAdapterBenchmarkAttempts({ + candidates: options.candidates, + storage, + attemptLogPath, + lease, + cleanupTimeoutMs, + maxConcurrency, + now: options.now, + runDir, + costLedger, + costPhase: options.costPhase ?? 'memory.adapter-benchmark', + }) + await lease.assertOwned() const rows: MemoryAdapterBenchmarkRankingRow[] = [] for (const candidate of options.candidates) { - const adapter = await candidate.createAdapter() + await lease.assertOwned() + const adapter = await candidate.createAdapter({ purpose: 'execute' }) + await lease.assertOwned() + const dispatchedExecutions: Promise[] = [] + let run: RunKnowledgeBenchmarkSuiteResult | undefined let primaryError: unknown try { - const run = await runKnowledgeBenchmarkSuite({ + const respond = createMemoryAdapterBenchmarkResponder({ + adapter, + candidateId: candidate.id, + candidateRef: candidate.ref, + storage, + attemptLogPath, + lease, + cleanupTimeoutMs, + searchLimit: candidate.searchLimit, + scope: candidate.scope, + costUsdPerCase: candidate.costUsdPerCase, + now: options.now, + }) + run = await runKnowledgeBenchmarkSuite({ cases: options.cases, - respond: createMemoryAdapterBenchmarkResponder({ - adapter, - candidateId: candidate.id, - searchLimit: candidate.searchLimit, - scope: candidate.scope, - costUsdPerCase: candidate.costUsdPerCase, - now: options.now, - }), - runDir: join(options.runDir, candidate.id), + respond(input) { + const operation = Promise.resolve().then(() => respond(input)) + dispatchedExecutions.push(operation) + return operation + }, + respondRef: stableId( + 'memory_adapter_benchmark', + canonicalJson({ + implementationRef: MEMORY_ADAPTER_BENCHMARK_IMPLEMENTATION_REF, + candidateRef: candidate.ref, + adapterId: adapter.id, + searchLimit: candidate.searchLimit ?? null, + costUsdPerCase: candidate.costUsdPerCase ?? 0, + recoveryCostUsdPerAttempt: candidate.recoveryCostUsdPerAttempt ?? 0, + scope: candidate.scope ?? null, + }), + ), + runDir: join(runDir, candidate.id), storage, - repo: options.repo, seed: options.seed, reps: options.reps, resumable: options.resumable, costCeiling: options.costCeiling, + costLedger, + costPhase: `${options.costPhase ?? 'memory.adapter-benchmark'}.${candidate.id}`, maxConcurrency: options.maxConcurrency, dispatchTimeoutMs: options.dispatchTimeoutMs, expectUsage: options.expectUsage ?? 'off', now: options.now, }) + } catch (error) { + primaryError = error + } + + const settledExecutions = await Promise.allSettled(dispatchedExecutions) + const dispatchCleanupErrors = settledExecutions.flatMap((settled) => + settled.status === 'rejected' && settled.reason instanceof MemoryAdapterBenchmarkCleanupError + ? [settled.reason] + : [], + ) + if (run) { rows.push({ rank: 0, candidateId: candidate.id, @@ -981,39 +1272,342 @@ export async function runMemoryAdapterBenchmark( reportMarkdownPath: run.reportMarkdownPath, report: run.report, }) - } catch (error) { - primaryError = error } const cleanupErrors: unknown[] = [] + let cleanupOwned = true try { - await adapter.flush?.() + await lease.assertOwned() } catch (error) { + cleanupOwned = false cleanupErrors.push(error) } + if (cleanupOwned) { + try { + await runBoundedMemoryLifecycle({ + operation: `${candidate.id}: benchmark adapter flush`, + timeoutMs: cleanupTimeoutMs, + run: () => adapter.flush?.(), + }) + } catch (error) { + cleanupErrors.push(error) + } + } try { - await adapter.close?.() + await runBoundedMemoryLifecycle({ + operation: `${candidate.id}: benchmark adapter close`, + timeoutMs: cleanupTimeoutMs, + run: () => adapter.close?.(), + }) } catch (error) { cleanupErrors.push(error) } - if (primaryError || cleanupErrors.length > 0) { - const errors = [...(primaryError ? [primaryError] : []), ...cleanupErrors] + if (primaryError || dispatchCleanupErrors.length > 0 || cleanupErrors.length > 0) { + const errors = [ + ...(primaryError ? [primaryError] : []), + ...dispatchCleanupErrors, + ...cleanupErrors, + ] if (errors.length === 1) throw errors[0] throw new AggregateError(errors, `${candidate.id}: memory adapter benchmark cleanup failed`) } + await lease.assertOwned() } const ranked = rows - .sort((a, b) => b.scoreMean - a.scoreMean || a.totalCostUsd - b.totalCostUsd) + .sort( + (a, b) => + Number(a.cellsFailed > 0) - Number(b.cellsFailed > 0) || + b.scoreMean - a.scoreMean || + b.passRate - a.passRate || + a.totalCostUsd - b.totalCostUsd || + a.candidateId.localeCompare(b.candidateId), + ) .map((row, index) => ({ ...row, rank: index + 1 })) - const rankingJsonPath = join(options.runDir, 'memory-adapter-ranking.json') - const rankingMarkdownPath = join(options.runDir, 'memory-adapter-ranking.md') - storage.write(rankingJsonPath, `${JSON.stringify({ rows: ranked }, null, 2)}\n`) - storage.write(rankingMarkdownPath, renderMemoryAdapterRankingMarkdown(ranked)) + const rankingJsonPath = join(runDir, 'memory-adapter-ranking.json') + const rankingMarkdownPath = join(runDir, 'memory-adapter-ranking.md') + const totalCostUsd = ranked.reduce((sum, row) => sum + row.totalCostUsd, 0) + storage.write(rankingJsonPath, `${JSON.stringify({ totalCostUsd, rows: ranked }, null, 2)}\n`) + storage.write(rankingMarkdownPath, renderMemoryAdapterRankingMarkdown(ranked, totalCostUsd)) return { rows: ranked, + totalCostUsd, rankingJsonPath, rankingMarkdownPath, + attemptLogPath, + } +} + +async function recoverMemoryAdapterBenchmarkAttempts(input: { + candidates: readonly MemoryAdapterBenchmarkCandidate[] + storage: CampaignStorage + attemptLogPath: string + lease: OwnedAgentMemoryRunLease + cleanupTimeoutMs: number + maxConcurrency: number + now?: () => Date + runDir: string + costLedger: CostLedgerHandle + costPhase: string +}): Promise { + const attempts = readActiveMemoryBenchmarkAttempts(input.storage, input.attemptLogPath) + const candidateById = new Map(input.candidates.map((candidate) => [candidate.id, candidate])) + const grouped = new Map() + for (const attempt of attempts) { + const group = grouped.get(attempt.candidateId) ?? [] + group.push(attempt) + grouped.set(attempt.candidateId, group) + } + const pool = createMemoryExecutionPool(input.maxConcurrency) + const settled = await Promise.allSettled( + [...grouped] + .sort(([left], [right]) => left.localeCompare(right)) + .map(([candidateId, candidateAttempts]) => + pool.run(async () => { + await input.lease.assertOwned() + const candidate = candidateById.get(candidateId) + if (!candidate) { + throw new Error( + `cannot recover memory benchmark attempts: candidate '${candidateId}' is missing`, + ) + } + for (const attempt of candidateAttempts) { + if (attempt.candidateRef !== candidate.ref) { + throw new Error( + `cannot recover memory benchmark attempt '${attempt.attemptId}': candidate ref changed from '${attempt.candidateRef}' to '${candidate.ref}'`, + ) + } + } + + const recover = async (): Promise => { + let adapter: AgentMemoryAdapter | undefined + let primaryError: unknown + const cleared: MemoryAdapterBenchmarkAttemptEvent[] = [] + try { + adapter = await runBoundedMemoryLifecycle({ + operation: `${candidate.id}: benchmark recovery adapter creation`, + timeoutMs: input.cleanupTimeoutMs, + run: () => candidate.createAdapter({ purpose: 'recovery' }), + }) + assertScopedMemoryBenchmarkAdapter(adapter) + for (const attempt of candidateAttempts) { + if (adapter.id !== attempt.adapterId) { + throw new Error( + `cannot recover memory benchmark attempt '${attempt.attemptId}': adapter changed from '${attempt.adapterId}' to '${adapter.id}'`, + ) + } + } + await sleepForMemoryRecovery(memoryRecoveryDelayMs(adapter), () => + input.lease.assertOwned(), + ) + for (const attempt of candidateAttempts.sort((left, right) => + left.attemptId.localeCompare(right.attemptId), + )) { + await input.lease.assertOwned() + try { + await runBoundedMemoryLifecycle({ + operation: `${candidate.id}: abandoned benchmark attempt cleanup`, + timeoutMs: input.cleanupTimeoutMs, + run: () => adapter!.clear!(attempt.scope), + }) + await input.lease.assertOwned() + cleared.push(attempt) + } catch (error) { + primaryError = primaryError + ? new AggregateError( + [primaryError, error], + `${candidate.id}: multiple benchmark attempts failed recovery`, + ) + : error + } + } + } catch (error) { + primaryError = primaryError + ? new AggregateError( + [primaryError, error], + `${candidate.id}: benchmark recovery failed in multiple operations`, + ) + : error + } + + let closeError: unknown + if (adapter) { + try { + await runBoundedMemoryLifecycle({ + operation: `${candidate.id}: benchmark recovery adapter close`, + timeoutMs: input.cleanupTimeoutMs, + run: () => adapter!.close?.(), + }) + } catch (error) { + closeError = error + } + } + if (!closeError) { + for (const attempt of cleared) { + await input.lease.assertOwned() + appendMemoryBenchmarkAttemptEvent(input.storage, input.attemptLogPath, { + ...attempt, + status: 'cleaned', + recovery: true, + recordedAt: (input.now ?? (() => new Date()))().toISOString(), + }) + } + } + if (primaryError && closeError) { + throw new AggregateError( + [primaryError, closeError], + `${candidate.id}: benchmark attempt recovery and adapter close failed`, + ) + } + if (primaryError) throw primaryError + if (closeError) throw closeError + } + + const recoveryCostUsd = candidate.recoveryCostUsdPerAttempt ?? 0 + if (recoveryCostUsd === 0) { + await recover() + return + } + const maximumCostUsd = recoveryCostUsd * candidateAttempts.length + const receipt = { + model: candidate.id, + inputTokens: 0, + outputTokens: 0, + usageUnknown: true, + actualCostUsd: maximumCostUsd, + } as const + const paid = await input.costLedger.runPaidCall({ + channel: 'driver', + phase: `${input.costPhase}.${candidate.id}.recovery`, + actor: `agent-knowledge:memory-adapter-recovery:${candidate.id}`, + model: candidate.id, + tags: { + runDir: join(input.runDir, candidate.id), + candidateId: candidate.id, + }, + maximumCharge: { externallyEnforcedMaximumUsd: maximumCostUsd }, + execute: recover, + receipt: () => receipt, + receiptFromError: () => receipt, + }) + if (!paid.succeeded) throw paid.error + }), + ), + ) + const failures = settled.flatMap((result) => + result.status === 'rejected' ? [result.reason] : [], + ) + if (failures.length === 1) throw failures[0] + if (failures.length > 1) { + throw new AggregateError(failures, 'multiple memory benchmark candidates failed recovery') + } +} + +function appendMemoryBenchmarkAttemptEvent( + storage: CampaignStorage, + path: string, + event: MemoryAdapterBenchmarkAttemptEvent, +): void { + appendAttemptJournalEvent({ + storage, + path, + event, + label: 'memory benchmark attempt log', + }) +} + +function readActiveMemoryBenchmarkAttempts( + storage: CampaignStorage, + path: string, +): MemoryAdapterBenchmarkAttemptEvent[] { + return readActiveAttemptJournal({ + storage, + path, + label: 'memory benchmark attempt log', + parse: parseMemoryBenchmarkAttemptEvent, + id: (event) => event.attemptId, + sameAttempt: sameMemoryBenchmarkAttempt, + }) +} + +function parseMemoryBenchmarkAttemptEvent( + value: unknown, + path: string, + line: number, +): MemoryAdapterBenchmarkAttemptEvent { + const event = value as Partial | null + const valid = + typeof event === 'object' && + event !== null && + event.schema === 1 && + (event.status === 'started' || event.status === 'cleaned') && + isNonEmptyString(event.attemptId) && + isNonEmptyString(event.candidateId) && + isNonEmptyString(event.candidateRef) && + isNonEmptyString(event.adapterId) && + isNonEmptyString(event.caseId) && + isNonEmptyString(event.cellId) && + isAgentMemoryScope(event.scope) && + typeof event.recordedAt === 'string' && + !Number.isNaN(Date.parse(event.recordedAt)) && + typeof event.recovery === 'boolean' + if (!valid) { + throw new Error(`invalid memory benchmark attempt event in '${path}' line ${line}`) } + return value as MemoryAdapterBenchmarkAttemptEvent +} + +function sameMemoryBenchmarkAttempt( + left: MemoryAdapterBenchmarkAttemptEvent, + right: MemoryAdapterBenchmarkAttemptEvent, +): boolean { + return ( + left.attemptId === right.attemptId && + left.candidateId === right.candidateId && + left.candidateRef === right.candidateRef && + left.adapterId === right.adapterId && + left.caseId === right.caseId && + left.cellId === right.cellId && + canonicalJson(left.scope) === canonicalJson(right.scope) + ) +} + +function isAgentMemoryScope(value: unknown): value is AgentMemoryScope { + if (!isRecordValue(value)) return false + const allowed = new Set([ + 'tenantId', + 'userId', + 'agentId', + 'teamId', + 'runId', + 'sessionId', + 'namespace', + 'tags', + ]) + if (Object.keys(value).some((key) => !allowed.has(key))) return false + for (const key of [ + 'tenantId', + 'userId', + 'agentId', + 'teamId', + 'runId', + 'sessionId', + 'namespace', + ]) { + if (value[key] !== undefined && typeof value[key] !== 'string') return false + } + if (value.tags === undefined) return true + return ( + isRecordValue(value.tags) && + Object.values(value.tags).every((entry) => typeof entry === 'string') + ) +} + +function isNonEmptyString(value: unknown): value is string { + return typeof value === 'string' && value.trim().length > 0 +} + +function isRecordValue(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value) } export function createNoopMemoryBenchmarkAdapter(id = 'no-memory'): AgentMemoryAdapter { @@ -1203,6 +1797,11 @@ export async function runKnowledgeBenchmarkSuite, ): Promise> { assertKnowledgeBenchmarkCases(options.cases) + if (options.respondRef !== undefined) { + assertNonEmptyBenchmarkString(options.respondRef, 'knowledge benchmark respondRef') + } else if (options.resumable !== false) { + throw new Error('knowledge benchmark respondRef is required when resumable is enabled') + } const storage = options.storage ?? fsCampaignStorage() const scenarios = buildKnowledgeBenchmarkScenarios(options.cases, options.splits) const dispatch: RunCampaignOptions['dispatch'] = async ( @@ -1215,7 +1814,13 @@ export async function runKnowledgeBenchmarkSuite({ scenarios, dispatch, - dispatchRef: 'agent-knowledge:benchmark-suite', + dispatchRef: stableId( + 'knowledge_benchmark', + canonicalJson({ + implementationRef: KNOWLEDGE_BENCHMARK_IMPLEMENTATION_REF, + respondRef: options.respondRef ?? 'non-resumable', + }), + ), judges: [knowledgeBenchmarkJudge()], runDir: options.runDir, repo: options.repo, @@ -1223,6 +1828,8 @@ export async function runKnowledgeBenchmarkSuite( > { return { name: 'knowledge-benchmark', + judgeVersion: 'agent-knowledge:knowledge-benchmark:v2', dimensions: [ { key: 'score', description: 'primary knowledge benchmark score' }, { key: 'passed', description: '1 when the benchmark case passes' }, @@ -1451,20 +2059,29 @@ export function scoreMemoryBenchmarkArtifact( testCase.forbiddenFacts && testCase.forbiddenFacts.length > 0 ? forbidden.safe : undefined, ].filter((value): value is number => value !== undefined) const score = mean(components) + const dimensions: Record = {} + if (required.totalWeight > 0) { + dimensions.memory_fact_recall = required.recall + dimensions.memory_required_fact_count = required.total + dimensions.memory_matched_fact_count = required.matched + } + if (testCase.expectedEventIds && testCase.expectedEventIds.length > 0) { + dimensions.memory_event_recall = eventRecall + } + if (testCase.expectedActorIds && testCase.expectedActorIds.length > 0) { + dimensions.memory_actor_recall = actorRecall + } + if (testCase.forbiddenFacts && testCase.forbiddenFacts.length > 0) { + dimensions.memory_stale_safe = forbidden.safe + dimensions.memory_stale_rate = forbidden.rate + dimensions.memory_forbidden_fact_count = forbidden.total + dimensions.memory_matched_forbidden_fact_count = forbidden.matched + } return { score, passed: score >= 1, - dimensions: { - memory_fact_recall: required.recall, - memory_event_recall: eventRecall, - memory_actor_recall: actorRecall, - memory_stale_safe: forbidden.safe, - memory_stale_rate: forbidden.rate, - memory_required_fact_count: required.total, - memory_matched_fact_count: required.matched, - memory_forbidden_fact_count: forbidden.total, - memory_matched_forbidden_fact_count: forbidden.matched, - }, + dimensions, + applicableDimensions: Object.keys(dimensions), notes: `memory required=${required.matched}/${required.total}; stale=${forbidden.matched}/${forbidden.total}; event_recall=${eventRecall.toFixed(3)}; actor_recall=${actorRecall.toFixed(3)}`, raw: { matchedRequiredFactIds: required.matchedIds, @@ -1649,10 +2266,13 @@ function memoryLifecycleCase(input: { function renderMemoryAdapterRankingMarkdown( rows: readonly MemoryAdapterBenchmarkRankingRow[], + totalCostUsd: number, ): string { return [ '# Memory Adapter Ranking', '', + `- total cost: $${formatNumber(totalCostUsd)}`, + '', '| rank | candidate | adapter | cases | cells | failed | mean score | pass rate | cost |', '| ---: | --- | --- | ---: | ---: | ---: | ---: | ---: | ---: |', ...rows.map( @@ -1666,19 +2286,34 @@ function renderMemoryAdapterRankingMarkdown( function benchmarkMemoryScope( candidateId: string, testCase: KnowledgeMemoryBenchmarkCase, + cellId: string, + attemptId: string, scope: AgentMemoryScope = {}, ): AgentMemoryScope { return { ...scope, - namespace: scope.namespace ?? 'agent-knowledge-memory-benchmark', + namespace: `${scope.namespace ?? 'agent-knowledge-memory-benchmark'}:${attemptId}`, tags: { ...(scope.tags ?? {}), benchmarkCandidateId: candidateId, benchmarkCaseId: testCase.id, + benchmarkCellId: cellId, + benchmarkAttemptId: attemptId, }, } } +function assertScopedMemoryBenchmarkAdapter(adapter: AgentMemoryAdapter): void { + if (adapter.branchIsolation?.mode !== 'scoped') { + throw new Error( + `${adapter.id}: direct memory benchmark requires branchIsolation mode scoped; use runAgentMemoryExperiment for dedicated instances`, + ) + } + if (!adapter.clear) { + throw new Error(`${adapter.id}: direct memory benchmark requires exact scoped clear`) + } +} + function memoryEventId(hit: AgentMemoryHit): string | undefined { const eventId = hit.metadata?.eventId return typeof eventId === 'string' ? eventId : undefined diff --git a/src/memory/attempt-log.ts b/src/memory/attempt-log.ts new file mode 100644 index 0000000..ed5514b --- /dev/null +++ b/src/memory/attempt-log.ts @@ -0,0 +1,74 @@ +import type { CampaignStorage } from '@tangle-network/agent-eval/campaign' + +export interface AttemptJournalEvent { + status: 'started' | 'cleaned' + recordedAt: string +} + +export function appendAttemptJournalEvent(input: { + storage: CampaignStorage + path: string + event: TEvent + label: string +}): void { + const { storage, path, event, label } = input + if (!storage.append) throw new Error(`${label} requires CampaignStorage.append`) + const line = `${JSON.stringify(event)}\n` + for (let retry = 0; retry < 100; retry += 1) { + const current = storage.read(path) + if (current === undefined && storage.exists(path)) { + throw new Error(`cannot read ${label} '${path}'`) + } + const expectedBytes = new TextEncoder().encode(current ?? '').byteLength + if (storage.append(path, line, expectedBytes) !== undefined) return + } + throw new Error(`${label} '${path}' remained contended after 100 retries`) +} + +export function readActiveAttemptJournal(input: { + storage: CampaignStorage + path: string + label: string + parse(value: unknown, path: string, line: number): TEvent + id(event: TEvent): string + sameAttempt(left: TEvent, right: TEvent): boolean +}): TEvent[] { + const { storage, path, label, parse, id, sameAttempt } = input + const stored = storage.read(path) + if (stored === undefined) { + if (storage.exists(path)) throw new Error(`cannot read ${label} '${path}'`) + return [] + } + const active = new Map() + const completed = new Map() + for (const [index, line] of stored.split('\n').entries()) { + if (!line) continue + let raw: unknown + try { + raw = JSON.parse(line) + } catch (error) { + throw new Error(`invalid ${label} '${path}' line ${index + 1}`, { cause: error }) + } + const event = parse(raw, path, index + 1) + const attemptId = id(event) + if (event.status === 'started') { + if (active.has(attemptId) || completed.has(attemptId)) { + throw new Error(`${label} '${path}' repeats attempt '${attemptId}'`) + } + active.set(attemptId, event) + continue + } + + const started = active.get(attemptId) + if (started && sameAttempt(started, event)) { + active.delete(attemptId) + completed.set(attemptId, started) + continue + } + + const prior = completed.get(attemptId) + if (prior && sameAttempt(prior, event)) continue + throw new Error(`${label} '${path}' has an unmatched cleanup for '${attemptId}'`) + } + return [...active.values()] +} diff --git a/src/memory/branch.ts b/src/memory/branch.ts index 7bede10..9ca2a1d 100644 --- a/src/memory/branch.ts +++ b/src/memory/branch.ts @@ -1,15 +1,16 @@ import { canonicalJson } from '@tangle-network/agent-eval' import { sha256, stableId } from '../ids' import { defaultGetMemoryContext } from './adapter' +import { mergeRankedMemoryHits } from './rank' import type { AgentMemoryAdapter, - AgentMemoryHit, AgentMemoryScope, AgentMemoryWriteInput, AgentMemoryWriteResult, } from './types' export type AgentMemoryVisibility = 'private' | 'team' | 'shared' +export type AgentMemoryBranchLifetime = 'resumable' | 'attempt' export interface AgentMemorySharingPolicy { read: readonly AgentMemoryVisibility[] @@ -24,6 +25,7 @@ export interface AgentMemoryJournalEntry { export interface AgentMemoryBranchSnapshot { branchId: string + lifetime: AgentMemoryBranchLifetime parentBranchId?: string adapterId: string policy: AgentMemorySharingPolicy @@ -34,6 +36,7 @@ export interface AgentMemoryBranchSnapshot { export interface AgentMemoryBranch extends AgentMemoryAdapter { readonly branchId: string + readonly lifetime: AgentMemoryBranchLifetime readonly parentBranchId?: string readonly policy: AgentMemorySharingPolicy snapshot(): Promise @@ -42,6 +45,7 @@ export interface AgentMemoryBranch extends AgentMemoryAdapter { adapter?: AgentMemoryAdapter policy?: AgentMemorySharingPolicy baseScope?: AgentMemoryScope + lifetime?: AgentMemoryBranchLifetime }): Promise } @@ -51,12 +55,23 @@ export interface CreateAgentMemoryBranchOptions { parentBranchId?: string policy?: AgentMemorySharingPolicy baseScope?: AgentMemoryScope + /** Attempt branches use a new ID after a process restart and cannot bind persisted state. */ + lifetime?: AgentMemoryBranchLifetime /** Rebind a persisted branch whose external memory is still present. */ snapshot?: AgentMemoryBranchSnapshot /** Optional exact logical scopes allowed to write, used by crash-safe experiments. */ allowedWriteScopes?: readonly AgentMemoryScope[] } +export interface ForkAgentMemoryBranchSnapshotOptions { + snapshot: AgentMemoryBranchSnapshot + adapter: AgentMemoryAdapter + branchId: string + policy?: AgentMemorySharingPolicy + baseScope?: AgentMemoryScope + lifetime?: AgentMemoryBranchLifetime +} + const DEFAULT_POLICY: AgentMemorySharingPolicy = { read: ['private'], write: 'private', @@ -91,6 +106,14 @@ export function createAgentMemoryBranch( `memory branch snapshot uses ${snapshot.adapterId}, not ${options.adapter.id}; fork it to replay into another adapter`, ) } + if (snapshot?.lifetime === 'attempt') { + throw new Error( + 'memory branch attempt snapshots cannot be resumed; replay them into a fresh branch instead', + ) + } + if (options.lifetime !== undefined && !['resumable', 'attempt'].includes(options.lifetime)) { + throw new Error('memory branch lifetime must be resumable or attempt') + } if (snapshot) { assertDurableValue(snapshot, `${options.branchId}: branch snapshot`) assertSnapshotDigest(snapshot) @@ -110,11 +133,15 @@ export function createAgentMemoryBranch( ) { throw new Error('memory branch snapshot scope cannot change during resume; fork it instead') } + if (snapshot && options.lifetime && options.lifetime !== snapshot.lifetime) { + throw new Error('memory branch snapshot lifetime cannot change during resume; fork it instead') + } const branchId = options.branchId const parentBranchId = options.parentBranchId ?? snapshot?.parentBranchId + const lifetime = options.lifetime ?? snapshot?.lifetime ?? 'resumable' const policy = normalizePolicy(options.policy ?? snapshot?.policy ?? DEFAULT_POLICY) - assertBranchIsolation(options.adapter, branchId, policy) + assertBranchIsolation(options.adapter, branchId, policy, lifetime) const mergedBaseScope = mergeScopes(undefined, snapshot?.baseScope ?? options.baseScope) assertDurableValue(mergedBaseScope, `${branchId}: base scope`) const baseScope = cloneDurableValue(mergedBaseScope) @@ -206,6 +233,7 @@ export function createAgentMemoryBranch( const branch: AgentMemoryBranch = { id: `${options.adapter.id}:branch:${stableId('branch', branchId)}`, branchId, + lifetime, ...(parentBranchId !== undefined ? { parentBranchId } : {}), policy, async search(query, searchOptions = {}) { @@ -235,7 +263,7 @@ export function createAgentMemoryBranch( })) }), ) - return mergeHits(groups.flat(), searchOptions.limit) + return mergeRankedMemoryHits(groups, searchOptions.limit) })(), ) }, @@ -249,6 +277,7 @@ export function createAgentMemoryBranch( nextSequence += 1 assertDurableValue(input, `${branch.id}: write input`) const logicalInput = cloneWriteInput(input) + logicalInput.id ??= stableId('memory_write', `${branchId}:${sequence}`) const logicalScope = mergeScopes(baseScope, logicalInput.scope) if ( allowedWriteScopeKeys && @@ -311,39 +340,50 @@ export function createAgentMemoryBranch( } catch (error) { flushErrors.push(error) } - const logicalScopes = scope - ? [mergeScopes(baseScope, scope)] - : touchedScopes.size > 0 - ? [...touchedScopes.values()] - : [baseScope] + const requestedScope = scope ? mergeScopes(baseScope, scope) : undefined + const logicalScopes = new Map() + if (requestedScope) { + logicalScopes.set(canonicalJson(stripUndefined(requestedScope)), requestedScope) + for (const touched of touchedScopes.values()) { + if (scopeMatches(touched, requestedScope)) { + logicalScopes.set(canonicalJson(stripUndefined(touched)), touched) + } + } + } else if (touchedScopes.size > 0) { + for (const [key, touched] of touchedScopes) logicalScopes.set(key, touched) + } else { + logicalScopes.set(canonicalJson(stripUndefined(baseScope)), baseScope) + } const visibilities = new Set([...policy.read, policy.write]) const physicalScopes = new Map() - for (const logicalScope of logicalScopes) { + for (const logicalScope of logicalScopes.values()) { for (const visibility of visibilities) { const physical = physicalScope(branchId, logicalScope, visibility) physicalScopes.set(canonicalJson(stripUndefined(physical)), physical) } } const clearErrors: unknown[] = [] - for (const physical of physicalScopes.values()) { + const clearedPhysicalScopes = new Set() + for (const [key, physical] of physicalScopes) { try { await options.adapter.clear(physical) + clearedPhysicalScopes.add(key) } catch (error) { clearErrors.push(error) } } - if (clearErrors.length === 0) { - if (!scope) { - journal.splice(0, journal.length) - touchedScopes.clear() - } else { - for (let index = journal.length - 1; index >= 0; index -= 1) { - if (scopeMatches(mergeScopes(baseScope, journal[index]!.input.scope), scope)) { - journal.splice(index, 1) - } + if (clearedPhysicalScopes.size > 0) { + for (let index = journal.length - 1; index >= 0; index -= 1) { + const logical = mergeScopes(baseScope, journal[index]!.input.scope) + const physical = physicalScope(branchId, logical, policy.write) + if (clearedPhysicalScopes.has(canonicalJson(stripUndefined(physical)))) { + journal.splice(index, 1) } - for (const [key, touched] of touchedScopes) { - if (scopeMatches(touched, scope)) touchedScopes.delete(key) + } + for (const [key, touched] of touchedScopes) { + const physical = physicalScope(branchId, touched, policy.write) + if (clearedPhysicalScopes.has(canonicalJson(stripUndefined(physical)))) { + touchedScopes.delete(key) } } } @@ -392,6 +432,7 @@ export function createAgentMemoryBranch( runExclusive(async () => { return createSnapshot({ branchId, + lifetime, adapterId: options.adapter.id, policy, baseScope, @@ -407,57 +448,76 @@ export function createAgentMemoryBranch( throw new Error(`memory branch ${branchId}: child branchId must differ from its parent`) } return track( - (async () => { - const parent = await branch.snapshot() - const child = createAgentMemoryBranch({ + (async () => + forkAgentMemoryBranchSnapshot({ + snapshot: await branch.snapshot(), adapter: forkOptions.adapter ?? options.adapter, branchId: forkOptions.branchId, - parentBranchId: branchId, - policy: forkOptions.policy ?? policy, - baseScope: mergeScopes(baseScope, forkOptions.baseScope), - }) - try { - for (const entry of parent.journal) { - const result = await child.write(entry.input) - if (!result.accepted) { - throw new Error( - `memory branch ${forkOptions.branchId}: adapter rejected replay of journal entry ${entry.sequence}`, - ) - } - } - await child.flush?.() - return child - } catch (error) { - const cleanupErrors: unknown[] = [] - try { - await child.clear?.() - } catch (cleanupError) { - cleanupErrors.push(cleanupError) - } - try { - await child.close?.() - } catch (cleanupError) { - cleanupErrors.push(cleanupError) - } - if (cleanupErrors.length > 0) { - throw new AggregateError( - [error, ...cleanupErrors], - `memory branch ${forkOptions.branchId}: replay and cleanup failed`, - ) - } - throw error - } - })(), + policy: forkOptions.policy, + baseScope: forkOptions.baseScope, + lifetime: forkOptions.lifetime, + }))(), ) }, } return branch } +/** Replays a durable journal into a fresh provider branch. */ +export async function forkAgentMemoryBranchSnapshot( + options: ForkAgentMemoryBranchSnapshotOptions, +): Promise { + assertDurableValue(options.snapshot, `${options.snapshot.branchId}: branch snapshot`) + assertSnapshotDigest(options.snapshot) + if (options.branchId === options.snapshot.branchId) { + throw new Error(`memory branch ${options.branchId}: child branchId must differ from its parent`) + } + const child = createAgentMemoryBranch({ + adapter: options.adapter, + branchId: options.branchId, + parentBranchId: options.snapshot.branchId, + policy: options.policy ?? options.snapshot.policy, + baseScope: mergeScopes(options.snapshot.baseScope, options.baseScope), + lifetime: options.lifetime ?? options.snapshot.lifetime, + }) + try { + for (const entry of options.snapshot.journal) { + const result = await child.write(entry.input) + if (!result.accepted) { + throw new Error( + `memory branch ${options.branchId}: adapter rejected replay of journal entry ${entry.sequence}`, + ) + } + } + await child.flush?.() + return child + } catch (error) { + const cleanupErrors: unknown[] = [] + try { + await child.clear?.() + } catch (cleanupError) { + cleanupErrors.push(cleanupError) + } + try { + await child.close?.() + } catch (cleanupError) { + cleanupErrors.push(cleanupError) + } + if (cleanupErrors.length > 0) { + throw new AggregateError( + [error, ...cleanupErrors], + `memory branch ${options.branchId}: replay and cleanup failed`, + ) + } + throw error + } +} + function assertBranchIsolation( adapter: AgentMemoryAdapter, branchId: string, policy: AgentMemorySharingPolicy, + lifetime: AgentMemoryBranchLifetime, ): void { const isolation = adapter.branchIsolation if (!isolation) { @@ -470,6 +530,22 @@ function assertBranchIsolation( `${adapter.id}: cannot isolate memory branch '${branchId}': ${isolation.reason}`, ) } + if ( + isolation.mode === 'scoped' && + isolation.processExitSafe === false && + lifetime !== 'attempt' + ) { + throw new Error( + `${adapter.id}: memory branch '${branchId}' must use lifetime='attempt' because provider writes can outlive the worker process`, + ) + } + if ( + isolation.mode === 'scoped' && + isolation.processExitSafe === false && + (!Number.isSafeInteger(isolation.recoveryDelayMs) || isolation.recoveryDelayMs! <= 0) + ) { + throw new Error(`${adapter.id}: processExitSafe=false requires a positive recoveryDelayMs`) + } if (isolation.mode === 'instance' && isolation.branchId !== branchId) { throw new Error( `${adapter.id}: adapter instance belongs to branch '${isolation.branchId}', not '${branchId}'`, @@ -575,24 +651,6 @@ function normalizePolicy(policy: AgentMemorySharingPolicy): AgentMemorySharingPo return { read, write: policy.write } } -function mergeHits(hits: AgentMemoryHit[], limit?: number): AgentMemoryHit[] { - const byIdentity = new Map() - for (const hit of hits) { - const key = `${hit.uri}\n${hit.text}` - const prior = byIdentity.get(key) - const score = hit.normalizedScore ?? hit.score ?? Number.NEGATIVE_INFINITY - const priorScore = prior?.normalizedScore ?? prior?.score ?? Number.NEGATIVE_INFINITY - if (!prior || score > priorScore) byIdentity.set(key, hit) - } - return [...byIdentity.values()] - .sort( - (a, b) => - (b.normalizedScore ?? b.score ?? 0) - (a.normalizedScore ?? a.score ?? 0) || - a.id.localeCompare(b.id), - ) - .slice(0, limit ?? Number.POSITIVE_INFINITY) -} - function createSnapshot( input: Omit, ): AgentMemoryBranchSnapshot { diff --git a/src/memory/experiment.ts b/src/memory/experiment.ts index f890b5c..c12f6a9 100644 --- a/src/memory/experiment.ts +++ b/src/memory/experiment.ts @@ -1,9 +1,11 @@ +import { randomUUID } from 'node:crypto' import { join } from 'node:path' import { canonicalJson } from '@tangle-network/agent-eval' import { type CampaignResult, type CampaignStorage, type CostLedgerHandle, + createRunCostLedger, type DispatchContext, fsCampaignStorage, type JudgeConfig, @@ -21,14 +23,31 @@ import { scoreMemoryBenchmarkArtifact, } from '../benchmarks/index' import { stableId } from '../ids' +import { appendAttemptJournalEvent, readActiveAttemptJournal } from './attempt-log' import { type AgentMemoryBranch, type AgentMemoryBranchSnapshot, type AgentMemorySharingPolicy, createAgentMemoryBranch, } from './branch' +import { + createMemoryExecutionPool, + memoryRecoveryDelayMs, + resolveMemoryCleanupTimeoutMs, + runBoundedMemoryLifecycle, + sleepForMemoryRecovery, +} from './lifecycle' +import { + type AgentMemoryAcquireRunLease, + type AgentMemoryControllerMode, + type AgentMemoryRunLease, + acquireAgentMemoryRunLease, + type OwnedAgentMemoryRunLease, +} from './run-control' import type { AgentMemoryAdapter, AgentMemoryScope, AgentMemoryWriteInput } from './types' +const MEMORY_EXPERIMENT_IMPLEMENTATION_REF = 'agent-knowledge:memory-experiment:v3' + export interface AgentMemorySequenceProbe { id: string query: string @@ -78,17 +97,20 @@ export interface AgentMemoryExperimentCandidate { id: string label?: string /** Change when provider configuration changes so cached cells cannot be reused. */ - ref?: string + ref: string createAdapter(input: { branchId: string sequence: AgentMemorySequence rep: number seed: number - }): AgentMemoryAdapter | Promise + purpose: 'execute' | 'recovery' + }): AgentMemoryAdapter | null | Promise policy?: AgentMemorySharingPolicy baseScope?: AgentMemoryScope /** Conservative external provider charge for one complete history. */ externalCostUsdPerSequence?: number + /** Conservative extra provider charge when recovering one interrupted history. */ + externalRecoveryCostUsdPerAttempt?: number /** Release resources and, when cleanupBranches is false, delete the isolated state. */ disposeAdapter?(adapter: AgentMemoryAdapter): Promise } @@ -100,6 +122,7 @@ export interface AgentMemorySequenceProbeResult { score: number passed: boolean dimensions: Record + applicableDimensions: readonly string[] notes: string hitIds: readonly string[] } @@ -110,6 +133,7 @@ export interface AgentMemorySequenceArtifact { score: number passed: boolean dimensions: Record + dimensionSampleCounts: Record probes: readonly AgentMemorySequenceProbeResult[] branchDigest: string journalEntries: number @@ -173,7 +197,29 @@ export interface RunAgentMemoryExperimentOptions { costPhase?: string maxConcurrency?: number dispatchTimeoutMs?: number + /** Total deadline for each provider cleanup, close, or recovery operation. */ + cleanupTimeoutMs?: number now?: () => Date + /** Required with custom storage when all controllers are confined to one process. */ + controllerMode?: AgentMemoryControllerMode + /** Required for distributed controllers that share custom storage. */ + acquireRunLease?: AgentMemoryAcquireRunLease +} + +export type AgentMemoryExperimentRunLease = AgentMemoryRunLease + +export interface AgentMemoryAttemptEvent { + schema: 1 + status: 'started' | 'cleaned' + branchId: string + candidateId: string + candidateRef: string + sequenceId: string + rep: number + seed: number + cleanupBranches: boolean + recordedAt: string + recovery: boolean } export interface RunAgentMemoryExperimentResult { @@ -182,19 +228,18 @@ export interface RunAgentMemoryExperimentResult { leaderCandidateId?: string rankingJsonPath: string rankingMarkdownPath: string + attemptLogPath: string } class AgentMemoryCleanupError extends AggregateError { - constructor( - errors: Iterable, - message: string, - readonly afterAbort: boolean, - ) { + constructor(errors: Iterable, message: string) { super(errors, message) this.name = 'AgentMemoryCleanupError' } } +type OwnedMemoryExperimentRunLease = OwnedAgentMemoryRunLease + /** Converts existing ordered memory benchmark cases into executable histories. */ export function buildAgentMemorySequencesFromBenchmarkCases( cases: readonly KnowledgeMemoryBenchmarkCase[], @@ -295,6 +340,11 @@ export async function runAgentMemoryExperiment( ) assertMemorySequences(options.sequences) for (const candidate of options.candidates) { + if (options.cleanupBranches === false && !candidate.disposeAdapter) { + throw new Error( + `${candidate.id}: cleanupBranches=false requires disposeAdapter to delete isolated external state`, + ) + } if ( candidate.externalCostUsdPerSequence !== undefined && (!Number.isFinite(candidate.externalCostUsdPerSequence) || @@ -304,11 +354,67 @@ export async function runAgentMemoryExperiment( `${candidate.id}: externalCostUsdPerSequence must be a non-negative finite number`, ) } - if (candidate.ref !== undefined) assertNonEmptyString(candidate.ref, `${candidate.id} ref`) + if ( + candidate.externalRecoveryCostUsdPerAttempt !== undefined && + (!Number.isFinite(candidate.externalRecoveryCostUsdPerAttempt) || + candidate.externalRecoveryCostUsdPerAttempt < 0) + ) { + throw new Error( + `${candidate.id}: externalRecoveryCostUsdPerAttempt must be a non-negative finite number`, + ) + } + assertNonEmptyString(candidate.ref, `${candidate.id} ref`) } const storage = options.storage ?? fsCampaignStorage() const runDir = resolveRunDir(options.runDir, options.repo) + if (!storage.append) { + throw new Error('memory experiment requires CampaignStorage.append for durable attempt state') + } + resolveMemoryCleanupTimeoutMs(options.cleanupTimeoutMs, 'memory experiment') + storage.ensureDir(runDir) + const lease = await acquireAgentMemoryRunLease({ + experimentId: options.experimentId, + runDir, + storage, + customStorage: options.storage !== undefined, + lockFileName: 'memory-experiment.lock', + label: 'memory experiment', + controllerMode: options.controllerMode, + acquireRunLease: options.acquireRunLease, + }) + let result: RunAgentMemoryExperimentResult | undefined + let primaryError: unknown + try { + await lease.assertOwned() + result = await runOwnedAgentMemoryExperiment(options, storage, runDir, lease) + } catch (error) { + primaryError = error + } + let releaseError: unknown + try { + await lease.release() + } catch (error) { + releaseError = error + } + if (primaryError && releaseError) { + throw new AggregateError( + [primaryError, releaseError], + 'memory experiment failed and its controller lease could not be released', + ) + } + if (primaryError) throw primaryError + if (releaseError) throw releaseError + if (!result) throw new Error('memory experiment produced no result') + return result +} + +async function runOwnedAgentMemoryExperiment( + options: RunAgentMemoryExperimentOptions, + storage: CampaignStorage, + runDir: string, + lease: OwnedMemoryExperimentRunLease, +): Promise { const runIdentity = stableId( 'memory_run', canonicalJson({ @@ -320,11 +426,37 @@ export async function runAgentMemoryExperiment( if (!Number.isSafeInteger(maxConcurrency) || maxConcurrency <= 0) { throw new Error('memory experiment maxConcurrency must be a positive safe integer') } - const executionPool = createExecutionPool(maxConcurrency) + const executionPool = createMemoryExecutionPool(maxConcurrency) const dispatchedExecutions: Promise[] = [] const candidateById = new Map(options.candidates.map((candidate) => [candidate.id, candidate])) + const sequenceById = new Map(options.sequences.map((sequence) => [sequence.id, sequence])) const scenarios = buildAgentMemorySequenceScenarios(options.sequences, options.candidates) - let campaign: CampaignResult + const attemptLogPath = join(runDir, 'memory-attempts.jsonl') + const costLedger = + options.costLedger ?? + createRunCostLedger({ + storage, + runDir, + costCeilingUsd: options.costCeiling, + }) + if (options.costCeiling !== undefined && costLedger.costCeilingUsd !== options.costCeiling) { + throw new Error('memory experiment costCeiling must match the shared cost ledger ceiling') + } + storage.ensureDir(runDir) + const recoveryCostByCandidate = await recoverAbandonedMemoryAttempts({ + options, + storage, + runDir, + attemptLogPath, + candidateById, + sequenceById, + lease, + maxConcurrency, + costLedger, + }) + await lease.assertOwned() + let campaign: CampaignResult | undefined + let campaignError: unknown let settledExecutions: PromiseSettledResult[] = [] try { campaign = await runCampaign({ @@ -333,7 +465,16 @@ export async function runAgentMemoryExperiment( const candidate = candidateById.get(scenario.candidateId) if (!candidate) throw new Error(`unknown memory candidate ${scenario.candidateId}`) const operation = executionPool.run(() => - runSequenceCell({ options, candidate, scenario, context, runIdentity }), + runSequenceCell({ + options, + candidate, + scenario, + context, + runIdentity, + storage, + attemptLogPath, + lease, + }), ) dispatchedExecutions.push(operation) return operation @@ -346,27 +487,41 @@ export async function runAgentMemoryExperiment( reps: options.reps, resumable: options.resumable, costCeiling: options.costCeiling, - costLedger: options.costLedger, + costLedger, costPhase: options.costPhase, maxConcurrency, dispatchTimeoutMs: options.dispatchTimeoutMs, expectUsage: 'off', now: options.now, }) + } catch (error) { + campaignError = error } finally { settledExecutions = await Promise.allSettled(dispatchedExecutions) } const cleanupFailures = settledExecutions.flatMap((settled) => - settled.status === 'rejected' && - settled.reason instanceof AgentMemoryCleanupError && - settled.reason.afterAbort + settled.status === 'rejected' && settled.reason instanceof AgentMemoryCleanupError ? [settled.reason] : [], ) + if (campaignError && cleanupFailures.length > 0) { + throw new AggregateError( + [campaignError, ...cleanupFailures], + 'memory experiment failed and provider cleanup also failed', + ) + } + if (campaignError) throw campaignError if (cleanupFailures.length > 0) { throw new AggregateError(cleanupFailures, 'memory experiment cleanup failed after dispatch') } - const rows = rankAgentMemoryExperiment(options.candidates, scenarios, campaign) + if (!campaign) throw new Error('memory experiment produced no campaign result') + await lease.assertOwned() + const rows = rankAgentMemoryExperiment( + options.candidates, + scenarios, + campaign, + recoveryCostByCandidate, + ) const rankingJsonPath = join(campaign.runDir, 'memory-experiment-ranking.json') const rankingMarkdownPath = join(campaign.runDir, 'memory-experiment-ranking.md') storage.write( @@ -380,6 +535,7 @@ export async function runAgentMemoryExperiment( leaderCandidateId: rows.find((row) => row.cellsFailed === 0)?.candidateId, rankingJsonPath, rankingMarkdownPath, + attemptLogPath, } } @@ -406,6 +562,7 @@ export function agentMemorySequenceJudge(): JudgeConfig< > { return { name: 'agent-memory-sequence', + judgeVersion: 'agent-knowledge:memory-sequence:v2', dimensions: [ { key: 'score', description: 'mean memory probe score' }, { key: 'passed', description: '1 when every memory probe passes' }, @@ -434,8 +591,12 @@ async function runSequenceCell(input: { scenario: AgentMemorySequenceScenario context: DispatchContext runIdentity: string + storage: CampaignStorage + attemptLogPath: string + lease: OwnedMemoryExperimentRunLease }): Promise { - const { options, candidate, scenario, context, runIdentity } = input + const { options, candidate, scenario, context, runIdentity, storage, attemptLogPath, lease } = + input const cleanupBranches = options.cleanupBranches ?? true const costUsd = candidate.externalCostUsdPerSequence ?? 0 if (!Number.isFinite(costUsd) || costUsd < 0) { @@ -448,21 +609,52 @@ async function runSequenceCell(input: { `${candidate.id}: cleanupBranches=false requires disposeAdapter to delete isolated external state`, ) } + const cleanupTimeoutMs = resolveMemoryCleanupTimeoutMs( + options.cleanupTimeoutMs, + `${candidate.id}: memory sequence`, + ) + let externalCallAttempted = false const execute = async (): Promise => { context.signal.throwIfAborted() + await lease.assertOwned() const startedAt = Date.now() - const branchId = stableId('memory_branch', `${runIdentity}:${context.cellId}:${context.seed}`) - const adapter = await candidate.createAdapter({ + const branchId = stableId( + 'memory_branch', + `${runIdentity}:${context.cellId}:${context.seed}:${randomUUID()}`, + ) + const attempt = memoryAttemptEvent({ + status: 'started', branchId, + candidate, sequence: scenario.sequence, rep: context.rep, seed: context.seed, + recovery: false, + cleanupBranches, + now: options.now, }) + appendMemoryAttemptEvent(storage, attemptLogPath, attempt) + let adapter: AgentMemoryAdapter | undefined let memory: AgentMemoryBranch | undefined let primaryError: unknown let completedArtifact: AgentMemorySequenceArtifact | undefined + let finalClearStarted = false + let finalClearCompleted = false try { + // A factory can contact a provider before it rejects, so charge conservatively once invoked. + externalCallAttempted = true + const created = await candidate.createAdapter({ + branchId, + sequence: scenario.sequence, + rep: context.rep, + seed: context.seed, + purpose: 'execute', + }) + if (!created) throw new Error(`${candidate.id}: createAdapter returned no execution adapter`) + adapter = created + await lease.assertOwned() + context.signal.throwIfAborted() if (cleanupBranches && !adapter.clear) { throw new Error( `${candidate.id}: cleanupBranches requires an adapter with scoped clear support`, @@ -471,21 +663,25 @@ async function runSequenceCell(input: { memory = createAgentMemoryBranch({ adapter, branchId, + lifetime: 'attempt', policy: candidate.policy, allowedWriteScopes: sequenceCleanupScopes(scenario.sequence), - baseScope: mergeScopes(candidate.baseScope, { - tags: { - memoryExperimentId: options.experimentId, - memoryCandidateId: candidate.id, - memorySequenceId: scenario.sequenceId, - }, - }), + baseScope: memoryExperimentBaseScope(options, candidate, scenario.sequenceId), }) - if (cleanupBranches) await clearSequenceScopes(memory, scenario.sequence) + if (cleanupBranches) { + await runBoundedMemoryLifecycle({ + operation: `${candidate.id}: initial branch cleanup`, + timeoutMs: cleanupTimeoutMs, + run: () => clearSequenceScopes(memory!, scenario.sequence), + }) + await lease.assertOwned() + } const probes: AgentMemorySequenceProbeResult[] = [] for (const step of scenario.sequence.steps) { context.signal.throwIfAborted() + await lease.assertOwned() await writeStep(memory, step) + await lease.assertOwned() await options.executeStep?.({ memory, candidateId: candidate.id, @@ -494,62 +690,137 @@ async function runSequenceCell(input: { context, }) context.signal.throwIfAborted() + await lease.assertOwned() const stepProbes = await probeStep(memory, scenario.sequence, step) + await lease.assertOwned() probes.push(...stepProbes) } const snapshot = await memory.snapshot() + await lease.assertOwned() await options.onBranchSnapshot?.({ candidateId: candidate.id, sequenceId: scenario.sequenceId, cellId: context.cellId, snapshot, }) + await lease.assertOwned() const dimensions = meanDimensions(probes.map((probe) => probe.dimensions)) + const dimensionSampleCounts = countDimensions( + probes.map((probe) => probe.applicableDimensions), + ) const artifact: AgentMemorySequenceArtifact = { candidateId: candidate.id, sequenceId: scenario.sequenceId, score: mean(probes.map((probe) => probe.score)), passed: probes.length > 0 && probes.every((probe) => probe.passed), dimensions, + dimensionSampleCounts, probes, branchDigest: snapshot.digest, journalEntries: snapshot.journal.length, durationMs: Math.max(0, Date.now() - startedAt), } - if (cleanupBranches) await clearSequenceScopes(memory, scenario.sequence) + if (cleanupBranches) { + finalClearStarted = true + await runBoundedMemoryLifecycle({ + operation: `${candidate.id}: final branch cleanup`, + timeoutMs: cleanupTimeoutMs, + run: () => clearSequenceScopes(memory!, scenario.sequence), + }) + finalClearCompleted = true + await lease.assertOwned() + } completedArtifact = artifact } catch (error) { primaryError = error } const cleanupErrors: unknown[] = [] - if (primaryError && memory && cleanupBranches && adapter.clear) { + let cleanupOwned = true + let ownershipError: unknown + try { + await lease.assertOwned() + } catch (error) { + cleanupOwned = false + ownershipError = error + } + if (finalClearStarted && !finalClearCompleted && primaryError) { + cleanupErrors.push(primaryError) + } + if ( + primaryError && + cleanupOwned && + !finalClearStarted && + memory && + cleanupBranches && + adapter?.clear + ) { try { - await clearSequenceScopes(memory, scenario.sequence) + await runBoundedMemoryLifecycle({ + operation: `${candidate.id}: failed branch cleanup`, + timeoutMs: cleanupTimeoutMs, + run: () => clearSequenceScopes(memory!, scenario.sequence), + }) } catch (error) { cleanupErrors.push(error) } } - try { - if (memory) await memory.close?.() - else { - await adapter.flush?.() - await adapter.close?.() + if (adapter) { + try { + await runBoundedMemoryLifecycle({ + operation: `${candidate.id}: adapter close`, + timeoutMs: cleanupTimeoutMs, + run: async () => { + if (memory && cleanupOwned) await memory.close?.() + else { + if (cleanupOwned) await adapter!.flush?.() + await adapter!.close?.() + } + }, + }) + } catch (error) { + cleanupErrors.push(error) } - } catch (error) { - cleanupErrors.push(error) + if (cleanupOwned) { + try { + await runBoundedMemoryLifecycle({ + operation: `${candidate.id}: adapter disposal`, + timeoutMs: cleanupTimeoutMs, + run: () => candidate.disposeAdapter?.(adapter!), + }) + } catch (error) { + cleanupErrors.push(error) + } + } + } else { + cleanupErrors.push( + new Error(`${candidate.id}: adapter creation failed before cleanup could be confirmed`), + ) } - try { - await candidate.disposeAdapter?.(adapter) - } catch (error) { - cleanupErrors.push(error) + if (cleanupOwned && cleanupErrors.length === 0) { + try { + appendMemoryAttemptEvent(storage, attemptLogPath, { + ...attempt, + status: 'cleaned', + recordedAt: (options.now ?? (() => new Date()))().toISOString(), + }) + } catch (error) { + cleanupErrors.push(error) + } + } + if (!cleanupOwned && cleanupErrors.length === 0) { + if (primaryError) throw primaryError + throw ownershipError } if (cleanupErrors.length > 0) { const primaryMessage = primaryError instanceof Error ? primaryError.message : String(primaryError ?? '') throw new AgentMemoryCleanupError( - primaryError ? [primaryError, ...cleanupErrors] : cleanupErrors, + [ + ...(primaryError && !cleanupErrors.includes(primaryError) ? [primaryError] : []), + ...(ownershipError ? [ownershipError] : []), + ...cleanupErrors, + ], `${candidate.id}: memory branch cleanup failed${primaryMessage ? ` after: ${primaryMessage}` : ''}`, - context.signal.aborted, ) } if (primaryError) throw primaryError @@ -571,31 +842,319 @@ async function runSequenceCell(input: { maximumCharge: { externallyEnforcedMaximumUsd: costUsd }, execute, receipt: () => receipt, - receiptFromError: () => receipt, + receiptFromError: () => ({ + ...receipt, + actualCostUsd: externalCallAttempted ? costUsd : 0, + }), }) if (!paid.succeeded) throw paid.error return paid.value } -function createExecutionPool(limit: number): { - run(operation: () => Promise): Promise -} { - let active = 0 - const waiters: Array<() => void> = [] - return { - async run(operation: () => Promise): Promise { - if (active >= limit) await new Promise((resolve) => waiters.push(resolve)) - active += 1 +async function recoverAbandonedMemoryAttempts(input: { + options: RunAgentMemoryExperimentOptions + storage: CampaignStorage + runDir: string + attemptLogPath: string + candidateById: ReadonlyMap + sequenceById: ReadonlyMap + lease: OwnedMemoryExperimentRunLease + maxConcurrency: number + costLedger: CostLedgerHandle +}): Promise> { + const attempts = readActiveMemoryAttempts(input.storage, input.attemptLogPath) + const pool = createMemoryExecutionPool(input.maxConcurrency) + const settled = await Promise.allSettled( + attempts + .sort((left, right) => left.branchId.localeCompare(right.branchId)) + .map((attempt) => + pool.run(async () => { + await input.lease.assertOwned() + const candidate = input.candidateById.get(attempt.candidateId) + if (!candidate) { + throw new Error( + `cannot recover memory branch '${attempt.branchId}': candidate '${attempt.candidateId}' is missing`, + ) + } + if (candidate.ref !== attempt.candidateRef) { + throw new Error( + `cannot recover memory branch '${attempt.branchId}': candidate ref changed from '${attempt.candidateRef}' to '${candidate.ref}'`, + ) + } + const sequence = input.sequenceById.get(attempt.sequenceId) + if (!sequence) { + throw new Error( + `cannot recover memory branch '${attempt.branchId}': sequence '${attempt.sequenceId}' is missing`, + ) + } + if ((input.options.cleanupBranches ?? true) !== attempt.cleanupBranches) { + throw new Error( + `cannot recover memory branch '${attempt.branchId}': cleanupBranches changed`, + ) + } + const recoveryCostUsd = candidate.externalRecoveryCostUsdPerAttempt ?? 0 + const recover = () => + recoverMemoryAttempt({ + options: input.options, + candidate, + sequence, + attempt, + lease: input.lease, + }) + if (recoveryCostUsd === 0) { + await recover() + } else { + const receipt = { + model: candidate.id, + inputTokens: 0, + outputTokens: 0, + usageUnknown: true, + actualCostUsd: recoveryCostUsd, + } as const + const paid = await input.costLedger.runPaidCall({ + channel: 'driver', + phase: `${input.options.costPhase ?? 'memory.experiment'}.recovery`, + actor: `agent-knowledge:memory-recovery:${candidate.id}`, + model: candidate.id, + tags: { + runDir: input.runDir, + candidateId: candidate.id, + branchId: attempt.branchId, + }, + maximumCharge: { externallyEnforcedMaximumUsd: recoveryCostUsd }, + execute: recover, + receipt: () => receipt, + receiptFromError: () => receipt, + }) + if (!paid.succeeded) throw paid.error + } + appendMemoryAttemptEvent(input.storage, input.attemptLogPath, { + ...attempt, + status: 'cleaned', + recovery: true, + recordedAt: (input.options.now ?? (() => new Date()))().toISOString(), + }) + }), + ), + ) + const failures = settled.flatMap((result) => + result.status === 'rejected' ? [result.reason] : [], + ) + if (failures.length === 1) throw failures[0] + if (failures.length > 1) { + throw new AggregateError(failures, 'multiple abandoned memory branches failed recovery') + } + return new Map( + [...input.candidateById.keys()].map((candidateId) => [ + candidateId, + input.costLedger.summary({ tags: { runDir: input.runDir, candidateId } }).totalCostUsd, + ]), + ) +} + +async function recoverMemoryAttempt(input: { + options: RunAgentMemoryExperimentOptions + candidate: AgentMemoryExperimentCandidate + sequence: AgentMemorySequence + attempt: AgentMemoryAttemptEvent + lease: OwnedMemoryExperimentRunLease +}): Promise { + const { options, candidate, sequence, attempt, lease } = input + const cleanupBranches = attempt.cleanupBranches + const cleanupTimeoutMs = resolveMemoryCleanupTimeoutMs( + options.cleanupTimeoutMs, + `${candidate.id}: abandoned branch recovery`, + ) + let adapter: AgentMemoryAdapter | undefined + let memory: AgentMemoryBranch | undefined + let primaryError: unknown + try { + const recovered = await runBoundedMemoryLifecycle({ + operation: `${candidate.id}: recovery adapter creation`, + timeoutMs: cleanupTimeoutMs, + run: () => + candidate.createAdapter({ + branchId: attempt.branchId, + sequence, + rep: attempt.rep, + seed: attempt.seed, + purpose: 'recovery', + }), + }) + await lease.assertOwned() + if (recovered === null) return + adapter = recovered + const recoveryDelayMs = memoryRecoveryDelayMs(adapter) + await sleepForMemoryRecovery(recoveryDelayMs, () => lease.assertOwned()) + if (cleanupBranches) { + if (!adapter.clear) { + throw new Error( + `${candidate.id}: abandoned branch recovery requires an adapter with scoped clear support`, + ) + } + memory = createAgentMemoryBranch({ + adapter, + branchId: attempt.branchId, + lifetime: 'attempt', + policy: candidate.policy, + allowedWriteScopes: sequenceCleanupScopes(sequence), + baseScope: memoryExperimentBaseScope(options, candidate, sequence.id), + }) + await runBoundedMemoryLifecycle({ + operation: `${candidate.id}: abandoned branch cleanup`, + timeoutMs: cleanupTimeoutMs, + run: () => clearSequenceScopes(memory!, sequence), + }) + await lease.assertOwned() + } + } catch (error) { + primaryError = error + } + + const cleanupErrors: unknown[] = [] + let cleanupOwned = true + try { + await lease.assertOwned() + } catch (error) { + cleanupOwned = false + cleanupErrors.push(error) + } + if (adapter) { + try { + await runBoundedMemoryLifecycle({ + operation: `${candidate.id}: recovery adapter close`, + timeoutMs: cleanupTimeoutMs, + run: async () => { + if (memory && cleanupOwned) await memory.close?.() + else { + if (cleanupOwned) await adapter!.flush?.() + await adapter!.close?.() + } + }, + }) + } catch (error) { + cleanupErrors.push(error) + } + if (cleanupOwned) { try { - return await operation() - } finally { - active -= 1 - waiters.shift()?.() + await runBoundedMemoryLifecycle({ + operation: `${candidate.id}: recovery adapter disposal`, + timeoutMs: cleanupTimeoutMs, + run: () => candidate.disposeAdapter?.(adapter!), + }) + } catch (error) { + cleanupErrors.push(error) } - }, + } + } else { + cleanupErrors.push( + new Error( + `${candidate.id}: recovery adapter creation failed before cleanup could be confirmed`, + ), + ) + } + if (!cleanupOwned || primaryError || cleanupErrors.length > 0) { + throw new AgentMemoryCleanupError( + [...(primaryError ? [primaryError] : []), ...cleanupErrors], + `${candidate.id}: abandoned memory branch '${attempt.branchId}' recovery failed`, + ) } } +function memoryAttemptEvent(input: { + status: AgentMemoryAttemptEvent['status'] + branchId: string + candidate: AgentMemoryExperimentCandidate + sequence: AgentMemorySequence + rep: number + seed: number + recovery: boolean + cleanupBranches?: boolean + now?: () => Date +}): AgentMemoryAttemptEvent { + return { + schema: 1, + status: input.status, + branchId: input.branchId, + candidateId: input.candidate.id, + candidateRef: input.candidate.ref, + sequenceId: input.sequence.id, + rep: input.rep, + seed: input.seed, + cleanupBranches: input.cleanupBranches ?? true, + recordedAt: (input.now ?? (() => new Date()))().toISOString(), + recovery: input.recovery, + } +} + +function appendMemoryAttemptEvent( + storage: CampaignStorage, + path: string, + event: AgentMemoryAttemptEvent, +): void { + appendAttemptJournalEvent({ + storage, + path, + event, + label: 'memory attempt log', + }) +} + +function readActiveMemoryAttempts( + storage: CampaignStorage, + path: string, +): AgentMemoryAttemptEvent[] { + return readActiveAttemptJournal({ + storage, + path, + label: 'memory attempt log', + parse: parseMemoryAttemptEvent, + id: (event) => event.branchId, + sameAttempt: sameMemoryAttempt, + }) +} + +function parseMemoryAttemptEvent( + value: unknown, + path: string, + line: number, +): AgentMemoryAttemptEvent { + const event = value as Partial | null + const valid = + typeof event === 'object' && + event !== null && + event.schema === 1 && + (event.status === 'started' || event.status === 'cleaned') && + typeof event.branchId === 'string' && + event.branchId.length > 0 && + typeof event.candidateId === 'string' && + event.candidateId.length > 0 && + typeof event.candidateRef === 'string' && + event.candidateRef.length > 0 && + typeof event.sequenceId === 'string' && + event.sequenceId.length > 0 && + Number.isSafeInteger(event.rep) && + Number.isSafeInteger(event.seed) && + typeof event.cleanupBranches === 'boolean' && + typeof event.recordedAt === 'string' && + !Number.isNaN(Date.parse(event.recordedAt)) && + typeof event.recovery === 'boolean' + if (!valid) throw new Error(`invalid memory attempt event in '${path}' line ${line}`) + return value as AgentMemoryAttemptEvent +} + +function sameMemoryAttempt(left: AgentMemoryAttemptEvent, right: AgentMemoryAttemptEvent): boolean { + return ( + left.branchId === right.branchId && + left.candidateId === right.candidateId && + left.candidateRef === right.candidateRef && + left.sequenceId === right.sequenceId && + left.rep === right.rep && + left.seed === right.seed && + left.cleanupBranches === right.cleanupBranches + ) +} + async function writeStep(memory: AgentMemoryBranch, step: AgentMemorySequenceStep): Promise { const writes = (step.writes ?? []).map((write) => ({ ...write, @@ -657,6 +1216,7 @@ async function probeStep( score: evaluation.score, passed: evaluation.passed, dimensions: evaluation.dimensions, + applicableDimensions: evaluation.applicableDimensions ?? Object.keys(evaluation.dimensions), notes: evaluation.notes, hitIds: context.hits.map((hit) => hit.id), } @@ -673,6 +1233,7 @@ function rankAgentMemoryExperiment( candidates: readonly AgentMemoryExperimentCandidate[], scenarios: readonly AgentMemorySequenceScenario[], campaign: CampaignResult, + recoveryCostByCandidate: ReadonlyMap, ): AgentMemoryExperimentRankingRow[] { const scenarioById = new Map(scenarios.map((scenario) => [scenario.id, scenario])) const rows = candidates.map((candidate): AgentMemoryExperimentRankingRow => { @@ -681,21 +1242,7 @@ function rankAgentMemoryExperiment( (cell) => scenarioById.get(cell.scenarioId)?.candidateId === candidate.id, ) const successful = cells.filter((cell) => !cell.error && cell.artifact) - const dimensionKeys = new Set([ - 'memory_fact_recall', - 'memory_event_recall', - 'memory_actor_recall', - 'memory_stale_safe', - ...successful.flatMap((cell) => Object.keys(cell.artifact.dimensions)), - ]) - const dimensionRows = cells.map((cell) => - Object.fromEntries( - [...dimensionKeys].map((key) => [ - key, - !cell.error && cell.artifact ? (cell.artifact.dimensions[key] ?? 0) : 0, - ]), - ), - ) + const dimensionRows = successful.map((cell) => cell.artifact.dimensions) return { rank: 0, candidateId: candidate.id, @@ -708,7 +1255,9 @@ function rankAgentMemoryExperiment( totalCells: cells.length, totalProbes: successful.reduce((sum, cell) => sum + cell.artifact.probes.length, 0), cellsFailed: cells.filter((cell) => Boolean(cell.error)).length, - totalCostUsd: cells.reduce((sum, cell) => sum + cell.costUsd, 0), + totalCostUsd: + cells.reduce((sum, cell) => sum + cell.costUsd, 0) + + (recoveryCostByCandidate.get(candidate.id) ?? 0), durationMs: cells.reduce((sum, cell) => sum + cell.durationMs, 0), dimensions: meanDimensions(dimensionRows), } @@ -745,6 +1294,7 @@ function memoryExperimentDispatchRef(options: RunAgentMemoryExperimentOptions): return stableId( 'memory_experiment', canonicalJson({ + implementationRef: MEMORY_EXPERIMENT_IMPLEMENTATION_REF, experimentId: options.experimentId, experimentRunId: options.experimentRunId ?? null, executeStepRef: options.executeStepRef ?? 'fixtures', @@ -752,10 +1302,11 @@ function memoryExperimentDispatchRef(options: RunAgentMemoryExperimentOptions): candidates: options.candidates .map((candidate) => ({ id: candidate.id, - ref: candidate.ref ?? candidate.id, + ref: candidate.ref, policy: candidate.policy ?? null, baseScope: candidate.baseScope ?? null, externalCostUsdPerSequence: candidate.externalCostUsdPerSequence ?? 0, + externalRecoveryCostUsdPerAttempt: candidate.externalRecoveryCostUsdPerAttempt ?? 0, })) .sort((a, b) => a.id.localeCompare(b.id)), }), @@ -775,6 +1326,14 @@ function meanDimensions(rows: readonly Record[]): Record [key, mean(bucket)])) } +function countDimensions(rows: readonly (readonly string[])[]): Record { + const counts = new Map() + for (const row of rows) { + for (const key of new Set(row)) counts.set(key, (counts.get(key) ?? 0) + 1) + } + return Object.fromEntries(counts) +} + function mean(values: readonly number[]): number { return values.length === 0 ? 0 : values.reduce((sum, value) => sum + value, 0) / values.length } @@ -787,6 +1346,20 @@ function mergeScopes(base?: AgentMemoryScope, extra?: AgentMemoryScope): AgentMe } } +function memoryExperimentBaseScope( + options: Pick, + candidate: Pick, + sequenceId: string, +): AgentMemoryScope { + return mergeScopes(candidate.baseScope, { + tags: { + memoryExperimentId: options.experimentId, + memoryCandidateId: candidate.id, + memorySequenceId: sequenceId, + }, + }) +} + function sequenceCleanupScopes(sequence: AgentMemorySequence): AgentMemoryScope[] { const scopes = new Map() for (const scope of sequence.cleanupScopes ?? []) { diff --git a/src/memory/graphiti.ts b/src/memory/graphiti.ts index 46131dd..b3f92df 100644 --- a/src/memory/graphiti.ts +++ b/src/memory/graphiti.ts @@ -1,6 +1,8 @@ +import { randomUUID } from 'node:crypto' import { canonicalJson } from '@tangle-network/agent-eval' import { sha256, stableId } from '../ids' import { defaultGetMemoryContext } from './adapter' +import { mergeRankedMemoryHits } from './rank' import { memoryWriteResultToSourceRecord } from './source-record' import type { AgentMemoryAdapter, AgentMemoryHit, AgentMemoryScope } from './types' @@ -38,6 +40,8 @@ interface GraphitiEpisode { group_id?: string } +const DEFAULT_GRAPHITI_INGESTION_TIMEOUT_MS = 120_000 + /** Connects the official Graphiti MCP server without coupling to its Python internals. */ export function createGraphitiMemoryAdapter( options: GraphitiMemoryAdapterOptions, @@ -52,7 +56,11 @@ export function createGraphitiMemoryAdapter( id, branchIsolation: consistency === 'visible' - ? { mode: 'scoped' } + ? { + mode: 'scoped', + processExitSafe: false, + recoveryDelayMs: options.ingestionTimeoutMs ?? DEFAULT_GRAPHITI_INGESTION_TIMEOUT_MS, + } : { mode: 'unsupported', reason: @@ -65,7 +73,8 @@ export function createGraphitiMemoryAdapter( const scope = mergeScopes(options.defaultScope, searchOptions.scope) const groupId = graphitiGroupId(id, scope) const modes = new Set(options.search ?? ['facts', 'nodes']) - const kinds = searchOptions.kinds ? new Set(searchOptions.kinds) : null + const kinds = + searchOptions.kinds && searchOptions.kinds.length > 0 ? new Set(searchOptions.kinds) : null const searchFacts = modes.has('facts') && (!kinds || kinds.has('fact')) const searchNodes = modes.has('nodes') && (!kinds || kinds.has('entity')) const limit = searchOptions.limit ?? 10 @@ -92,23 +101,19 @@ export function createGraphitiMemoryAdapter( }) : undefined, ]) - const hits = [ - ...normalizeGraphitiFacts(factPayload, id), - ...normalizeGraphitiNodes(nodePayload, id), - ] + const hits = mergeRankedMemoryHits( + [normalizeGraphitiFacts(factPayload, id), normalizeGraphitiNodes(nodePayload, id)].map( + (group) => + group.filter((hit) => { + if (searchOptions.minScore === undefined) return true + const score = hit.normalizedScore ?? hit.score + return score !== undefined && score >= searchOptions.minScore + }), + ), + limit, + ) await enrichGraphitiProvenance(options, groupId, hits) return hits - .filter((hit) => { - if (searchOptions.minScore === undefined) return true - const score = hit.normalizedScore ?? hit.score - return score !== undefined && score >= searchOptions.minScore - }) - .sort( - (a, b) => - (b.normalizedScore ?? b.score ?? Number.NEGATIVE_INFINITY) - - (a.normalizedScore ?? a.score ?? Number.NEGATIVE_INFINITY), - ) - .slice(0, limit) }, async getContext(query, searchOptions = {}) { return defaultGetMemoryContext(adapter, query, searchOptions) @@ -126,7 +131,8 @@ export function createGraphitiMemoryAdapter( metadata: input.metadata, }), ) - const episodeUuid = deterministicUuid(`${id}:${groupId}:${eventKey}`) + const episodeUuid = + input.id === undefined ? randomUUID() : deterministicUuid(`${id}:${groupId}:${eventKey}`) const name = input.title ?? `${input.kind}:${input.id ?? stableId('event', eventKey)}` const sourceDescription = canonicalJson( stripUndefined({ @@ -395,7 +401,7 @@ async function waitForGraphitiEpisode( groupId: string, episodeUuid: string, ): Promise { - const timeoutMs = options.ingestionTimeoutMs ?? 120_000 + const timeoutMs = options.ingestionTimeoutMs ?? DEFAULT_GRAPHITI_INGESTION_TIMEOUT_MS const pollIntervalMs = options.pollIntervalMs ?? 500 const maximum = options.episodeScanLimit ?? 10_000 const deadline = Date.now() + timeoutMs diff --git a/src/memory/improvement.ts b/src/memory/improvement.ts index ef9b483..462e5b6 100644 --- a/src/memory/improvement.ts +++ b/src/memory/improvement.ts @@ -1,7 +1,6 @@ import { join } from 'node:path' import { canonicalJson } from '@tangle-network/agent-eval' import { - acquireSingleRunLock, type CampaignStorage, type CostLedgerHandle, campaignLineageStore, @@ -28,10 +27,18 @@ import type { AgentMemoryExperimentCandidate, AgentMemorySequence, AgentMemorySequenceArtifact, + AgentMemorySequenceProbe, RunAgentMemoryExperimentOptions, RunAgentMemoryExperimentResult, } from './experiment' import { runAgentMemoryExperiment } from './experiment' +import { + type AgentMemoryAcquireRunLease, + type AgentMemoryControllerMode, + type AgentMemoryRunLease, + acquireAgentMemoryRunLease, + type OwnedAgentMemoryRunLease, +} from './run-control' export interface AgentMemoryImprovementSeed { config: TConfig @@ -43,6 +50,7 @@ export interface AgentMemoryImprovementSeed { export interface AgentMemoryDimensionComparison { dimension: string n: number + expectedN: number measured: boolean meanDelta: number low: number @@ -67,10 +75,7 @@ export interface AgentMemoryActivation { receiptPath: string } -export interface AgentMemoryImprovementRunLease { - assertOwned(): void | Promise - release(): void | Promise -} +export type AgentMemoryImprovementRunLease = AgentMemoryRunLease export interface AgentMemoryGovernor { decide( @@ -108,15 +113,15 @@ export interface RunAgentMemoryImprovementOptions { repo?: string storage?: CampaignStorage lineageStore?: LineageStore + /** Required with custom storage when all controllers are confined to one process. */ + controllerMode?: AgentMemoryControllerMode /** Required for distributed controllers using custom storage. Worker concurrency is independent. */ - acquireRunLease?: (input: { - experimentId: string - runDir: string - }) => AgentMemoryImprovementRunLease | Promise + acquireRunLease?: AgentMemoryAcquireRunLease seed?: number reps?: number resumable?: boolean dispatchTimeoutMs?: number + cleanupTimeoutMs?: number maxTotalCostUsd?: number executeStep?: RunAgentMemoryExperimentOptions['executeStep'] executeStepRef?: string @@ -201,7 +206,16 @@ export async function runAgentMemoryImprovement( const storage = options.storage ?? fsCampaignStorage() const runDir = resolveRunDir(options.runDir, options.repo) storage.ensureDir(runDir) - const lease = await acquireMemoryImprovementRunLease(options, storage, runDir) + const lease = await acquireAgentMemoryRunLease({ + experimentId: options.experimentId, + runDir, + storage, + customStorage: options.storage !== undefined, + lockFileName: 'memory-improvement.lock', + label: 'memory improvement', + controllerMode: options.controllerMode, + acquireRunLease: options.acquireRunLease, + }) let result: RunAgentMemoryImprovementResult | undefined let primaryError: unknown try { @@ -267,19 +281,23 @@ async function runAgentMemoryImprovementOwned( if (!pending) { pending = (async () => { const candidate = await buildCandidate(options, config, hash, `search-${hash}`) - return runAgentMemoryExperiment({ - ...experimentOptions(options, costLedger, storage), + await lease.assertOwned() + const experiment = await runAgentMemoryExperiment({ + ...experimentOptions(options, costLedger, storage, lease), experimentId: `${options.experimentId}:search:${hash}`, sequences: options.trainSequences, candidates: [candidate], runDir: join(runDir, 'search', hash), costPhase: `memory.search.${hash}`, }) + await lease.assertOwned() + return experiment })() evaluations.set(hash, pending) void pending.catch(() => evaluations.delete(hash)) } const result = await pending + await lease.assertOwned() const row = result.rows[0] if (!row || row.cellsFailed > 0) { throw new Error(`${hash}: memory candidate did not complete every training cell`) @@ -303,17 +321,19 @@ async function runAgentMemoryImprovementOwned( ...(seed.vision !== undefined ? { vision: seed.vision } : {}), })), scenarios: trainScenarios, - proposer: withCostContext(options.proposer, costLedger, 'default'), + proposer: withCostContext(options.proposer, costLedger, lease, 'default'), proposers: options.proposers ? Object.fromEntries( Object.entries(options.proposers).map(([name, proposer]) => [ name, - withCostContext(proposer, costLedger, name), + withCostContext(proposer, costLedger, lease, name), ]), ) : undefined, scoreSurface: evaluateSurface, - governor: options.governor ? withGovernorCostContext(options.governor, costLedger) : undefined, + governor: options.governor + ? withGovernorCostContext(options.governor, costLedger, lease) + : undefined, budget: { ...options.budget, maxNodes: options.seeds.length + options.budget.maxSteps, @@ -352,8 +372,9 @@ async function runAgentMemoryImprovementOwned( buildCandidate(options, baselineConfig, baselineHash, 'baseline'), buildCandidate(options, winnerConfig, winnerHash, 'winner'), ]) + await lease.assertOwned() holdout = await runAgentMemoryExperiment({ - ...experimentOptions(options, costLedger, storage), + ...experimentOptions(options, costLedger, storage, lease), experimentId: `${options.experimentId}:holdout`, sequences: options.holdoutSequences, candidates: [baselineCandidate, winnerCandidate], @@ -447,16 +468,20 @@ async function runAgentMemoryImprovementOwned( function withCostContext( proposer: SurfaceProposer, costLedger: CostLedgerHandle, + lease: OwnedRunLease, label: string, ): SurfaceProposer { return { kind: proposer.kind, async propose(context) { - return proposer.propose({ + await lease.assertOwned() + const proposal = await proposer.propose({ ...context, costLedger, costPhase: `memory.proposal.${context.track?.id ?? label}`, }) + await lease.assertOwned() + return proposal }, ...(proposer.decide ? { decide: (input) => proposer.decide!(input) } : {}), } @@ -465,14 +490,18 @@ function withCostContext( function withGovernorCostContext( governor: AgentMemoryGovernor, costLedger: CostLedgerHandle, + lease: OwnedRunLease, ): Governor { return { - decide(context) { - return governor.decide({ + async decide(context) { + await lease.assertOwned() + const decision = await governor.decide({ ...context, costLedger, costPhase: 'memory.governor', }) + await lease.assertOwned() + return decision }, } } @@ -491,7 +520,7 @@ async function buildCandidate( return { ...built, id: `${role}-${hash}`, - ref: built.ref ?? hash, + ref: built.ref, } } @@ -499,6 +528,7 @@ function experimentOptions( options: RunAgentMemoryImprovementOptions, costLedger: ReturnType, storage: CampaignStorage, + lease: OwnedRunLease, ): Pick< RunAgentMemoryExperimentOptions, | 'storage' @@ -507,11 +537,13 @@ function experimentOptions( | 'resumable' | 'maxConcurrency' | 'dispatchTimeoutMs' + | 'cleanupTimeoutMs' | 'executeStep' | 'executeStepRef' | 'onBranchSnapshot' | 'cleanupBranches' | 'costLedger' + | 'acquireRunLease' | 'now' > { return { @@ -521,21 +553,21 @@ function experimentOptions( resumable: options.resumable, maxConcurrency: options.sequenceConcurrency, dispatchTimeoutMs: options.dispatchTimeoutMs, + cleanupTimeoutMs: options.cleanupTimeoutMs, executeStep: options.executeStep, executeStepRef: options.executeStepRef, onBranchSnapshot: options.onBranchSnapshot, cleanupBranches: options.cleanupBranches ?? true, costLedger, + acquireRunLease: async () => ({ + assertOwned: () => lease.assertOwned(), + release() {}, + }), now: options.now, } } -interface OwnedRunLease { - assertOwned(): Promise - release(): Promise -} - -const activeCustomStorageRuns = new WeakMap>() +type OwnedRunLease = OwnedAgentMemoryRunLease function assertMemoryImprovementIdentity( options: RunAgentMemoryImprovementOptions, @@ -545,7 +577,7 @@ function assertMemoryImprovementIdentity( ): void { const path = join(runDir, 'memory-improvement-manifest.json') const identity = { - schema: 3, + schema: 4, experimentId: options.experimentId, improvementRef: options.improvementRef, proposerKind: options.proposer.kind, @@ -601,67 +633,6 @@ function assertMemoryImprovementIdentity( } } -async function acquireMemoryImprovementRunLease( - options: RunAgentMemoryImprovementOptions, - storage: CampaignStorage, - runDir: string, -): Promise { - if (options.acquireRunLease) { - const lease = await options.acquireRunLease({ - experimentId: options.experimentId, - runDir, - }) - if (!lease || typeof lease.release !== 'function') { - throw new Error('memory improvement acquireRunLease must return a releasable lease') - } - if (typeof lease.assertOwned !== 'function') { - throw new Error('memory improvement acquireRunLease must return assertOwned') - } - return { - async assertOwned() { - await lease.assertOwned() - }, - async release() { - await lease.release() - }, - } - } - - if (options.storage === undefined) { - const lock = acquireSingleRunLock({ - lockPath: join(runDir, 'memory-improvement.lock'), - releaseOnExit: false, - }) - return { - async assertOwned() {}, - async release() { - lock.release() - }, - } - } - - const active = activeCustomStorageRuns.get(storage) ?? new Set() - if (active.has(runDir)) { - throw new Error(`memory improvement run '${runDir}' already has an active controller`) - } - active.add(runDir) - activeCustomStorageRuns.set(storage, active) - let released = false - return { - async assertOwned() { - if (released || !active.has(runDir)) { - throw new Error(`memory improvement run '${runDir}' controller lease was lost`) - } - }, - async release() { - if (released) return - released = true - active.delete(runDir) - if (active.size === 0) activeCustomStorageRuns.delete(storage) - }, - } -} - function hasCompletedActivation( storage: CampaignStorage, path: string, @@ -729,14 +700,14 @@ function decidePromotion(input: { const paired = pairedArtifacts(result, baselineId, winnerId, (artifact) => artifact.score) const significance = heldoutSignificance(paired, options.significance) const tolerance = options.criticalDimensionTolerance ?? 0.05 - const expectedPairCount = options.holdoutSequences.length * (options.reps ?? 1) const criticalDimensions = (options.criticalDimensions ?? DEFAULT_CRITICAL_DIMENSIONS).map( (dimension) => { - const dimensionPairs = pairedArtifacts( - result, - baselineId, - winnerId, - (artifact) => artifact.dimensions[dimension], + const expectedN = + applicableSequenceCount(options.holdoutSequences, dimension) * (options.reps ?? 1) + const dimensionPairs = pairedArtifacts(result, baselineId, winnerId, (artifact) => + (artifact.dimensionSampleCounts?.[dimension] ?? 0) > 0 + ? artifact.dimensions[dimension] + : undefined, ) const comparison = heldoutSignificance(dimensionPairs, { ...options.significance, @@ -745,12 +716,14 @@ function decidePromotion(input: { return { dimension, n: comparison.n, - measured: comparison.n === expectedPairCount, + expectedN, + measured: expectedN > 0 && comparison.n === expectedN, meanDelta: comparison.bootstrap.mean, low: comparison.bootstrap.low, high: comparison.bootstrap.high, tolerance, - regressed: comparison.bootstrap.low < -tolerance, + regressed: + expectedN > 0 && comparison.n === expectedN && comparison.bootstrap.low < -tolerance, } }, ) @@ -771,7 +744,9 @@ function decidePromotion(input: { for (const dimension of criticalDimensions) { if (!dimension.measured) { reasons.push( - `critical dimension ${dimension.dimension} was measured on ${dimension.n}/${expectedPairCount} paired holdout cells`, + dimension.expectedN === 0 + ? `critical dimension ${dimension.dimension} has no applicable holdout histories` + : `critical dimension ${dimension.dimension} was measured on ${dimension.n}/${dimension.expectedN} applicable paired holdout cells`, ) } else if (dimension.regressed) { reasons.push(`${dimension.dimension} may regress beyond ${tolerance}`) @@ -788,6 +763,39 @@ function decidePromotion(input: { } } +function applicableSequenceCount( + sequences: readonly AgentMemorySequence[], + dimension: string, +): number { + return sequences.filter((sequence) => + sequence.steps.some((step) => + (step.probes ?? []).some((probe) => probeAppliesToDimension(probe, dimension)), + ), + ).length +} + +function probeAppliesToDimension(probe: AgentMemorySequenceProbe, dimension: string): boolean { + switch (dimension) { + case 'memory_fact_recall': + case 'memory_required_fact_count': + case 'memory_matched_fact_count': + return Boolean( + (probe.requiredFacts && probe.requiredFacts.length > 0) || probe.referenceAnswer, + ) + case 'memory_event_recall': + return Boolean(probe.expectedEventIds && probe.expectedEventIds.length > 0) + case 'memory_actor_recall': + return Boolean(probe.expectedActorIds && probe.expectedActorIds.length > 0) + case 'memory_stale_safe': + case 'memory_stale_rate': + case 'memory_forbidden_fact_count': + case 'memory_matched_forbidden_fact_count': + return Boolean(probe.forbiddenFacts && probe.forbiddenFacts.length > 0) + default: + return true + } +} + function normalizedPromotionPolicy( options: RunAgentMemoryImprovementOptions, ): Record { diff --git a/src/memory/index.ts b/src/memory/index.ts index 92b698c..7bef15c 100644 --- a/src/memory/index.ts +++ b/src/memory/index.ts @@ -4,8 +4,10 @@ export * from './experiment' export * from './graphiti' export * from './holdout' export * from './improvement' +export * from './lifecycle' export * from './mem0' export * from './neo4j' +export * from './run-control' export * from './schemas' export * from './source-record' export * from './types' diff --git a/src/memory/lifecycle.ts b/src/memory/lifecycle.ts new file mode 100644 index 0000000..9195151 --- /dev/null +++ b/src/memory/lifecycle.ts @@ -0,0 +1,81 @@ +import type { AgentMemoryAdapter } from './types' + +export const DEFAULT_MEMORY_CLEANUP_TIMEOUT_MS = 180_000 + +export class AgentMemoryLifecycleTimeoutError extends Error { + constructor( + readonly operation: string, + readonly timeoutMs: number, + ) { + super(`${operation} did not finish within ${timeoutMs}ms`) + this.name = 'AgentMemoryLifecycleTimeoutError' + } +} + +export function resolveMemoryCleanupTimeoutMs(value: number | undefined, label: string): number { + const timeoutMs = value ?? DEFAULT_MEMORY_CLEANUP_TIMEOUT_MS + if (!Number.isSafeInteger(timeoutMs) || timeoutMs <= 0) { + throw new Error(`${label} cleanupTimeoutMs must be a positive safe integer`) + } + return timeoutMs +} + +export async function runBoundedMemoryLifecycle(input: { + operation: string + timeoutMs: number + run(): Promise | T +}): Promise { + let timeout: ReturnType | undefined + const work = Promise.resolve().then(input.run) + const deadline = new Promise((_, reject) => { + timeout = setTimeout( + () => reject(new AgentMemoryLifecycleTimeoutError(input.operation, input.timeoutMs)), + input.timeoutMs, + ) + }) + try { + return await Promise.race([work, deadline]) + } finally { + if (timeout) clearTimeout(timeout) + } +} + +export function memoryRecoveryDelayMs(adapter: AgentMemoryAdapter): number { + const isolation = adapter.branchIsolation + if (isolation?.mode !== 'scoped' || isolation.processExitSafe !== false) return 0 + const delayMs = isolation.recoveryDelayMs + if (typeof delayMs !== 'number' || !Number.isSafeInteger(delayMs) || delayMs <= 0) { + throw new Error( + `${adapter.id}: processExitSafe=false requires a positive recoveryDelayMs for abandoned writes`, + ) + } + return delayMs +} + +export async function sleepForMemoryRecovery( + delayMs: number, + assertOwned: () => Promise, +): Promise { + if (delayMs <= 0) return + await new Promise((resolve) => setTimeout(resolve, delayMs)) + await assertOwned() +} + +export function createMemoryExecutionPool(limit: number): { + run(operation: () => Promise): Promise +} { + let active = 0 + const waiters: Array<() => void> = [] + return { + async run(operation: () => Promise): Promise { + if (active >= limit) await new Promise((resolve) => waiters.push(resolve)) + active += 1 + try { + return await operation() + } finally { + active -= 1 + waiters.shift()?.() + } + }, + } +} diff --git a/src/memory/mem0.ts b/src/memory/mem0.ts index 257f401..ca0f0f2 100644 --- a/src/memory/mem0.ts +++ b/src/memory/mem0.ts @@ -1,3 +1,4 @@ +import { randomUUID } from 'node:crypto' import { canonicalJson } from '@tangle-network/agent-eval' import { stableId } from '../ids' import { defaultGetMemoryContext } from './adapter' @@ -12,6 +13,7 @@ import type { } from './types' export type Mem0ClientMode = 'hosted' | 'oss' +export type Mem0Consistency = 'queued' | 'visible' export interface Mem0ClientLike { add( @@ -32,6 +34,11 @@ export interface Mem0MemoryAdapterOptions { infer?: boolean rerank?: boolean latestOnly?: boolean + consistency?: Mem0Consistency + /** Reads `GET /v1/event/{eventId}/`; required when a hosted add is asynchronous. */ + getEvent?: (eventId: string) => Promise + ingestionTimeoutMs?: number + pollIntervalMs?: number defaultScope?: AgentMemoryScope } @@ -39,32 +46,59 @@ export interface Mem0MemoryAdapterOptions { export function createMem0MemoryAdapter(options: Mem0MemoryAdapterOptions): AgentMemoryAdapter { assertMem0Options(options) const id = options.id ?? `mem0-${options.mode}` + const consistency = options.mode === 'oss' ? 'visible' : (options.consistency ?? 'visible') + const pendingWrites = new Map() const adapter: AgentMemoryAdapter = { id, - branchIsolation: { mode: 'scoped' }, + branchIsolation: + options.mode === 'oss' + ? { mode: 'scoped' } + : consistency === 'visible' + ? { + mode: 'scoped', + processExitSafe: false, + recoveryDelayMs: options.ingestionTimeoutMs ?? 30_000, + } + : { + mode: 'unsupported', + reason: + 'queued hosted Mem0 writes can become visible after branch cleanup; use consistency="visible" with an attempt branch', + }, async search(query, searchOptions = {}) { assertMem0SearchOptions(searchOptions, id) if (searchOptions.limit === 0) return [] const scope = mergeScopes(options.defaultScope, searchOptions.scope) - const raw = await options.client.search(query, { - filters: mem0Filters(scope, options.appId), - topK: searchOptions.limit, - threshold: searchOptions.minScore, - ...(options.rerank !== undefined ? { rerank: options.rerank } : {}), - ...(options.mode === 'hosted' && options.latestOnly !== undefined - ? { latestOnly: options.latestOnly } - : {}), - }) - return normalizeMem0Hits(raw, id, searchOptions) + const requestedKinds = [...new Set(searchOptions.kinds ?? [])] + const kindQueries = requestedKinds.length > 0 ? requestedKinds : [undefined] + const payloads = await Promise.all( + kindQueries.map((kind) => + options.client.search(query, { + filters: mem0Filters(scope, options.appId, kind), + topK: searchOptions.limit, + threshold: searchOptions.minScore, + ...(options.rerank !== undefined ? { rerank: options.rerank } : {}), + ...(options.mode === 'hosted' && options.latestOnly !== undefined + ? { latestOnly: options.latestOnly } + : {}), + }), + ), + ) + return mergeMem0Hits( + payloads.flatMap((raw) => normalizeMem0Hits(raw, id, searchOptions)), + searchOptions.limit, + ) }, async getContext(query, searchOptions = {}) { return defaultGetMemoryContext(adapter, query, searchOptions) }, async write(input) { const scope = mergeScopes(options.defaultScope, input.scope) + const writeId = input.id ?? randomUUID() const scopeMetadata = mem0ScopeMetadata(scope, options.appId) const metadata = compactRecord({ + ...input.metadata, + agentKnowledgeWriteId: writeId, memoryKind: input.kind, memoryTitle: input.title, entityName: input.entityName, @@ -75,7 +109,6 @@ export function createMem0MemoryAdapter(options: Mem0MemoryAdapterOptions): Agen object: input.object, confidence: input.confidence, originalRole: input.role, - ...input.metadata, ...scopeMetadata, }) const entity = mem0Entity(scope, options.appId) @@ -85,28 +118,36 @@ export function createMem0MemoryAdapter(options: Mem0MemoryAdapterOptions): Agen infer: options.infer ?? true, ...(options.mode === 'oss' ? { filters: mem0Filters(scope, options.appId) } : {}), }) - const items = extractMem0Items(raw) + let items = extractMem0Items(raw) + const pending = options.mode === 'hosted' && mem0ResponsePending(raw) + const eventId = stringField(isRecord(raw) ? raw : undefined, ['eventId', 'event_id']) + if (pending) { + if (!eventId) throw new Error(`${id}: asynchronous Mem0 add returned no event_id`) + pendingWrites.set(eventId, cloneScope(scope)) + if (consistency === 'visible') { + const terminal = await waitForMem0Event(options, eventId) + pendingWrites.delete(eventId) + assertMem0EventSucceeded(terminal, id, eventId) + items = extractMem0Items(terminal) + } + } else if (options.mode === 'hosted' && isRecord(raw) && mem0Status(raw) === 'FAILED') { + assertMem0EventSucceeded(raw, id, eventId ?? 'unknown') + } const mutations = items.filter((candidate) => mem0Event(candidate) !== 'NOOP') const item = mutations[0] ?? items[0] - const itemId = - stringField(item, ['id']) ?? - input.id ?? - stableId( - 'mem', - canonicalJson( - compactRecord({ adapterId: id, kind: input.kind, text: input.text, scope }), - ), - ) + const itemId = stringField(item, ['id']) ?? eventId ?? writeId const event = mem0Event(item) const events = [...new Set(mutations.map(mem0Event).filter(Boolean))] const result = { - accepted: mutations.length > 0, + accepted: pending && consistency === 'queued' ? true : mutations.length > 0, id: itemId, uri: `memory://${id}/${encodeURIComponent(itemId)}`, kind: input.kind, metadata: { provider: 'mem0', mode: options.mode, + consistency, + queued: pending && consistency === 'queued', ...(event ? { event } : {}), ...(events.length > 1 ? { events } : {}), }, @@ -118,9 +159,13 @@ export function createMem0MemoryAdapter(options: Mem0MemoryAdapterOptions): Agen }, async clear(scope) { const mergedScope = mergeScopes(options.defaultScope, scope) + await settlePendingMem0Writes(options, pendingWrites, mergedScope, false) if (options.client.getAll && options.client.delete) { - let deleted = 0 - for (let batch = 0; batch < 100; batch += 1) { + const deletedIds = new Set() + const visibilityTimeoutMs = options.ingestionTimeoutMs ?? 30_000 + const pollIntervalMs = options.pollIntervalMs ?? 250 + let visibilityDeadline = Date.now() + visibilityTimeoutMs + for (let batch = 0; batch < 100; ) { const raw = await options.client.getAll( options.mode === 'hosted' ? { @@ -143,17 +188,69 @@ export function createMem0MemoryAdapter(options: Mem0MemoryAdapterOptions): Agen throw new Error(`${id}: Mem0 returned memories without ids during scoped clear`) } const ids = [...new Set(rawIds)] - for (const memoryId of ids) await options.client.delete(memoryId) - deleted += ids.length + const unseenIds = ids.filter((memoryId) => !deletedIds.has(memoryId)) + if (unseenIds.length === 0) { + const remainingMs = visibilityDeadline - Date.now() + if (remainingMs <= 0) { + throw new Error( + `${id}: deleted Mem0 memories remained visible after ${visibilityTimeoutMs}ms`, + ) + } + await sleep(Math.min(pollIntervalMs, remainingMs)) + continue + } + for (const memoryId of unseenIds) { + await options.client.delete(memoryId) + deletedIds.add(memoryId) + } + batch += 1 + visibilityDeadline = Date.now() + visibilityTimeoutMs } - throw new Error(`${id}: refused to clear more than ${deleted} Mem0 memories in one call`) + throw new Error( + `${id}: refused to clear more than ${deletedIds.size} Mem0 memories in one call`, + ) } - if (!options.client.deleteAll || !canDeleteAllExactly(mergedScope)) { + if ( + !options.client.deleteAll || + !options.client.getAll || + !canDeleteAllExactly(mergedScope) + ) { throw new Error( - `${id}: exact scoped clear requires Mem0 getAll and delete for tenant, team, session, tag, or nested run scopes`, + `${id}: exact scoped clear requires Mem0 getAll plus delete or an exact deleteAll`, ) } await options.client.deleteAll(mem0Entity(mergedScope, options.appId)) + const visibilityTimeoutMs = options.ingestionTimeoutMs ?? 30_000 + const pollIntervalMs = options.pollIntervalMs ?? 250 + const visibilityDeadline = Date.now() + visibilityTimeoutMs + for (;;) { + const raw = await options.client.getAll( + options.mode === 'hosted' + ? { + filters: mem0Filters(mergedScope, options.appId), + page: 1, + pageSize: 1, + latestOnly: false, + showExpired: true, + } + : { + filters: mem0Filters(mergedScope, options.appId), + topK: 1, + showExpired: true, + }, + ) + if (extractMem0Items(raw).length === 0) return + const remainingMs = visibilityDeadline - Date.now() + if (remainingMs <= 0) { + throw new Error( + `${id}: bulk-deleted Mem0 memories remained visible after ${visibilityTimeoutMs}ms`, + ) + } + await sleep(Math.min(pollIntervalMs, remainingMs)) + } + }, + async flush() { + await settlePendingMem0Writes(options, pendingWrites, undefined, true) }, } return adapter @@ -166,6 +263,17 @@ function assertMem0Options(options: Mem0MemoryAdapterOptions): void { if (options.appId !== undefined && (typeof options.appId !== 'string' || !options.appId.trim())) { throw new Error('Mem0 appId must be a non-empty string') } + if (options.mode === 'oss' && options.consistency === 'queued') { + throw new Error('Mem0 OSS writes are synchronous and do not support consistency="queued"') + } + for (const [name, value] of [ + ['ingestionTimeoutMs', options.ingestionTimeoutMs], + ['pollIntervalMs', options.pollIntervalMs], + ] as const) { + if (value !== undefined && (!Number.isSafeInteger(value) || value <= 0)) { + throw new Error(`Mem0 ${name} must be a positive safe integer`) + } + } } function assertMem0SearchOptions(options: AgentMemorySearchOptions, adapterId: string): void { @@ -183,7 +291,7 @@ function normalizeMem0Hits( options: AgentMemorySearchOptions, ): AgentMemoryHit[] { const limit = options.limit ?? Number.POSITIVE_INFINITY - const kinds = options.kinds ? new Set(options.kinds) : null + const kinds = options.kinds && options.kinds.length > 0 ? new Set(options.kinds) : null return extractMem0Items(raw) .map((item): AgentMemoryHit | undefined => { const text = @@ -230,6 +338,33 @@ function normalizeMem0Hits( .slice(0, limit) } +function mergeMem0Hits(hits: readonly AgentMemoryHit[], limit?: number): AgentMemoryHit[] { + const unique = new Map() + for (const [index, hit] of hits.entries()) { + const key = `${hit.uri}\n${hit.text}` + const score = hit.normalizedScore ?? hit.score + const finiteScore = score !== undefined && Number.isFinite(score) ? score : undefined + const prior = unique.get(key) + if (!prior) { + unique.set(key, { hit, index, score: finiteScore }) + } else if ( + finiteScore !== undefined && + (prior.score === undefined || finiteScore > prior.score) + ) { + unique.set(key, { hit, index: prior.index, score: finiteScore }) + } + } + return [...unique.values()] + .sort( + (left, right) => + Number(right.score !== undefined) - Number(left.score !== undefined) || + (right.score ?? 0) - (left.score ?? 0) || + left.index - right.index, + ) + .map(({ hit }) => hit) + .slice(0, limit ?? Number.POSITIVE_INFINITY) +} + function extractMem0Items(raw: unknown): Array> { if (Array.isArray(raw)) return raw.filter(isRecord) if (!isRecord(raw)) return [] @@ -243,6 +378,87 @@ function mem0Event(item: Record | undefined): string | undefine return stringField(item, ['event'])?.toUpperCase() } +function mem0ResponsePending(raw: unknown): boolean { + if (!isRecord(raw)) return false + const status = stringField(raw, ['status'])?.toUpperCase() + return status === 'PENDING' || status === 'RUNNING' || status === 'QUEUED' +} + +async function waitForMem0Event( + options: Mem0MemoryAdapterOptions, + eventId: string, +): Promise> { + if (!options.getEvent) { + throw new Error( + `${options.id ?? 'mem0-hosted'}: hosted asynchronous writes require getEvent(eventId) for GET /v1/event/{eventId}/`, + ) + } + const timeoutMs = options.ingestionTimeoutMs ?? 30_000 + const pollIntervalMs = options.pollIntervalMs ?? 250 + const deadline = Date.now() + timeoutMs + for (;;) { + const raw = await options.getEvent(eventId) + if (!isRecord(raw)) throw new Error(`${options.id ?? 'mem0-hosted'}: invalid event response`) + const status = mem0Status(raw) + if (status === 'SUCCEEDED' || status === 'FAILED') return raw + if (status !== 'PENDING' && status !== 'RUNNING' && status !== 'QUEUED') { + throw new Error( + `${options.id ?? 'mem0-hosted'}: event ${eventId} returned unsupported status ${status ?? 'missing'}`, + ) + } + const remaining = deadline - Date.now() + if (remaining <= 0) { + throw new Error( + `${options.id ?? 'mem0-hosted'}: event ${eventId} did not finish within ${timeoutMs}ms`, + ) + } + await sleep(Math.min(pollIntervalMs, remaining)) + } +} + +function assertMem0EventSucceeded( + event: Record, + adapterId: string, + eventId: string, +): void { + if (mem0Status(event) === 'SUCCEEDED') return + const detail = + stringField(event, ['message', 'error', 'errorMessage', 'error_message']) ?? + stringField(recordField(event, 'metadata'), ['message', 'error']) + throw new Error(`${adapterId}: Mem0 event ${eventId} failed${detail ? `: ${detail}` : ''}`) +} + +function mem0Status(raw: Record): string | undefined { + return stringField(raw, ['status'])?.toUpperCase() +} + +async function settlePendingMem0Writes( + options: Mem0MemoryAdapterOptions, + pendingWrites: Map, + requestedScope?: AgentMemoryScope, + failOnProviderFailure = true, +): Promise { + const pending = [...pendingWrites].filter( + ([, scope]) => !requestedScope || scopeMatches(scope, requestedScope), + ) + const settled = await Promise.allSettled( + pending.map(async ([eventId]) => { + const terminal = await waitForMem0Event(options, eventId) + pendingWrites.delete(eventId) + if (failOnProviderFailure) { + assertMem0EventSucceeded(terminal, options.id ?? 'mem0-hosted', eventId) + } + }), + ) + const failures = settled.flatMap((result) => + result.status === 'rejected' ? [result.reason] : [], + ) + if (failures.length === 1) throw failures[0] + if (failures.length > 1) { + throw new AggregateError(failures, `${options.id ?? 'mem0-hosted'}: pending writes failed`) + } +} + function mem0Entity(scope: AgentMemoryScope, appId?: string): Record { return compactStringRecord({ userId: scope.userId, @@ -252,17 +468,35 @@ function mem0Entity(scope: AgentMemoryScope, appId?: string): Record { - return mem0ScopeMetadata(scope, appId) +function mem0Filters( + scope: AgentMemoryScope, + appId: string | undefined, + kind?: AgentMemoryKind, +): Record { + const entity = mem0EntityFilters(scope, appId) + const customMetadata = compactRecord({ + ...mem0CustomScopeMetadata(scope), + memoryKind: kind, + }) + return compactRecord({ ...entity, ...customMetadata }) } function mem0ScopeMetadata(scope: AgentMemoryScope, appId?: string): Record { - const metadata = compactStringRecord({ + return { ...mem0EntityFilters(scope, appId), ...mem0CustomScopeMetadata(scope) } +} + +function mem0EntityFilters(scope: AgentMemoryScope, appId?: string): Record { + return compactStringRecord({ user_id: scope.userId, agent_id: scope.agentId, run_id: scope.namespace ?? scope.runId, - logical_run_id: scope.namespace ? scope.runId : undefined, app_id: appId, + }) +} + +function mem0CustomScopeMetadata(scope: AgentMemoryScope): Record { + const metadata = compactStringRecord({ + logical_run_id: scope.namespace ? scope.runId : undefined, tenant_id: scope.tenantId, team_id: scope.teamId, session_id: scope.sessionId, @@ -309,6 +543,32 @@ function mergeScopes(base?: AgentMemoryScope, extra?: AgentMemoryScope): AgentMe } } +function cloneScope(scope: AgentMemoryScope): AgentMemoryScope { + return { ...scope, tags: { ...(scope.tags ?? {}) } } +} + +function scopeMatches(actual: AgentMemoryScope, requested: AgentMemoryScope): boolean { + for (const key of [ + 'tenantId', + 'userId', + 'agentId', + 'teamId', + 'runId', + 'sessionId', + 'namespace', + ] as const) { + if (requested[key] !== undefined && actual[key] !== requested[key]) return false + } + for (const [key, value] of Object.entries(requested.tags ?? {})) { + if (actual.tags?.[key] !== value) return false + } + return true +} + +function sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)) +} + function compactRecord(input: Record): Record { return Object.fromEntries(Object.entries(input).filter(([, value]) => value !== undefined)) } @@ -369,7 +629,16 @@ function dateField( export function mem0MemoryAdapterIdentity( options: Pick< Mem0MemoryAdapterOptions, - 'mode' | 'id' | 'appId' | 'infer' | 'rerank' | 'latestOnly' | 'defaultScope' + | 'mode' + | 'id' + | 'appId' + | 'infer' + | 'rerank' + | 'latestOnly' + | 'consistency' + | 'ingestionTimeoutMs' + | 'pollIntervalMs' + | 'defaultScope' >, ): string { return stableId('mem0', canonicalJson(compactRecord(options))) diff --git a/src/memory/neo4j.ts b/src/memory/neo4j.ts index 619e82a..0d90166 100644 --- a/src/memory/neo4j.ts +++ b/src/memory/neo4j.ts @@ -2,6 +2,7 @@ import { canonicalJson } from '@tangle-network/agent-eval' import { stableId } from '../ids' import { defaultGetMemoryContext } from './adapter' import { emitRetrievalHoldoutBypass } from './holdout' +import { mergeRankedMemoryHits } from './rank' import { memoryHitToSourceRecord, memoryWriteResultToSourceRecord } from './source-record' import type { AgentMemoryAdapter, @@ -358,10 +359,7 @@ async function searchNeo4jMemory( ) } - const hits = (await Promise.all(searches)).flat() - return hits - .sort((a, b) => (b.normalizedScore ?? b.score ?? 0) - (a.normalizedScore ?? a.score ?? 0)) - .slice(0, limit) + return mergeRankedMemoryHits(await Promise.all(searches), limit) } // Mirrors defaultGetMemoryContext's holdout call context so bypass events join the same log stream. diff --git a/src/memory/rank.ts b/src/memory/rank.ts new file mode 100644 index 0000000..d1c6859 --- /dev/null +++ b/src/memory/rank.ts @@ -0,0 +1,62 @@ +import type { AgentMemoryHit } from './types' + +/** Merges independently ranked memory lists without letting the first unscored list dominate. */ +export function mergeRankedMemoryHits( + groups: readonly (readonly AgentMemoryHit[])[], + limit?: number, +): AgentMemoryHit[] { + const byIdentity = new Map< + string, + { + hit: AgentMemoryHit + score?: number + reciprocalRank: number + bestRank: number + firstGroup: number + encounter: number + } + >() + let encounter = 0 + for (const [groupIndex, group] of groups.entries()) { + for (const [rank, hit] of group.entries()) { + const key = `${hit.uri}\n${hit.text}` + const score = memoryHitScore(hit) + const prior = byIdentity.get(key) + if (!prior) { + byIdentity.set(key, { + hit, + score, + reciprocalRank: 1 / (60 + rank + 1), + bestRank: rank, + firstGroup: groupIndex, + encounter, + }) + } else { + prior.reciprocalRank += 1 / (60 + rank + 1) + prior.bestRank = Math.min(prior.bestRank, rank) + if (score !== undefined && (prior.score === undefined || score > prior.score)) { + prior.hit = hit + prior.score = score + } + } + encounter += 1 + } + } + return [...byIdentity.values()] + .sort( + (left, right) => + right.reciprocalRank - left.reciprocalRank || + Number(right.score !== undefined) - Number(left.score !== undefined) || + (right.score ?? 0) - (left.score ?? 0) || + left.bestRank - right.bestRank || + left.firstGroup - right.firstGroup || + left.encounter - right.encounter, + ) + .map(({ hit }) => hit) + .slice(0, limit ?? Number.POSITIVE_INFINITY) +} + +function memoryHitScore(hit: AgentMemoryHit): number | undefined { + const score = hit.normalizedScore ?? hit.score + return score !== undefined && Number.isFinite(score) ? score : undefined +} diff --git a/src/memory/run-control.ts b/src/memory/run-control.ts new file mode 100644 index 0000000..bf78273 --- /dev/null +++ b/src/memory/run-control.ts @@ -0,0 +1,93 @@ +import { join } from 'node:path' +import { acquireSingleRunLock, type CampaignStorage } from '@tangle-network/agent-eval/campaign' + +export interface AgentMemoryRunLease { + assertOwned(): Promise | void + release(): Promise | void +} + +export type AgentMemoryAcquireRunLease = (input: { + experimentId: string + runDir: string +}) => AgentMemoryRunLease | Promise + +export type AgentMemoryControllerMode = 'process-local' + +export interface OwnedAgentMemoryRunLease { + assertOwned(): Promise + release(): Promise +} + +const activeProcessLocalRuns = new Set() + +export async function acquireAgentMemoryRunLease(input: { + experimentId: string + runDir: string + storage: CampaignStorage + customStorage: boolean + lockFileName: string + label: string + controllerMode?: AgentMemoryControllerMode + acquireRunLease?: AgentMemoryAcquireRunLease +}): Promise { + if (input.acquireRunLease && input.controllerMode) { + throw new Error(`${input.label} cannot combine acquireRunLease with controllerMode`) + } + if (input.acquireRunLease) { + const lease = await input.acquireRunLease({ + experimentId: input.experimentId, + runDir: input.runDir, + }) + if (!lease || typeof lease.assertOwned !== 'function' || typeof lease.release !== 'function') { + throw new Error(`${input.label} acquireRunLease must return assertOwned and release`) + } + return { + async assertOwned() { + await lease.assertOwned() + }, + async release() { + await lease.release() + }, + } + } + + if (!input.customStorage) { + if (input.controllerMode) { + throw new Error(`${input.label} controllerMode is only valid with custom storage`) + } + const lock = acquireSingleRunLock({ + lockPath: join(input.runDir, input.lockFileName), + releaseOnExit: false, + }) + return { + async assertOwned() {}, + async release() { + lock.release() + }, + } + } + + if (input.controllerMode !== 'process-local') { + throw new Error( + `${input.label} custom storage requires acquireRunLease or controllerMode='process-local'`, + ) + } + const runKey = `${input.label}:${input.runDir}` + if (activeProcessLocalRuns.has(runKey)) { + throw new Error(`${input.label} run '${input.runDir}' already has an active controller`) + } + activeProcessLocalRuns.add(runKey) + let released = false + return { + async assertOwned() { + if (released || !activeProcessLocalRuns.has(runKey)) { + throw new Error(`${input.label} run '${input.runDir}' controller lease was lost`) + } + }, + async release() { + if (released) return + released = true + activeProcessLocalRuns.delete(runKey) + }, + } +} diff --git a/src/memory/types.ts b/src/memory/types.ts index 8d8b46f..0a73408 100644 --- a/src/memory/types.ts +++ b/src/memory/types.ts @@ -83,7 +83,13 @@ export interface AgentMemoryWriteResult { } export type AgentMemoryBranchIsolation = - | { mode: 'scoped' } + | { + mode: 'scoped' + /** False when writes may outlive the worker process that issued them. */ + processExitSafe?: boolean + /** Wait before clearing an abandoned branch so accepted asynchronous writes become visible. */ + recoveryDelayMs?: number + } | { mode: 'instance' branchId: string @@ -99,6 +105,7 @@ export interface AgentMemoryAdapter { search(query: string, options?: AgentMemorySearchOptions): Promise getContext(query: string, options?: AgentMemorySearchOptions): Promise write(input: AgentMemoryWriteInput): Promise + /** Delete exactly this scope. Repeated and concurrent calls for the same scope must be safe. */ clear?(scope?: AgentMemoryScope): Promise flush?(): Promise close?(): Promise diff --git a/tests/benchmarks.test.ts b/tests/benchmarks.test.ts index c38ded3..98ebe83 100644 --- a/tests/benchmarks.test.ts +++ b/tests/benchmarks.test.ts @@ -12,17 +12,44 @@ import { isKnowledgeMemoryBenchmarkCase, type KnowledgeAnswerBenchmarkCase, type KnowledgeMemoryBenchmarkCase, + knowledgeBenchmarkJudge, parseKnowledgeBenchmarkJsonl, parseKnowledgeBenchmarkQrels, + type RunMemoryAdapterBenchmarkOptions, respondToIndustryMemoryBenchmarkSmokeCase, respondToIndustryRagBenchmarkSmokeCase, runKnowledgeBenchmarkSuite, - runMemoryAdapterBenchmark, + runMemoryAdapterBenchmark as runMemoryAdapterBenchmarkRaw, scoreKnowledgeBenchmarkArtifact, scoreMemoryBenchmarkArtifact, } from '../src/benchmarks/index' +import type { AgentMemoryAdapter, AgentMemoryHit, AgentMemoryScope } from '../src/memory/types' + +function runMemoryAdapterBenchmark(options: RunMemoryAdapterBenchmarkOptions) { + return runMemoryAdapterBenchmarkRaw({ + ...options, + ...(options.storage && !options.controllerMode && !options.acquireRunLease + ? { controllerMode: 'process-local' as const } + : {}), + }) +} describe('knowledge benchmark adapters', () => { + it('versions benchmark scoring for resumable cache safety', () => { + expect(knowledgeBenchmarkJudge().judgeVersion).toBe('agent-knowledge:knowledge-benchmark:v2') + }) + + it('requires a responder version for resumable benchmark rows', async () => { + await expect( + runKnowledgeBenchmarkSuite({ + cases: buildIndustryRagBenchmarkSmokeCases().slice(0, 1), + runDir: '/runs/missing-responder-ref', + storage: inMemoryCampaignStorage(), + respond: respondToIndustryRagBenchmarkSmokeCase, + }), + ).rejects.toThrow('respondRef is required when resumable is enabled') + }) + it('parses qrels/jsonl and builds retrieval cases for public benchmark formats', () => { expect(parseKnowledgeBenchmarkJsonl('{"id":"q1"}\n{"id":"q2"}\n')).toEqual([ { id: 'q1' }, @@ -76,6 +103,7 @@ describe('knowledge benchmark adapters', () => { cases, runDir: '/runs/knowledge-benchmark-smoke', storage, + respondRef: 'retriever-fixture:v1', respond: async ({ case: testCase, context }) => { const paid = await context.cost.runPaidCall({ actor: 'benchmark-retriever', @@ -170,6 +198,7 @@ describe('knowledge benchmark adapters', () => { cases, runDir: '/runs/knowledge-benchmark-family-smoke', storage, + respondRef: 'industry-rag-smoke:v1', respond: respondToIndustryRagBenchmarkSmokeCase, }) @@ -233,6 +262,22 @@ describe('knowledge benchmark adapters', () => { expect(weak.passed).toBe(false) expect(strong.score - weak.score).toBeGreaterThanOrEqual(0.4) expect(weak.dimensions.memory_stale_safe).toBe(0) + expect(strong.applicableDimensions).toEqual( + expect.arrayContaining([ + 'memory_fact_recall', + 'memory_event_recall', + 'memory_actor_recall', + 'memory_stale_safe', + ]), + ) + + const recallOnly = scoreMemoryBenchmarkArtifact( + { ...testCase, forbiddenFacts: undefined, expectedActorIds: undefined }, + { answer: 'The user currently prefers email briefings.', citedEventIds: ['e-new'] }, + ) + expect(recallOnly.dimensions).not.toHaveProperty('memory_stale_safe') + expect(recallOnly.dimensions).not.toHaveProperty('memory_actor_recall') + expect(recallOnly.applicableDimensions).not.toContain('memory_stale_safe') }) it('runs one persisted benchmark cell for every declared memory benchmark family', async () => { @@ -242,6 +287,7 @@ describe('knowledge benchmark adapters', () => { cases, runDir: '/runs/memory-benchmark-family-smoke', storage, + respondRef: 'industry-memory-smoke:v1', respond: ({ case: testCase }) => { expect(isKnowledgeMemoryBenchmarkCase(testCase)).toBe(true) if (!isKnowledgeMemoryBenchmarkCase(testCase)) { @@ -285,10 +331,12 @@ describe('knowledge benchmark adapters', () => { candidates: [ { id: 'no-memory', + ref: 'no-memory:v1', createAdapter: () => createNoopMemoryBenchmarkAdapter(), }, { id: 'in-memory', + ref: 'in-memory:v1', createAdapter: () => createInMemoryBenchmarkAdapter(), searchLimit: 1, }, @@ -308,6 +356,291 @@ describe('knowledge benchmark adapters', () => { expect(storage.read(result.rows[0]!.reportJsonPath)).toContain('"memory_stale_safe"') }) + it('requires an explicit controller policy for custom benchmark storage', async () => { + await expect( + runMemoryAdapterBenchmarkRaw({ + cases: buildFirstPartyMemoryLifecycleBenchmarkCases().slice(0, 1), + runDir: '/runs/custom-benchmark-storage', + storage: inMemoryCampaignStorage(), + candidates: [ + { + id: 'in-memory', + ref: 'in-memory:v1', + createAdapter: () => createInMemoryBenchmarkAdapter(), + }, + ], + }), + ).rejects.toThrow("requires acquireRunLease or controllerMode='process-local'") + }) + + it('recovers an unfinished provider scope before retrying a direct benchmark cell', async () => { + const storage = inMemoryCampaignStorage() + const providerRows = new Map() + const operations: string[] = [] + const purposes: string[] = [] + let firstExecution = true + const candidate = { + id: 'recoverable', + ref: 'recoverable:v1', + recoveryCostUsdPerAttempt: 0.1, + createAdapter({ purpose }: { purpose: 'execute' | 'recovery' }) { + purposes.push(purpose) + const label = purpose === 'recovery' ? 'recovery' : firstExecution ? 'first' : 'retry' + if (purpose === 'execute') firstExecution = false + const adapter: AgentMemoryAdapter = { + id: 'recoverable-provider', + branchIsolation: { mode: 'scoped' }, + async search(_query, options) { + return [...(providerRows.get(options?.scope?.namespace ?? '') ?? [])] + }, + async getContext(query, options) { + const hits = await adapter.search(query, options) + return { query, text: hits.map((hit) => hit.text).join('\n'), hits, sourceRecords: [] } + }, + async write(input) { + operations.push(`write:${label}`) + const namespace = input.scope?.namespace ?? '' + const hit: AgentMemoryHit = { + id: input.id ?? `${label}:memory`, + uri: `memory://recoverable/${label}`, + kind: input.kind, + text: input.text, + metadata: input.metadata, + } + providerRows.set(namespace, [...(providerRows.get(namespace) ?? []), hit]) + return { accepted: true, id: hit.id, uri: hit.uri, kind: hit.kind } + }, + async clear(scope) { + operations.push(`clear:${label}`) + if (label === 'first') throw new Error('provider cleanup unavailable') + providerRows.delete(scope?.namespace ?? '') + }, + async close() {}, + } + return adapter + }, + } + const run = () => + runMemoryAdapterBenchmark({ + cases: buildFirstPartyMemoryLifecycleBenchmarkCases().slice(0, 1), + runDir: '/runs/direct-benchmark-recovery', + storage, + candidates: [candidate], + }) + + await expect(run()).rejects.toThrow('memory benchmark attempt cleanup failed') + expect(providerRows.size).toBe(1) + + const result = await run() + expect(result.rows[0]).toMatchObject({ candidateId: 'recoverable', cellsFailed: 0 }) + expect(result.rows[0]?.totalCostUsd).toBe(0.1) + expect(result.totalCostUsd).toBe(0.1) + expect(purposes).toEqual(['execute', 'recovery', 'execute']) + expect(operations.indexOf('clear:recovery')).toBeLessThan(operations.indexOf('write:retry')) + expect(providerRows.size).toBe(0) + const events = storage + .read(result.attemptLogPath)! + .trim() + .split('\n') + .map((line) => JSON.parse(line) as { status: string; recovery: boolean }) + expect(events.map(({ status, recovery }) => ({ status, recovery }))).toEqual([ + { status: 'started', recovery: false }, + { status: 'cleaned', recovery: true }, + { status: 'started', recovery: false }, + { status: 'cleaned', recovery: false }, + ]) + }) + + it('bounds direct benchmark cleanup and preserves its recovery record', async () => { + const storage = inMemoryCampaignStorage() + const adapter = createInMemoryBenchmarkAdapter({ id: 'hung-cleanup' }) + adapter.clear = () => new Promise(() => {}) + const startedAt = Date.now() + + await expect( + runMemoryAdapterBenchmark({ + cases: buildFirstPartyMemoryLifecycleBenchmarkCases().slice(0, 1), + runDir: '/runs/direct-benchmark-cleanup-timeout', + storage, + cleanupTimeoutMs: 10, + candidates: [ + { + id: 'hung-cleanup', + ref: 'hung-cleanup:v1', + createAdapter: () => adapter, + }, + ], + }), + ).rejects.toThrow('memory benchmark attempt cleanup failed') + + expect(Date.now() - startedAt).toBeLessThan(500) + expect( + storage + .read('/runs/direct-benchmark-cleanup-timeout/memory-adapter-attempts.jsonl') + ?.trim() + .split('\n'), + ).toHaveLength(1) + }) + + it('enforces one cost ceiling across every compared adapter', async () => { + const result = await runMemoryAdapterBenchmark({ + cases: buildFirstPartyMemoryLifecycleBenchmarkCases().slice(0, 1), + runDir: '/runs/shared-adapter-cost-ceiling', + storage: inMemoryCampaignStorage(), + costCeiling: 0.75, + candidates: ['first', 'second'].map((id) => ({ + id, + ref: `${id}:v1`, + costUsdPerCase: 0.5, + createAdapter: () => createInMemoryBenchmarkAdapter({ id }), + })), + }) + + expect(result.totalCostUsd).toBe(0.5) + expect(result.rows).toHaveLength(2) + expect(result.rows.find((row) => row.candidateId === 'first')).toMatchObject({ + cellsFailed: 0, + totalCostUsd: 0.5, + }) + expect(result.rows.find((row) => row.candidateId === 'second')).toMatchObject({ + cellsFailed: 1, + totalCostUsd: 0, + }) + }) + + it('does not reuse resumable rows when adapter benchmark options change', async () => { + const storage = inMemoryCampaignStorage() + const run = (searchLimit: number) => + runMemoryAdapterBenchmark({ + cases: buildFirstPartyMemoryLifecycleBenchmarkCases().slice(0, 1), + runDir: '/runs/memory-adapter-cache-identity', + storage, + candidates: [ + { + id: 'in-memory', + ref: 'in-memory:v1', + createAdapter: () => createInMemoryBenchmarkAdapter(), + searchLimit, + }, + ], + }) + + const initial = await run(1) + const changed = await run(2) + const repeated = await run(2) + + expect(initial.rows[0]?.report.cellsCached).toBe(0) + expect(changed.rows[0]?.report.cellsCached).toBe(0) + expect(repeated.rows[0]?.report.cellsCached).toBe(1) + }) + + it('isolates and clears every concurrent adapter repetition', async () => { + const adapter = createInMemoryBenchmarkAdapter() + const clearedScopes: AgentMemoryScope[] = [] + const clear = adapter.clear! + adapter.clear = async (scope) => { + clearedScopes.push(scope ?? {}) + await clear(scope) + } + + const result = await runMemoryAdapterBenchmark({ + cases: buildFirstPartyMemoryLifecycleBenchmarkCases().slice(0, 1), + runDir: '/runs/isolated-memory-repetitions', + storage: inMemoryCampaignStorage(), + reps: 3, + maxConcurrency: 3, + candidates: [ + { + id: 'in-memory', + ref: 'in-memory:v1', + createAdapter: () => adapter, + }, + ], + }) + + expect(result.rows[0]?.totalCells).toBe(3) + expect(result.rows[0]?.cellsFailed).toBe(0) + expect(clearedScopes).toHaveLength(3) + expect(new Set(clearedScopes.map((scope) => scope.namespace)).size).toBe(3) + expect(new Set(clearedScopes.map((scope) => scope.tags?.benchmarkAttemptId)).size).toBe(3) + }) + + it('settles timed-out provider work before clearing and closing the adapter', async () => { + let closed = false + let writes = 0 + let writesAfterClose = 0 + let clears = 0 + let clearsAfterClose = 0 + let contextReads = 0 + const adapter: AgentMemoryAdapter = { + id: 'delayed-provider', + branchIsolation: { mode: 'scoped' }, + async search() { + return [] + }, + async getContext(query) { + contextReads += 1 + return { query, text: '', hits: [], sourceRecords: [] } + }, + async write(input) { + await new Promise((resolve) => setTimeout(resolve, 25)) + writes += 1 + if (closed) writesAfterClose += 1 + return { + accepted: true, + id: input.id ?? String(writes), + uri: `memory://delayed-provider/${writes}`, + kind: input.kind, + } + }, + async clear() { + await new Promise((resolve) => setTimeout(resolve, 1)) + clears += 1 + if (closed) clearsAfterClose += 1 + }, + async close() { + closed = true + }, + } + + const result = await runMemoryAdapterBenchmark({ + cases: buildFirstPartyMemoryLifecycleBenchmarkCases().slice(0, 1), + runDir: '/runs/timed-out-memory-provider', + storage: inMemoryCampaignStorage(), + dispatchTimeoutMs: 5, + candidates: [ + { + id: 'delayed-provider', + ref: 'delayed-provider:v1', + createAdapter: () => adapter, + }, + { + id: 'no-memory', + ref: 'no-memory:v1', + createAdapter: () => createNoopMemoryBenchmarkAdapter(), + }, + ], + }) + + expect(result.rows[0]).toMatchObject({ candidateId: 'no-memory', cellsFailed: 0 }) + expect(result.rows[1]).toMatchObject({ candidateId: 'delayed-provider', cellsFailed: 1 }) + expect({ closed, writes, writesAfterClose, clears, clearsAfterClose, contextReads }).toEqual({ + closed: true, + writes: 1, + writesAfterClose: 0, + clears: 1, + clearsAfterClose: 0, + contextReads: 0, + }) + await new Promise((resolve) => setTimeout(resolve, 50)) + expect({ writes, writesAfterClose, clears, clearsAfterClose }).toEqual({ + writes: 1, + writesAfterClose: 0, + clears: 1, + clearsAfterClose: 0, + }) + }) + it('rejects every unsafe candidate id before creating any adapter', async () => { let creates = 0 await expect( @@ -318,6 +651,7 @@ describe('knowledge benchmark adapters', () => { candidates: [ { id: 'valid', + ref: 'valid:v1', createAdapter() { creates += 1 return createInMemoryBenchmarkAdapter() @@ -325,6 +659,7 @@ describe('knowledge benchmark adapters', () => { }, { id: '../invalid', + ref: 'invalid:v1', createAdapter() { creates += 1 return createInMemoryBenchmarkAdapter() diff --git a/tests/memory-systems.test.ts b/tests/memory-systems.test.ts index 5071213..83bdc61 100644 --- a/tests/memory-systems.test.ts +++ b/tests/memory-systems.test.ts @@ -1,5 +1,9 @@ import type { MemoryClient as Neo4jMemoryClient } from '@neo4j-labs/agent-memory' -import { inMemoryCampaignStorage, type SurfaceProposer } from '@tangle-network/agent-eval/campaign' +import { + createRunCostLedger, + inMemoryCampaignStorage, + type SurfaceProposer, +} from '@tangle-network/agent-eval/campaign' import type { MemoryClient } from 'mem0ai' import type { Memory as OssMemory } from 'mem0ai/oss' import { describe, expect, it } from 'vitest' @@ -7,20 +11,42 @@ import { type AgentMemoryAdapter, type AgentMemoryHit, type AgentMemoryScope, + agentMemorySequenceJudge, buildAgentMemorySequencesFromBenchmarkCases, createAgentMemoryBranch, createGraphitiMemoryAdapter, createMem0MemoryAdapter, createNeo4jAgentMemoryAdapter, + forkAgentMemoryBranchSnapshot, type GraphitiMcpClientLike, graphitiMemoryAdapterIdentity, type Mem0ClientLike, mem0MemoryAdapterIdentity, + type RunAgentMemoryExperimentOptions, type RunAgentMemoryImprovementOptions, - runAgentMemoryExperiment, - runAgentMemoryImprovement, + runAgentMemoryExperiment as runAgentMemoryExperimentRaw, + runAgentMemoryImprovement as runAgentMemoryImprovementRaw, } from '../src/memory/index' +function withProcessLocalController< + T extends { + storage?: unknown + controllerMode?: 'process-local' + acquireRunLease?: unknown + }, +>(options: T): T { + if (!options.storage || options.controllerMode || options.acquireRunLease) return options + return { ...options, controllerMode: 'process-local' } +} + +function runAgentMemoryExperiment(options: RunAgentMemoryExperimentOptions) { + return runAgentMemoryExperimentRaw(withProcessLocalController(options)) +} + +function runAgentMemoryImprovement(options: RunAgentMemoryImprovementOptions) { + return runAgentMemoryImprovementRaw(withProcessLocalController(options)) +} + describe('Mem0 adapter', () => { it('is type-compatible with both current Mem0 clients', () => { const hosted = null as unknown as MemoryClient @@ -72,7 +98,7 @@ describe('Mem0 adapter', () => { title: 'Writing style', text: 'Use concise status updates.', scope, - metadata: { run_id: 'attempted-override' }, + metadata: { run_id: 'attempted-override', memoryKind: 'entity' }, }) const hits = await adapter.search('status style', { scope, limit: 3 }) @@ -84,6 +110,7 @@ describe('Mem0 adapter', () => { runId: 'branch-1/private', infer: true, metadata: { + memoryKind: 'preference', tenant_id: 'tenant-1', user_id: 'user-1', agent_id: 'agent-1', @@ -95,11 +122,11 @@ describe('Mem0 adapter', () => { }) expect(search).toMatchObject({ filters: { - tenant_id: 'tenant-1', user_id: 'user-1', agent_id: 'agent-1', - team_id: 'team-1', run_id: 'branch-1/private', + tenant_id: 'tenant-1', + team_id: 'team-1', session_id: 'session-1', tag_task: 'support', }, @@ -116,6 +143,200 @@ describe('Mem0 adapter', () => { }) }) + it('pushes memory kinds into provider filters before top-k truncation', async () => { + const searchOptions: Record[] = [] + const adapter = createMem0MemoryAdapter({ + mode: 'hosted', + client: { + async add() { + return [] + }, + async search(_query, options) { + searchOptions.push(options ?? {}) + const filters = options?.filters as Record | undefined + if (filters?.memoryKind === 'preference') { + return { + results: [ + { + id: 'preference-1', + memory: 'Use concise updates.', + metadata: { memoryKind: 'preference' }, + }, + ], + } + } + return { + results: [ + { + id: 'fact-1', + memory: 'Higher-ranked unrelated fact.', + metadata: { memoryKind: 'fact' }, + }, + ], + } + }, + }, + }) + + const filtered = await adapter.search('updates', { + scope: { userId: 'user-1' }, + kinds: ['preference'], + limit: 1, + }) + const unfiltered = await adapter.search('updates', { + scope: { userId: 'user-1' }, + kinds: [], + limit: 1, + }) + + expect(filtered.map((hit) => hit.id)).toEqual(['preference-1']) + expect(unfiltered.map((hit) => hit.id)).toEqual(['fact-1']) + expect(searchOptions[0]).toMatchObject({ + filters: { user_id: 'user-1', memoryKind: 'preference' }, + topK: 1, + }) + expect(searchOptions[1]).toMatchObject({ filters: { user_id: 'user-1' }, topK: 1 }) + }) + + it('waits for the official hosted Mem0 event to succeed', async () => { + let addOptions: Record | undefined + const eventIds: string[] = [] + const adapter = createMem0MemoryAdapter({ + mode: 'hosted', + ingestionTimeoutMs: 50, + pollIntervalMs: 1, + client: { + async add(_messages, options) { + addOptions = options + return { status: 'PENDING', eventId: 'event-1' } + }, + async search() { + return { results: [] } + }, + }, + async getEvent(eventId) { + eventIds.push(eventId) + return eventIds.length === 1 + ? { id: eventId, status: 'RUNNING', results: [] } + : { + id: eventId, + status: 'SUCCEEDED', + results: [ + { + id: 'memory-visible', + memory: 'Launch is Friday.', + metadata: { memoryKind: 'fact' }, + }, + ], + } + }, + }) + + const result = await adapter.write({ + id: 'source-event', + kind: 'fact', + text: 'Launch is Friday.', + scope: { userId: 'user-1' }, + }) + + const metadata = addOptions?.metadata as Record + expect(metadata.agentKnowledgeWriteId).toBe('source-event') + expect(metadata.agentKnowledgeWriteAttemptId).toBeUndefined() + expect(eventIds).toEqual(['event-1', 'event-1']) + expect(result).toMatchObject({ + accepted: true, + id: 'memory-visible', + metadata: { consistency: 'visible', queued: false }, + }) + }) + + it('treats a successful hosted Mem0 event with no results as a no-op', async () => { + const adapter = createMem0MemoryAdapter({ + mode: 'hosted', + client: { + async add() { + return { status: 'PENDING', eventId: 'event-noop' } + }, + async search() { + return { results: [] } + }, + }, + async getEvent(eventId) { + return { id: eventId, status: 'SUCCEEDED', results: [] } + }, + }) + + await expect(adapter.write({ kind: 'fact', text: 'No durable fact.' })).resolves.toMatchObject({ + accepted: false, + id: 'event-noop', + }) + }) + + it('surfaces a failed hosted Mem0 event', async () => { + const adapter = createMem0MemoryAdapter({ + mode: 'hosted', + client: { + async add() { + return { status: 'PENDING', eventId: 'event-failed' } + }, + async search() { + return { results: [] } + }, + }, + async getEvent(eventId) { + return { id: eventId, status: 'FAILED', message: 'extraction failed' } + }, + }) + + await expect(adapter.write({ kind: 'fact', text: 'Remember this.' })).rejects.toThrow( + 'Mem0 event event-failed failed: extraction failed', + ) + }) + + it('requires fresh attempt branches for visible hosted Mem0', async () => { + const client: Mem0ClientLike = { + async add() { + return { status: 'PENDING', eventId: 'event-1' } + }, + async search() { + return { results: [] } + }, + } + const visible = createMem0MemoryAdapter({ + mode: 'hosted', + client, + async getEvent(eventId) { + return { id: eventId, status: 'SUCCEEDED', results: [] } + }, + }) + const queued = createMem0MemoryAdapter({ + mode: 'hosted', + consistency: 'queued', + client, + }) + + expect(() => createAgentMemoryBranch({ adapter: visible, branchId: 'mem0-resumable' })).toThrow( + "must use lifetime='attempt'", + ) + const attempt = createAgentMemoryBranch({ + adapter: visible, + branchId: 'mem0-attempt', + lifetime: 'attempt', + }) + const snapshot = await attempt.snapshot() + await attempt.close() + expect(() => + createAgentMemoryBranch({ + adapter: visible, + branchId: 'mem0-attempt', + snapshot, + }), + ).toThrow('attempt snapshots cannot be resumed') + expect(() => createAgentMemoryBranch({ adapter: queued, branchId: 'mem0-queued' })).toThrow( + 'queued hosted Mem0 writes can become visible after branch cleanup', + ) + }) + it('does not journal an empty provider result as an accepted write', async () => { const adapter = createMem0MemoryAdapter({ mode: 'hosted', @@ -139,7 +360,7 @@ describe('Mem0 adapter', () => { const childCalls: string[] = [] const adapter = (calls: string[]) => createMem0MemoryAdapter({ - mode: 'hosted', + mode: 'oss', client: { async add(messages) { const text = messages[0]!.content @@ -180,6 +401,9 @@ describe('Mem0 adapter', () => { expect(mem0MemoryAdapterIdentity({ ...base, defaultScope: { tenantId: 'tenant-a' } })).not.toBe( mem0MemoryAdapterIdentity({ ...base, defaultScope: { tenantId: 'tenant-b' } }), ) + expect(mem0MemoryAdapterIdentity({ ...base, consistency: 'visible' })).not.toBe( + mem0MemoryAdapterIdentity({ ...base, consistency: 'queued' }), + ) }) it('passes OSS entity filters on add', async () => { @@ -254,9 +478,9 @@ describe('Mem0 adapter', () => { expect(getAllOptions[0]).toMatchObject({ filters: { app_id: 'support-app', - tenant_id: 'tenant-1', user_id: 'user-1', run_id: 'physical-branch', + tenant_id: 'tenant-1', logical_run_id: 'logical-run', session_id: 'session-1', tag_visibility: 'private', @@ -266,9 +490,93 @@ describe('Mem0 adapter', () => { showExpired: true, }) }) + + it('waits for hosted deletes to disappear without deleting the same memory twice', async () => { + let deletedAt: number | undefined + let getAllCalls = 0 + const deletedIds: string[] = [] + const adapter = createMem0MemoryAdapter({ + mode: 'hosted', + ingestionTimeoutMs: 50, + pollIntervalMs: 1, + client: { + async add() { + return [] + }, + async search() { + return { results: [] } + }, + async getAll() { + getAllCalls += 1 + const visible = deletedAt === undefined || Date.now() - deletedAt < 5 + return { results: visible ? [{ id: 'memory-1', memory: 'stale listing' }] : [] } + }, + async delete(memoryId) { + deletedIds.push(memoryId) + deletedAt = Date.now() + return { message: 'deletion accepted' } + }, + }, + }) + + await adapter.clear?.({ userId: 'user-1', namespace: 'branch-1' }) + + expect(deletedIds).toEqual(['memory-1']) + expect(getAllCalls).toBeGreaterThan(2) + }) + + it('waits for bulk-deleted memories to disappear before reporting exact cleanup', async () => { + let deletedAt: number | undefined + let getAllCalls = 0 + let deleteAllCalls = 0 + const adapter = createMem0MemoryAdapter({ + mode: 'hosted', + ingestionTimeoutMs: 50, + pollIntervalMs: 1, + client: { + async add() { + return [] + }, + async search() { + return { results: [] } + }, + async getAll() { + getAllCalls += 1 + const visible = deletedAt === undefined || Date.now() - deletedAt < 5 + return { results: visible ? [{ id: 'memory-1', memory: 'stale listing' }] : [] } + }, + async deleteAll() { + deleteAllCalls += 1 + deletedAt = Date.now() + return { message: 'deletion accepted' } + }, + }, + }) + + await adapter.clear?.({ userId: 'user-1', namespace: 'branch-1' }) + + expect(deleteAllCalls).toBe(1) + expect(getAllCalls).toBeGreaterThan(1) + }) }) describe('Graphiti adapter', () => { + it('uses the full default ingestion window before recovering an abandoned branch', () => { + const adapter = createGraphitiMemoryAdapter({ + client: { + async callTool() { + return { structuredContent: {} } + }, + }, + }) + + expect(adapter.branchIsolation).toEqual({ + mode: 'scoped', + processExitSafe: false, + recoveryDelayMs: 120_000, + }) + }) + it('uses official MCP request shapes and rejoins fact provenance to episodes', async () => { const requests: Array<{ name: string; arguments?: Record }> = [] const episodes = new Map>() @@ -505,7 +813,7 @@ describe('Graphiti adapter', () => { expect(new Set(groupIds).size).toBe(2) }) - it('keeps episode ids stable when independent groups execute in a different order', async () => { + it('keeps caller-identified episode ids stable when groups execute in a different order', async () => { const run = async (scopes: AgentMemoryScope[]) => { const ids = new Map() const adapter = createGraphitiMemoryAdapter({ @@ -520,7 +828,7 @@ describe('Graphiti adapter', () => { }, }) for (const scope of scopes) { - await adapter.write({ kind: 'fact', text: 'same fact', scope }) + await adapter.write({ id: 'source-1', kind: 'fact', text: 'same fact', scope }) } return ids } @@ -551,6 +859,103 @@ describe('Graphiti adapter', () => { expect(episodeIds[0]).not.toBe(episodeIds[1]) }) + it('keeps repeated id-less Graphiti writes as distinct episodes', async () => { + const episodeIds: string[] = [] + const adapter = createGraphitiMemoryAdapter({ + consistency: 'queued', + client: { + async callTool(request) { + if (request.name === 'add_memory') { + episodeIds.push(String(request.arguments?.uuid)) + } + return { structuredContent: { message: 'queued' } } + }, + }, + }) + + await adapter.write({ kind: 'fact', text: 'The same observation.' }) + await adapter.write({ kind: 'fact', text: 'The same observation.' }) + + expect(episodeIds).toHaveLength(2) + expect(episodeIds[0]).not.toBe(episodeIds[1]) + }) + + it('keeps id-less Graphiti writes distinct across adapter restarts', async () => { + const episodeIds: string[] = [] + const client: GraphitiMcpClientLike = { + async callTool(request) { + if (request.name === 'add_memory') episodeIds.push(String(request.arguments?.uuid)) + return { structuredContent: { message: 'queued' } } + }, + } + + await createGraphitiMemoryAdapter({ client, consistency: 'queued' }).write({ + kind: 'fact', + text: 'The same observation.', + }) + await createGraphitiMemoryAdapter({ client, consistency: 'queued' }).write({ + kind: 'fact', + text: 'The same observation.', + }) + + expect(episodeIds).toHaveLength(2) + expect(episodeIds[0]).not.toBe(episodeIds[1]) + }) + + it('treats an empty Graphiti kind list as unfiltered search', async () => { + const names: string[] = [] + const adapter = createGraphitiMemoryAdapter({ + client: { + async callTool(request) { + names.push(request.name) + if (request.name === 'search_memory_facts') { + return { structuredContent: { facts: [] } } + } + if (request.name === 'search_nodes') return { structuredContent: { nodes: [] } } + throw new Error(`unexpected tool ${request.name}`) + }, + }, + }) + + await adapter.search('anything', { kinds: [] }) + + expect(names).toEqual(['search_memory_facts', 'search_nodes']) + }) + + it('fuses unscored Graphiti fact and node rankings before applying the limit', async () => { + const adapter = createGraphitiMemoryAdapter({ + client: { + async callTool(request) { + if (request.name === 'search_memory_facts') { + return { + structuredContent: { + facts: [ + { uuid: 'fact-1', fact: 'first fact' }, + { uuid: 'fact-2', fact: 'second fact' }, + ], + }, + } + } + if (request.name === 'search_nodes') { + return { + structuredContent: { + nodes: [ + { uuid: 'node-1', name: 'first node' }, + { uuid: 'node-2', name: 'second node' }, + ], + }, + } + } + throw new Error(`unexpected tool ${request.name}`) + }, + }, + }) + + const hits = await adapter.search('anything', { limit: 2 }) + + expect(hits.map((hit) => hit.id)).toEqual(['fact-1', 'node-1']) + }) + it('uses Graphiti clear_graph for exact group cleanup', async () => { const requests: Array<{ name: string; arguments?: Record }> = [] const adapter = createGraphitiMemoryAdapter({ @@ -737,6 +1142,42 @@ describe('Neo4j Agent Memory adapter', () => { { id: 'preference-1', kind: 'preference', text: 'Be concise.' }, ]) }) + + it('fuses unscored Neo4j memory-type rankings before applying the limit', async () => { + const adapter = createNeo4jAgentMemoryAdapter({ + transport: 'bridge', + client: { + shortTerm: { + async searchMessages() { + return [ + { id: 'message-1', role: 'user', content: 'first message' }, + { id: 'message-2', role: 'user', content: 'second message' }, + ] + }, + }, + longTerm: { + async searchEntities() { + return [ + { id: 'entity-1', name: 'first entity', type: 'custom' }, + { id: 'entity-2', name: 'second entity', type: 'custom' }, + ] + }, + async searchPreferences() { + return [] + }, + }, + reasoning: { + async getSimilarTraces() { + return [] + }, + }, + }, + }) + + const hits = await adapter.search('anything', { limit: 2 }) + + expect(hits.map((hit) => hit.id)).toEqual(['message-1', 'entity-1']) + }) }) describe('memory branches', () => { @@ -830,6 +1271,133 @@ describe('memory branches', () => { expect(teamHits.map(hitText)).toEqual(['shared with the team']) }) + it('removes every journal entry in a provider partition cleared through a narrower scope', async () => { + const branch = createAgentMemoryBranch({ + adapter: createScopedTestAdapter('team-clear'), + branchId: 'team-clear', + policy: { read: ['team'], write: 'team' }, + baseScope: { tenantId: 'tenant', teamId: 'team-1' }, + }) + await branch.write({ kind: 'fact', text: 'from a', scope: { agentId: 'agent-a' } }) + await branch.write({ kind: 'fact', text: 'from b', scope: { agentId: 'agent-b' } }) + + await branch.clear?.({ agentId: 'agent-a' }) + + expect((await branch.snapshot()).journal).toEqual([]) + await expect(branch.search('from b', { scope: { agentId: 'agent-b' } })).resolves.toEqual([]) + }) + + it('preserves provider order for unscored branch hits', async () => { + const hits: AgentMemoryHit[] = [ + { id: 'z-top', uri: 'memory://rank/z', kind: 'fact', text: 'provider first' }, + { id: 'a-lower', uri: 'memory://rank/a', kind: 'fact', text: 'provider second' }, + ] + const adapter: AgentMemoryAdapter = { + id: 'provider-ranked', + branchIsolation: { mode: 'scoped' }, + async search() { + return hits + }, + async getContext(query) { + return { query, text: '', hits: [], sourceRecords: [] } + }, + async write(input) { + return { accepted: true, id: 'write', uri: 'memory://rank/write', kind: input.kind } + }, + } + const branch = createAgentMemoryBranch({ + adapter, + branchId: 'provider-ranked', + baseScope: { agentId: 'agent-a' }, + }) + + const ranked = await branch.search('anything', { limit: 1 }) + + expect(ranked.map((hit) => hit.id)).toEqual(['z-top']) + }) + + it('fuses unscored visibility lists by provider rank', async () => { + const adapter: AgentMemoryAdapter = { + id: 'visibility-ranked', + branchIsolation: { mode: 'scoped' }, + async search(_query, options) { + return options?.scope?.tags?.memoryVisibility === 'private' + ? [ + { id: 'private-1', uri: 'memory://rank/private-1', kind: 'fact', text: 'p1' }, + { id: 'private-2', uri: 'memory://rank/private-2', kind: 'fact', text: 'p2' }, + ] + : [{ id: 'team-1', uri: 'memory://rank/team-1', kind: 'fact', text: 't1' }] + }, + async getContext(query) { + return { query, text: '', hits: [], sourceRecords: [] } + }, + async write(input) { + return { accepted: true, id: 'write', uri: 'memory://rank/write', kind: input.kind } + }, + } + const branch = createAgentMemoryBranch({ + adapter, + branchId: 'visibility-ranked', + policy: { read: ['private', 'team'], write: 'private' }, + baseScope: { agentId: 'agent-a', teamId: 'team-a' }, + }) + + const ranked = await branch.search('anything', { limit: 2 }) + + expect(ranked.map((hit) => hit.id)).toEqual(['private-1', 'team-1']) + }) + + it('uses rank fusion before incomparable scores from separate provider searches', async () => { + const adapter: AgentMemoryAdapter = { + id: 'visibility-scored', + branchIsolation: { mode: 'scoped' }, + async search(_query, options) { + return options?.scope?.tags?.memoryVisibility === 'private' + ? [ + { + id: 'private-first', + uri: 'memory://rank/private-first', + kind: 'fact', + text: 'private first', + score: 0.01, + }, + { + id: 'private-second', + uri: 'memory://rank/private-second', + kind: 'fact', + text: 'private second', + score: 1, + }, + ] + : [ + { + id: 'team-first', + uri: 'memory://rank/team-first', + kind: 'fact', + text: 'team first', + score: 0.02, + }, + ] + }, + async getContext(query) { + return { query, text: '', hits: [], sourceRecords: [] } + }, + async write(input) { + return { accepted: true, id: 'write', uri: 'memory://rank/write', kind: input.kind } + }, + } + const branch = createAgentMemoryBranch({ + adapter, + branchId: 'visibility-scored', + policy: { read: ['private', 'team'], write: 'private' }, + baseScope: { agentId: 'agent-a', teamId: 'team-a' }, + }) + + const ranked = await branch.search('anything', { limit: 2 }) + + expect(ranked.map((hit) => hit.id)).toEqual(['team-first', 'private-first']) + }) + it('serializes writes per actor and permits independent actors in parallel', async () => { let active = 0 let maxActive = 0 @@ -1086,6 +1654,70 @@ describe('memory branches', () => { await expect(first.search('after close')).rejects.toThrow('closed branch') }) + it('waits for a concurrent fork to finish before closing the parent', async () => { + let releaseSnapshot!: () => void + let releaseReplay!: () => void + let reportReplayStarted!: () => void + const snapshotBlocked = new Promise((resolve) => { + releaseSnapshot = resolve + }) + const replayBlocked = new Promise((resolve) => { + releaseReplay = resolve + }) + const replayStarted = new Promise((resolve) => { + reportReplayStarted = resolve + }) + const base = createScopedTestAdapter('fork-close-race') + let writes = 0 + let flushes = 0 + let closes = 0 + const adapter: AgentMemoryAdapter = { + ...base, + async write(input) { + writes += 1 + if (writes === 2) { + reportReplayStarted() + await replayBlocked + } + return base.write(input) + }, + async flush() { + flushes += 1 + if (flushes === 1) await snapshotBlocked + }, + async close() { + closes += 1 + }, + } + const parent = createAgentMemoryBranch({ + adapter, + branchId: 'fork-close-parent', + baseScope: { agentId: 'worker' }, + }) + await parent.write({ kind: 'fact', text: 'replay me' }) + + const fork = parent.fork({ branchId: 'fork-close-child' }) + const close = parent.close!() + let closeSettled = false + void close.finally(() => { + closeSettled = true + }) + releaseSnapshot() + await replayStarted + await Promise.resolve() + await Promise.resolve() + + expect(closeSettled).toBe(false) + expect(closes).toBe(0) + releaseReplay() + const child = await fork + await close + expect((await child.search('replay')).map(hitText)).toEqual(['replay me']) + expect(closes).toBe(0) + await child.close?.() + expect(closes).toBe(1) + }) + it('forks by replaying accepted writes into another adapter and resumes exactly', async () => { const parentStorage = createScopedTestAdapter('parent') const childStorage = createScopedTestAdapter('child') @@ -1108,6 +1740,8 @@ describe('memory branches', () => { expect((await parent.search('fact')).map(hitText)).toEqual(['seed fact']) expect((await child.search('fact')).map(hitText)).toEqual(['seed fact', 'child only']) const snapshot = await child.snapshot() + expect(snapshot.journal.every((entry) => Boolean(entry.input.id))).toBe(true) + expect(new Set(snapshot.journal.map((entry) => entry.input.id)).size).toBe(2) const resumed = createAgentMemoryBranch({ adapter: childStorage, branchId: 'child', @@ -1118,19 +1752,209 @@ describe('memory branches', () => { expect((await resumed.snapshot()).baseScope.tenantId).toBe('tenant') expect((await resumed.snapshot()).digest).toBe(snapshot.digest) - await expect( - Promise.resolve().then(() => - createAgentMemoryBranch({ - adapter: childStorage, - branchId: 'child', - snapshot: { ...snapshot, digest: 'sha256:bad' }, - }), - ), - ).rejects.toThrow('digest mismatch') + await expect( + Promise.resolve().then(() => + createAgentMemoryBranch({ + adapter: childStorage, + branchId: 'child', + snapshot: { ...snapshot, digest: 'sha256:bad' }, + }), + ), + ).rejects.toThrow('digest mismatch') + }) + + it('replays an attempt snapshot into a fresh branch', async () => { + const adapter: AgentMemoryAdapter = { + ...createScopedTestAdapter('attempt-source'), + branchIsolation: { mode: 'scoped', processExitSafe: false, recoveryDelayMs: 1 }, + } + const source = createAgentMemoryBranch({ + adapter, + branchId: 'attempt-source', + lifetime: 'attempt', + baseScope: { agentId: 'worker' }, + }) + await source.write({ kind: 'fact', text: 'durable observation' }) + const snapshot = await source.snapshot() + const target = await forkAgentMemoryBranchSnapshot({ + snapshot, + adapter: createScopedTestAdapter('resumable-target'), + branchId: 'resumable-target', + lifetime: 'resumable', + }) + + expect((await target.search('observation')).map(hitText)).toEqual(['durable observation']) + expect(target.parentBranchId).toBe('attempt-source') + expect(target.lifetime).toBe('resumable') + }) +}) + +describe('agent memory experiments', () => { + it('versions memory scoring for resumable cache safety', () => { + expect(agentMemorySequenceJudge().judgeVersion).toBe('agent-knowledge:memory-sequence:v2') + }) + + it('requires an explicit controller policy for custom storage', async () => { + await expect( + runAgentMemoryExperimentRaw({ + experimentId: 'custom-storage-controller', + sequences: [ + { + id: 'history', + family: 'first-party', + steps: [{ id: 'probe', probes: [{ id: 'probe', query: 'x', referenceAnswer: 'x' }] }], + }, + ], + candidates: [ + { + id: 'memory', + ref: 'memory:v1', + createAdapter: () => createScopedTestAdapter('memory'), + }, + ], + runDir: '/runs/custom-storage-controller', + storage: inMemoryCampaignStorage(), + }), + ).rejects.toThrow("requires acquireRunLease or controllerMode='process-local'") + }) + + it('bounds provider cleanup and leaves the attempt available for recovery', async () => { + const storage = inMemoryCampaignStorage() + let clearCalls = 0 + const adapter = createScopedTestAdapter('bounded-cleanup') + const clear = adapter.clear! + adapter.clear = async (scope) => { + clearCalls += 1 + if (clearCalls > 1) return new Promise(() => {}) + await clear(scope) + } + const startedAt = Date.now() + + await expect( + runAgentMemoryExperiment({ + experimentId: 'bounded-cleanup', + sequences: [ + { + id: 'history', + family: 'first-party', + steps: [ + { + id: 'remember', + scope: { agentId: 'worker' }, + writes: [{ kind: 'fact', text: 'bounded fact' }], + probes: [{ id: 'probe', query: 'bounded', referenceAnswer: 'bounded fact' }], + }, + ], + }, + ], + candidates: [{ id: 'memory', ref: 'memory:v1', createAdapter: () => adapter }], + runDir: '/runs/bounded-cleanup', + storage, + cleanupTimeoutMs: 10, + }), + ).rejects.toThrow('memory experiment cleanup failed after dispatch') + + expect(Date.now() - startedAt).toBeLessThan(500) + expect( + storage.read('/runs/bounded-cleanup/memory-attempts.jsonl')?.trim().split('\n'), + ).toHaveLength(1) + }) + + it('charges conservative external cost once adapter creation is attempted', async () => { + const storage = inMemoryCampaignStorage() + const costLedger = createRunCostLedger({ + storage, + runDir: '/runs/create-adapter-cost', + costCeilingUsd: 1, + }) + + await expect( + runAgentMemoryExperiment({ + experimentId: 'create-adapter-cost', + sequences: [ + { + id: 'history', + family: 'first-party', + steps: [{ id: 'probe', probes: [{ id: 'probe', query: 'x', referenceAnswer: 'x' }] }], + }, + ], + candidates: [ + { + id: 'paid-memory', + ref: 'paid-memory:v1', + externalCostUsdPerSequence: 0.25, + createAdapter() { + throw new Error('provider setup rejected') + }, + }, + ], + runDir: '/runs/create-adapter-cost', + storage, + costLedger, + }), + ).rejects.toThrow('memory experiment cleanup failed after dispatch') + + expect(costLedger.summary().totalCostUsd).toBe(0.25) + }) + + it('does not reuse cells after the candidate implementation reference changes', async () => { + const storage = inMemoryCampaignStorage() + const sequence = { + id: 'candidate-ref', + family: 'first-party' as const, + steps: [ + { + id: 'remember', + scope: { agentId: 'worker' }, + writes: [{ kind: 'fact' as const, text: 'remember me' }], + probes: [ + { + id: 'recall', + query: 'remember', + requiredFacts: [{ id: 'fact', anyOf: ['remember me'] }], + }, + ], + }, + ], + } + let creates = 0 + const run = (ref: string, visible: boolean) => + runAgentMemoryExperiment({ + experimentId: 'candidate-ref-cache', + sequences: [sequence], + candidates: [ + { + id: 'memory', + ref, + createAdapter() { + creates += 1 + const adapter = createScopedTestAdapter(`memory:${ref}`) + if (visible) return adapter + return { + ...adapter, + async search() { + return [] + }, + } + }, + }, + ], + runDir: '/runs/candidate-ref-cache', + storage, + }) + + const first = await run('memory:v1', true) + const cached = await run('memory:v1', false) + const changed = await run('memory:v2', false) + + expect(first.rows[0]?.scoreMean).toBe(1) + expect(cached.campaign.cells[0]?.cached).toBe(true) + expect(cached.rows[0]?.scoreMean).toBe(1) + expect(changed.campaign.cells[0]?.cached).toBe(false) + expect(changed.rows[0]?.scoreMean).toBe(0) + expect(creates).toBe(2) }) -}) -describe('agent memory experiments', () => { it('runs existing ordered memory benchmark cases without reshaping the dataset', async () => { const sequences = buildAgentMemorySequencesFromBenchmarkCases([ { @@ -1155,6 +1979,7 @@ describe('agent memory experiments', () => { candidates: [ { id: 'literal-memory', + ref: 'literal-memory:v1', createAdapter: () => createScopedTestAdapter('literal-memory'), }, ], @@ -1225,11 +2050,13 @@ describe('agent memory experiments', () => { candidates: [ { id: 'private', + ref: 'private:v1', policy: { read: ['private'], write: 'private' }, createAdapter: ({ branchId }) => createScopedTestAdapter(`private:${branchId}`), }, { id: 'team', + ref: 'team:v1', policy: { read: ['team'], write: 'team' }, createAdapter: ({ branchId }) => createScopedTestAdapter(`team:${branchId}`), }, @@ -1298,6 +2125,7 @@ describe('agent memory experiments', () => { candidates: [ { id: 'memory', + ref: 'memory:v1', createAdapter({ branchId }) { branchIds.push(branchId) return createScopedTestAdapter(branchId) @@ -1315,7 +2143,7 @@ describe('agent memory experiments', () => { expect(branchIds[0]).not.toBe(branchIds[1]) }) - it('uses a stable external branch id across worker paths when experimentRunId matches', async () => { + it('uses a fresh external branch id for each distributed execution attempt', async () => { const branchIds: string[] = [] const run = (runDir: string) => runAgentMemoryExperiment({ @@ -1337,6 +2165,7 @@ describe('agent memory experiments', () => { candidates: [ { id: 'memory', + ref: 'memory:v1', createAdapter({ branchId }) { branchIds.push(branchId) return createScopedTestAdapter(branchId) @@ -1351,7 +2180,7 @@ describe('agent memory experiments', () => { await run('/worker-b/run') expect(branchIds).toHaveLength(2) - expect(branchIds[0]).toBe(branchIds[1]) + expect(branchIds[0]).not.toBe(branchIds[1]) }) it('waits for timed-out provider work to finish cleanup before returning', async () => { @@ -1407,7 +2236,7 @@ describe('agent memory experiments', () => { ], }, ], - candidates: [{ id: 'slow', createAdapter: () => adapter }], + candidates: [{ id: 'slow', ref: 'slow:v1', createAdapter: () => adapter }], runDir: '/runs/timeout-cleanup', storage: inMemoryCampaignStorage(), dispatchTimeoutMs: 5, @@ -1484,7 +2313,7 @@ describe('agent memory experiments', () => { ], }, ], - candidates: [{ id: 'slow', createAdapter: () => adapter }], + candidates: [{ id: 'slow', ref: 'slow:v1', createAdapter: () => adapter }], runDir: '/runs/timeout-cleanup-failure', storage: inMemoryCampaignStorage(), dispatchTimeoutMs: 5, @@ -1496,6 +2325,457 @@ describe('agent memory experiments', () => { await expect(run).rejects.toThrow('memory experiment cleanup failed after dispatch') }) + it('recovers an unfinished provider branch before retrying the history', async () => { + const storage = inMemoryCampaignStorage() + const rows = new Map() + const operations: string[] = [] + const branchIds: Record<'first' | 'recovery' | 'retry', string | undefined> = { + first: undefined, + recovery: undefined, + retry: undefined, + } + let firstExecution = true + const sequence = { + id: 'recover-history', + family: 'first-party' as const, + steps: [ + { + id: 'remember', + scope: { agentId: 'worker' }, + writes: [{ kind: 'fact' as const, text: 'sensitive provider fact' }], + probes: [ + { + id: 'recall', + query: 'provider fact', + referenceAnswer: 'sensitive provider fact', + }, + ], + }, + ], + } + const candidate = { + id: 'recoverable', + ref: 'recoverable:v1', + externalRecoveryCostUsdPerAttempt: 0.1, + createAdapter({ branchId, purpose }: { branchId: string; purpose: 'execute' | 'recovery' }) { + const executionLabel = + purpose === 'recovery' ? 'recovery' : firstExecution ? 'first' : 'retry' + branchIds[executionLabel] = branchId + operations.push(`create:${executionLabel}`) + let clearCalls = 0 + const adapter: AgentMemoryAdapter = { + id: 'recoverable-provider', + branchIsolation: { mode: 'scoped' }, + async search(_query, options) { + return [...(rows.get(options.scope?.namespace ?? '') ?? [])] + }, + async getContext(query, options) { + const hits = await adapter.search(query, options) + return { query, text: hits.map(hitText).join('\n'), hits, sourceRecords: [] } + }, + async write(input) { + operations.push(`write:${executionLabel}`) + const namespace = input.scope?.namespace ?? '' + const hit = { + id: input.id ?? `${executionLabel}-fact`, + uri: `memory://recoverable/${executionLabel}`, + kind: input.kind, + text: input.text, + } + rows.set(namespace, [...(rows.get(namespace) ?? []), hit]) + return { accepted: true, id: hit.id, uri: hit.uri, kind: hit.kind } + }, + async clear(scope) { + clearCalls += 1 + operations.push(`clear:${executionLabel}`) + if (executionLabel === 'first' && clearCalls > 1) { + throw new Error('provider cleanup unavailable') + } + rows.delete(scope?.namespace ?? '') + }, + } + if (purpose === 'execute') firstExecution = false + return adapter + }, + } + const run = () => + runAgentMemoryExperiment({ + experimentId: 'restart-recovery', + sequences: [sequence], + candidates: [candidate], + runDir: '/runs/restart-recovery', + storage, + }) + + await expect(run()).rejects.toThrow('memory experiment cleanup failed after dispatch') + expect(rows.size).toBe(1) + + const result = await run() + + expect(result.rows[0]).toMatchObject({ candidateId: 'recoverable', cellsFailed: 0 }) + expect(result.rows[0]?.totalCostUsd).toBe(0.1) + expect(result.campaign.aggregates.totalCostUsd).toBe(0.1) + expect(branchIds.recovery).toBe(branchIds.first) + expect(branchIds.retry).not.toBe(branchIds.first) + expect(operations.indexOf('clear:recovery')).toBeLessThan(operations.indexOf('write:retry')) + expect(rows.size).toBe(0) + const attemptEvents = storage + .read(result.attemptLogPath)! + .trim() + .split('\n') + .map((line) => JSON.parse(line) as { status: string; recovery: boolean }) + expect(attemptEvents.map(({ status, recovery }) => ({ status, recovery }))).toEqual([ + { status: 'started', recovery: false }, + { status: 'cleaned', recovery: true }, + { status: 'started', recovery: false }, + { status: 'cleaned', recovery: false }, + ]) + }) + + it('recovers independent abandoned branches in parallel before retrying them', async () => { + const storage = inMemoryCampaignStorage() + let phase: 'leave-active' | 'recover' = 'leave-active' + let activeRecoveries = 0 + let maxActiveRecoveries = 0 + let releaseRecoveries: (() => void) | undefined + const recoveriesStarted = new Promise((resolve) => { + releaseRecoveries = resolve + }) + const sequences = ['one', 'two'].map((id) => ({ + id, + family: 'first-party' as const, + steps: [ + { + id: 'remember', + scope: { agentId: 'worker' }, + writes: [{ kind: 'fact' as const, text: `${id} fact` }], + probes: [{ id: 'recall', query: id, referenceAnswer: `${id} fact` }], + }, + ], + })) + const run = () => + runAgentMemoryExperiment({ + experimentId: 'parallel-recovery', + sequences, + candidates: [ + { + id: 'memory', + ref: 'memory:v1', + async createAdapter({ purpose }) { + if (purpose === 'recovery') { + activeRecoveries += 1 + maxActiveRecoveries = Math.max(maxActiveRecoveries, activeRecoveries) + if (activeRecoveries === 2) releaseRecoveries?.() + await recoveriesStarted + activeRecoveries -= 1 + return null + } + if (phase === 'leave-active') return null + return createScopedTestAdapter('parallel-recovery') + }, + }, + ], + runDir: '/runs/parallel-recovery', + storage, + maxConcurrency: 2, + }) + + await expect(run()).rejects.toThrow('memory experiment cleanup failed after dispatch') + phase = 'recover' + const result = await run() + + expect(maxActiveRecoveries).toBe(2) + expect(result.rows[0]).toMatchObject({ candidateId: 'memory', cellsFailed: 0 }) + const recovered = storage + .read(result.attemptLogPath)! + .trim() + .split('\n') + .map((line) => JSON.parse(line) as { status: string; recovery: boolean }) + .filter((event) => event.status === 'cleaned' && event.recovery) + expect(recovered).toHaveLength(2) + }) + + it('closes an attempt whose provider was never created before retrying it', async () => { + const storage = inMemoryCampaignStorage() + const purposes: Array<'execute' | 'recovery'> = [] + let executionCalls = 0 + const sequence = { + id: 'provider-creation', + family: 'first-party' as const, + steps: [ + { + id: 'remember', + scope: { agentId: 'worker' }, + writes: [{ kind: 'fact' as const, text: 'created provider fact' }], + probes: [ + { + id: 'recall', + query: 'provider fact', + referenceAnswer: 'created provider fact', + }, + ], + }, + ], + } + const run = () => + runAgentMemoryExperiment({ + experimentId: 'provider-creation-recovery', + sequences: [sequence], + candidates: [ + { + id: 'sometimes-created', + ref: 'sometimes-created:v1', + createAdapter({ purpose }) { + purposes.push(purpose) + if (purpose === 'recovery') return null + executionCalls += 1 + if (executionCalls === 1) return null + return createScopedTestAdapter('created-provider') + }, + }, + ], + runDir: '/runs/provider-creation-recovery', + storage, + }) + + await expect(run()).rejects.toThrow('memory experiment cleanup failed after dispatch') + const result = await run() + + expect(result.rows[0]).toMatchObject({ candidateId: 'sometimes-created', cellsFailed: 0 }) + expect(purposes).toEqual(['execute', 'recovery', 'execute']) + const attemptEvents = storage + .read(result.attemptLogPath)! + .trim() + .split('\n') + .map((line) => JSON.parse(line) as { status: string; recovery: boolean }) + expect(attemptEvents.map(({ status, recovery }) => ({ status, recovery }))).toEqual([ + { status: 'started', recovery: false }, + { status: 'cleaned', recovery: true }, + { status: 'started', recovery: false }, + { status: 'cleaned', recovery: false }, + ]) + }) + + it('allows one controller per run while its history workers run in parallel', async () => { + const storage = inMemoryCampaignStorage() + let reportStarted: (() => void) | undefined + const started = new Promise((resolve) => { + reportStarted = resolve + }) + let releaseWorker: (() => void) | undefined + const continueWorker = new Promise((resolve) => { + releaseWorker = resolve + }) + const options: RunAgentMemoryExperimentOptions = { + experimentId: 'single-controller', + sequences: [ + { + id: 'one', + family: 'first-party', + steps: [ + { + id: 'work', + scope: { agentId: 'worker' }, + writes: [{ kind: 'fact', text: 'parallel worker fact' }], + probes: [ + { + id: 'recall', + query: 'worker fact', + referenceAnswer: 'parallel worker fact', + }, + ], + }, + ], + }, + ], + candidates: [ + { + id: 'memory', + ref: 'memory:v1', + createAdapter: () => createScopedTestAdapter('single-controller'), + }, + ], + runDir: '/runs/single-controller', + storage, + executeStepRef: 'blocking-step:v1', + async executeStep() { + reportStarted?.() + await continueWorker + }, + } + + const firstRun = runAgentMemoryExperiment(options) + await started + await expect( + runAgentMemoryExperiment({ ...options, storage: inMemoryCampaignStorage() }), + ).rejects.toThrow('active controller') + releaseWorker?.() + const result = await firstRun + + expect(result.rows[0]).toMatchObject({ candidateId: 'memory', cellsFailed: 0 }) + }) + + it('leaves a lost controller branch for the next owner and accepts duplicate cleanup receipts', async () => { + const storage = inMemoryCampaignStorage() + let firstControllerOwned = true + let expireFirstController = true + let controllerCount = 0 + let searches = 0 + let clears = 0 + let closes = 0 + let factVisible = false + const purposes: string[] = [] + const options: RunAgentMemoryExperimentOptions = { + experimentId: 'lost-controller', + sequences: [ + { + id: 'one', + family: 'first-party', + steps: [ + { + id: 'work', + scope: { agentId: 'worker' }, + writes: [{ kind: 'fact', text: 'must be cleaned' }], + probes: [{ id: 'probe', query: 'must', referenceAnswer: 'must be cleaned' }], + }, + ], + }, + ], + candidates: [ + { + id: 'memory', + ref: 'memory:v1', + createAdapter({ purpose }) { + purposes.push(purpose) + const adapter: AgentMemoryAdapter = { + id: 'lost-controller', + branchIsolation: { mode: 'scoped' }, + async search() { + searches += 1 + return factVisible + ? [ + { + id: 'write', + uri: 'memory://lost-controller/write', + kind: 'fact', + text: 'must be cleaned', + }, + ] + : [] + }, + async getContext(query, searchOptions) { + const hits = await adapter.search(query, searchOptions) + return { query, text: hits.map(hitText).join('\n'), hits, sourceRecords: [] } + }, + async write(input) { + factVisible = true + return { + accepted: true, + id: 'write', + uri: 'memory://lost-controller/write', + kind: input.kind, + } + }, + async clear() { + clears += 1 + factVisible = false + }, + async close() { + closes += 1 + }, + } + return adapter + }, + }, + ], + runDir: '/runs/lost-controller', + storage, + acquireRunLease: async () => { + controllerCount += 1 + const controller = controllerCount + return { + assertOwned() { + if (controller === 1 && !firstControllerOwned) { + throw new Error('controller ownership expired') + } + }, + release() {}, + } + }, + executeStepRef: 'expire-controller:v1', + async executeStep() { + if (expireFirstController) { + expireFirstController = false + firstControllerOwned = false + } + }, + } + const run = () => runAgentMemoryExperiment(options) + + await expect(run()).rejects.toThrow('controller ownership expired') + expect(searches).toBe(0) + expect(clears).toBe(1) + expect(closes).toBe(1) + expect(factVisible).toBe(true) + + const result = await run() + expect(result.rows[0]).toMatchObject({ candidateId: 'memory', cellsFailed: 0 }) + expect(purposes).toEqual(['execute', 'recovery', 'execute']) + expect(clears).toBe(4) + expect(closes).toBe(3) + expect(factVisible).toBe(false) + + const journal = storage.read(result.attemptLogPath)! + const lines = journal.trim().split('\n') + const last = lines.at(-1)! + expect(storage.append!(result.attemptLogPath, `${last}\n`, Buffer.byteLength(journal))).toBe( + Buffer.byteLength(journal) + Buffer.byteLength(`${last}\n`), + ) + const cached = await run() + expect(cached.campaign.aggregates.cellsCached).toBe(1) + }) + + it('preserves both the run failure and controller release failure', async () => { + const error = await runAgentMemoryExperiment({ + experimentId: 'run-and-release-failure', + sequences: [ + { + id: 'one', + family: 'first-party', + steps: [ + { + id: 'probe', + probes: [{ id: 'probe', query: 'state', referenceAnswer: 'state' }], + }, + ], + }, + ], + candidates: [ + { + id: 'unused', + ref: 'unused:v1', + createAdapter: () => createScopedTestAdapter('unused'), + }, + ], + runDir: '/runs/run-and-release-failure', + storage: inMemoryCampaignStorage(), + acquireRunLease: async () => ({ + assertOwned() { + throw new Error('run ownership failed') + }, + release() { + throw new Error('release failed') + }, + }), + }).catch((caught: unknown) => caught) + + expect(error).toBeInstanceOf(AggregateError) + expect((error as AggregateError).errors.map((item) => String(item))).toEqual([ + 'Error: run ownership failed', + 'Error: release failed', + ]) + }) + it('uses the same random seed for every candidate on one history and repetition', async () => { const seeds = new Map() const sequence = { @@ -1514,6 +2794,7 @@ describe('agent memory experiments', () => { sequences: [sequence], candidates: ['a', 'b'].map((candidateId) => ({ id: candidateId, + ref: `${candidateId}:v1`, createAdapter: ({ sequence: candidateSequence, rep, seed }) => { seeds.set(`${candidateId}:${candidateSequence.id}:${rep}`, seed) return createScopedTestAdapter(`${candidateId}:${rep}`) @@ -1530,11 +2811,11 @@ describe('agent memory experiments', () => { expect(seeds.get('a:paired-history:0')).not.toBe(seeds.get('a:paired-history:1')) }) - it('clears accepted writes and disposes the adapter after a failed step', async () => { + it('fails the experiment when accepted writes cannot be cleared after a failed step', async () => { let clears = 0 let closes = 0 let disposals = 0 - const result = await runAgentMemoryExperiment({ + const run = runAgentMemoryExperiment({ experimentId: 'failed-step-cleanup', sequences: [ { @@ -1553,6 +2834,7 @@ describe('agent memory experiments', () => { candidates: [ { id: 'scoped', + ref: 'scoped:v1', createAdapter() { const base = createScopedTestAdapter('failure') return { @@ -1581,10 +2863,7 @@ describe('agent memory experiments', () => { }, }) - expect(result.rows[0]).toMatchObject({ cellsFailed: 1 }) - expect(result.campaign.cells[0]?.error).toContain( - 'memory branch cleanup failed after: worker failed', - ) + await expect(run).rejects.toThrow('memory experiment cleanup failed after dispatch') expect(clears).toBe(2) expect(closes).toBe(1) expect(disposals).toBe(1) @@ -1612,6 +2891,7 @@ describe('agent memory experiments', () => { candidates: [ { id: 'side-effect-then-error', + ref: 'side-effect-then-error:v1', createAdapter: () => ({ id: 'side-effect-then-error', branchIsolation: { mode: 'scoped' }, @@ -1633,6 +2913,7 @@ describe('agent memory experiments', () => { }, { id: 'complete', + ref: 'complete:v1', createAdapter: () => createScopedTestAdapter('complete'), }, ], @@ -1678,6 +2959,7 @@ describe('agent memory experiments', () => { candidates: [ { id: 'legacy', + ref: 'legacy:v1', createAdapter() { const { branchIsolation: _branchIsolation, ...legacy } = createScopedTestAdapter('legacy') @@ -1724,6 +3006,7 @@ describe('agent memory experiments', () => { candidates: [ { id: 'no-clear', + ref: 'no-clear:v1', createAdapter() { const { clear: _clear, ...adapter } = createScopedTestAdapter('no-clear') return adapter @@ -1763,6 +3046,7 @@ describe('agent memory experiments', () => { candidates: [ { id: 'scoped', + ref: 'scoped:v1', createAdapter: () => createScopedTestAdapter('scoped', async () => { providerWrites += 1 @@ -1791,38 +3075,39 @@ describe('agent memory experiments', () => { it('requires an external-state disposer when scoped cleanup is disabled', async () => { let creates = 0 - const result = await runAgentMemoryExperiment({ - experimentId: 'missing-disposer', - sequences: [ - { - id: 'one', - family: 'first-party', - steps: [ - { - id: 'probe', - scope: { agentId: 'worker' }, - probes: [{ id: 'state', query: 'state', referenceAnswer: 'state' }], + await expect( + runAgentMemoryExperiment({ + experimentId: 'missing-disposer', + sequences: [ + { + id: 'one', + family: 'first-party', + steps: [ + { + id: 'probe', + scope: { agentId: 'worker' }, + probes: [{ id: 'state', query: 'state', referenceAnswer: 'state' }], + }, + ], + }, + ], + candidates: [ + { + id: 'persistent', + ref: 'persistent:v1', + createAdapter() { + creates += 1 + return createScopedTestAdapter('persistent') }, - ], - }, - ], - candidates: [ - { - id: 'persistent', - createAdapter() { - creates += 1 - return createScopedTestAdapter('persistent') }, - }, - ], - runDir: '/runs/missing-disposer', - storage: inMemoryCampaignStorage(), - cleanupBranches: false, - }) + ], + runDir: '/runs/missing-disposer', + storage: inMemoryCampaignStorage(), + cleanupBranches: false, + }), + ).rejects.toThrow('requires disposeAdapter') expect(creates).toBe(0) - expect(result.rows[0]).toMatchObject({ cellsFailed: 1, scoreMean: 0 }) - expect(result.campaign.cells[0]?.error).toContain('requires disposeAdapter') }) it('rejects probes that cannot distinguish a useful memory system', async () => { @@ -1842,7 +3127,13 @@ describe('agent memory experiments', () => { ], }, ], - candidates: [{ id: 'memory', createAdapter: () => createScopedTestAdapter('memory') }], + candidates: [ + { + id: 'memory', + ref: 'memory:v1', + createAdapter: () => createScopedTestAdapter('memory'), + }, + ], runDir: '/runs/no-target', storage: inMemoryCampaignStorage(), }), @@ -1851,6 +3142,32 @@ describe('agent memory experiments', () => { }) describe('agent memory improvement', () => { + it('requires an explicit controller policy for custom improvement storage', async () => { + await expect( + runAgentMemoryImprovementRaw({ + experimentId: 'custom-improvement-storage', + trainSequences: [improvementSequence('train', 'train')], + holdoutSequences: [improvementSequence('holdout', 'holdout')], + seeds: [ + { + config: { mode: 'baseline' }, + track: 'baseline', + proposer: 'default', + }, + ], + createCandidate: () => ({ + ref: 'memory:v1', + createAdapter: () => createScopedTestAdapter('memory'), + }), + proposer: { kind: 'noop', propose: async () => [] }, + improvementRef: 'custom-improvement-storage:v1', + budget: { maxSteps: 1 }, + runDir: '/runs/custom-improvement-storage', + storage: inMemoryCampaignStorage(), + }), + ).rejects.toThrow("requires acquireRunLease or controllerMode='process-local'") + }) + it('searches isolated configs and activates only a fresh holdout win', async () => { type Config = { visibility: 'private' | 'team' | 'shared' } const storage = inMemoryCampaignStorage() @@ -1916,6 +3233,7 @@ describe('agent memory improvement', () => { createCandidate: ({ config, candidateId }) => { if (config.visibility !== 'private') contenderIds.add(candidateId) return { + ref: `visibility:${config.visibility}:v1`, label: config.visibility, policy: { read: [config.visibility], write: config.visibility }, createAdapter: ({ branchId }) => createScopedTestAdapter(`${candidateId}:${branchId}`), @@ -1949,9 +3267,9 @@ describe('agent memory improvement', () => { expect(result.decision).toMatchObject({ status: 'promote', reasons: [], - baselineScore: 0, + baselineScore: 0.25, winnerScore: 1, - lift: 1, + lift: 0.75, }) expect(result.decision.significance).toMatchObject({ n: 2, significant: true }) expect(result.winnerConfig).toEqual({ visibility: 'team' }) @@ -1960,6 +3278,10 @@ describe('agent memory improvement', () => { expect(promoted).toEqual([{ visibility: 'team' }]) expect(result.activation).toMatchObject({ status: 'activated' }) expect(storage.read(result.resultJsonPath)).toContain('"status": "promote"') + expect( + JSON.parse(storage.read('/runs/improve-team-memory/memory-improvement-manifest.json') ?? '{}') + .identity?.schema, + ).toBe(4) const resumed = await runAgentMemoryImprovement(options) expect(proposalCalls).toBe(1) @@ -2045,6 +3367,7 @@ describe('agent memory improvement', () => { runDir: '/runs/named-track-proposers', storage: inMemoryCampaignStorage(), createCandidate: ({ config, candidateId }) => ({ + ref: `visibility:${config.visibility}:v1`, policy: { read: [config.visibility], write: config.visibility }, createAdapter: ({ branchId }) => createScopedTestAdapter(`${candidateId}:${branchId}`), }), @@ -2064,15 +3387,15 @@ describe('agent memory improvement', () => { expect(governorHasCostLedger).toBe(true) }) - it('holds a winner when a configured critical dimension is missing', async () => { + it('holds a winner when holdout histories do not test a critical dimension', async () => { type Config = { visibility: 'private' | 'team' } const result = await runAgentMemoryImprovement({ experimentId: 'missing-critical-dimension', trainSequences: [improvementSequence('critical-train', 'train')], holdoutSequences: [ - improvementSequence('critical-holdout-a', 'holdout'), - improvementSequence('critical-holdout-b', 'holdout'), - improvementSequence('critical-holdout-c', 'holdout'), + improvementSequence('critical-holdout-a', 'holdout', false), + improvementSequence('critical-holdout-b', 'holdout', false), + improvementSequence('critical-holdout-c', 'holdout', false), ], seeds: [{ config: { visibility: 'private' }, track: 'baseline', proposer: 'sharing' }], proposer: { @@ -2087,8 +3410,9 @@ describe('agent memory improvement', () => { runDir: '/runs/missing-critical-dimension', storage: inMemoryCampaignStorage(), significance: { minProductiveRuns: 1, resamples: 100, seed: 9 }, - criticalDimensions: ['missing_safety'], + criticalDimensions: ['memory_stale_safe'], createCandidate: ({ config, candidateId }) => ({ + ref: `visibility:${config.visibility}:v1`, policy: { read: [config.visibility], write: config.visibility }, createAdapter: ({ branchId }) => createScopedTestAdapter(`${candidateId}:${branchId}`), }), @@ -2096,10 +3420,15 @@ describe('agent memory improvement', () => { expect(result.decision.status).toBe('hold') expect(result.decision.criticalDimensions).toEqual([ - expect.objectContaining({ dimension: 'missing_safety', n: 0, measured: false }), + expect.objectContaining({ + dimension: 'memory_stale_safe', + n: 0, + expectedN: 0, + measured: false, + }), ]) expect(result.decision.reasons).toContain( - 'critical dimension missing_safety was measured on 0/3 paired holdout cells', + 'critical dimension memory_stale_safe has no applicable holdout histories', ) }) @@ -2150,6 +3479,7 @@ describe('agent memory improvement', () => { runDir: '/runs/proposer-cost-limit', storage: inMemoryCampaignStorage(), createCandidate: ({ config, candidateId }) => ({ + ref: `visibility:${config.visibility}:v1`, policy: { read: [config.visibility], write: config.visibility }, createAdapter: ({ branchId }) => createScopedTestAdapter(`${candidateId}:${branchId}`), }), @@ -2177,6 +3507,7 @@ describe('agent memory improvement', () => { runDir: '/runs/overlap', storage: inMemoryCampaignStorage(), createCandidate: () => ({ + ref: 'unused:v1', createAdapter: () => createScopedTestAdapter('unused'), }), }), @@ -2202,6 +3533,7 @@ describe('agent memory improvement', () => { runDir: '/runs/renamed-overlap', storage: inMemoryCampaignStorage(), createCandidate: () => ({ + ref: 'unused:v1', createAdapter: () => createScopedTestAdapter('unused'), }), }), @@ -2209,7 +3541,7 @@ describe('agent memory improvement', () => { }) }) -function improvementSequence(id: string, split: 'train' | 'holdout') { +function improvementSequence(id: string, split: 'train' | 'holdout', includeStaleTarget = true) { return { id, family: 'first-party' as const, @@ -2235,6 +3567,17 @@ function improvementSequence(id: string, split: 'train' | 'holdout') { id: 'launch-date', query: `${id} launch date`, requiredFacts: [{ id: 'current', anyOf: [`${id} launch date is Friday`] }], + ...(includeStaleTarget + ? { + forbiddenFacts: [ + { + id: 'stale', + anyOf: [`${id} launch date is Thursday`], + obsolete: true, + }, + ], + } + : {}), expectedEventIds: [`${id}-event`], expectedActorIds: ['researcher'], }, From 06f13a4bdb32a7c5bf4d303a3dfda8a6557b7721 Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Fri, 17 Jul 2026 22:51:08 -0600 Subject: [PATCH 3/4] fix(memory): harden parallel resume and activation --- CHANGELOG.md | 18 + README.md | 47 ++- package.json | 5 +- pnpm-lock.yaml | 137 +------ src/benchmarks/index.ts | 380 ++++++++++++----- src/memory/attempt-log.ts | 123 +++++- src/memory/branch.ts | 12 +- src/memory/experiment.ts | 342 +++++++++++++--- src/memory/graphiti.ts | 13 +- src/memory/improvement.ts | 345 +++++++++++++--- src/memory/lifecycle.ts | 8 +- src/memory/mem0.ts | 126 +++--- src/memory/neo4j.ts | 63 ++- tests/benchmarks.test.ts | 124 +++++- tests/memory-systems.test.ts | 770 ++++++++++++++++++++++++++++++++++- 15 files changed, 2064 insertions(+), 449 deletions(-) create mode 100644 CHANGELOG.md diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..188041f --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,18 @@ +# Changelog + +## 4.0.0 + +### Breaking Changes + +- `runAgentMemoryImprovement()` now activates a measured winner through `activation.readCurrent()` and atomic `activation.compareAndSet()` instead of `onPromote`. +- `AgentMemoryActivation.receiptPath` is now `journalPath` because the file is an append-only activation record. +- `mem0MemoryAdapterIdentity()` and `graphitiMemoryAdapterIdentity()` require a stable, non-secret `backendRef` so two deployments cannot share a cache identity. +- Paid benchmark and improvement work now defaults to a zero dollar limit and requires an explicit `costCeiling` or `maxTotalCostUsd`. + +### Added + +- Official-client adapters for Mem0 hosted and open-source deployments, Graphiti MCP, and Neo4j Agent Memory. +- Isolated memory branches with snapshots, replayable forks, private, team, and shared visibility, and ordered writes per agent. +- Parallel multi-track memory experiments and configuration search on `agent-eval`, with fresh-history comparison before activation. +- Durable controller ownership, interrupted-attempt cleanup, retired-candidate recovery, bounded cleanup work, and conservative recovery cost reconciliation. +- Exact scoped Mem0 deletion with list and search convergence checks. diff --git a/README.md b/README.md index 46c8347..b7e5564 100644 --- a/README.md +++ b/README.md @@ -132,7 +132,7 @@ from `@tangle-network/agent-knowledge`. - Zod schemas define the stable wire shape. - Graph/search/lint are deterministic and fast. - `searchKnowledge` returns hits with three score fields. `score` and - `rrfScore` are the raw reciprocal-rank-fusion value (typically 0.01–0.05); + `rrfScore` are the raw reciprocal-rank-fusion value (typically 0.01 to 0.05); use them when intent matters or when fusing across queries. `normalizedScore` is the same value scaled into [0, 1] relative to the top hit *in this result set* (top hit = 1, others = score / topScore). Use it @@ -192,6 +192,7 @@ await runKnowledgeBenchmarkSuite({ const memoryRanking = await runMemoryAdapterBenchmark({ cases: buildFirstPartyMemoryLifecycleBenchmarkCases(), runDir: '.agent-knowledge/benchmark-runs/memory-ranking', + costCeiling: 5, candidates: [ { id: 'no-memory', ref: 'no-memory:v1', createAdapter: () => createNoopMemoryBenchmarkAdapter() }, { @@ -218,6 +219,7 @@ const result = await runKnowledgeBenchmarkSuite({ cases, runDir: '.agent-knowledge/benchmark-runs/beir-nfcorpus-smoke', respondRef: 'product-retriever:v3', + costCeiling: 5, respond: async ({ case: testCase }) => { if (testCase.taskKind !== 'retrieval') return { hits: [] } const hits = await retrieveFromYourKb(testCase.query) @@ -237,6 +239,8 @@ Use `runAgentMemoryExperiment()` below when an agent profile needs ordered multi Set `cleanupTimeoutMs` to bound each provider cleanup and close operation; the default is 180 seconds. Set `costUsdPerCase` for normal provider work and `recoveryCostUsdPerAttempt` when reconnecting and deleting an interrupted case can add provider charges. One `costCeiling` covers every candidate and recovery call in the comparison, and `totalCostUsd` reports the complete comparison cost. +The default `costCeiling` is zero, so paid work is refused until the caller sets a limit. +Keep removed implementations in `recoveryCandidates` until their unfinished scopes are gone, and use `maxRecoveryAttempts` to reject an unexpectedly large backlog before provider work starts. The default filesystem storage uses an OS lock; custom storage requires `controllerMode: 'process-local'` or a distributed `acquireRunLease` implementation. Billable responders must execute paid work through `context.cost.runPaidCall()`; `costUsd` on an artifact is display-only. @@ -458,6 +462,7 @@ if (!mem0ApiKey) throw new Error('MEM0_API_KEY is required') const mem0Hosted = createMem0MemoryAdapter({ client: mem0Client, mode: 'hosted', + backendRef: 'mem0:production-account', consistency: 'visible', async getEvent(eventId) { const response = await fetch( @@ -472,10 +477,12 @@ const mem0Hosted = createMem0MemoryAdapter({ const mem0Oss = createMem0MemoryAdapter({ client: mem0OssClient, mode: 'oss', + backendRef: 'mem0:local-postgres', }) const graphiti = createGraphitiMemoryAdapter({ client: graphitiMcpClient, + backendRef: 'graphiti:production-cluster', }) const neo4j = createNeo4jAgentMemoryAdapter({ @@ -491,6 +498,9 @@ The adapter never retries a write under another name because that could enqueue Visible writes expand the episode scan when Graphiti truncates a long group and fail explicitly at `episodeScanLimit` instead of reporting a false success. Hosted Mem0 also defaults to visible writes and polls the official event endpoint until the add succeeds or fails. The current JavaScript SDK does not expose that endpoint, so hosted mode accepts the small `getEvent` callback shown above. +Mem0 cleanup requires `getAll` plus per-memory `delete`, refuses an empty identity scope, and waits until both list and search results stop returning deleted IDs. +The adapter does not call Mem0's account-wide `deleteAll` method. +Use a stable, non-secret `backendRef` in Mem0 and Graphiti adapter identities so caches and dispatch keys cannot alias two accounts or clusters. Each adapter uses the same `search`, `getContext`, and `write` method shape, but provider capabilities remain explicit. @@ -584,6 +594,8 @@ Private writes stay visible only to one agent. Team writes are shared inside one team. Shared writes are visible to every agent in the branch. `fork()` replays accepted writes into a child branch, including a branch backed by a different provider. +One adapter object can have only one live handle for a given branch ID in a process. +Close the prior handle before resuming it, and use the run lease to prevent distributed controllers from owning the same durable run at once. Use `runAgentMemoryExperiment()` to compare known providers or configurations across ordered histories. `buildAgentMemorySequencesFromBenchmarkCases()` feeds existing LoCoMo, LongMemEval, GroupMemBench, or first-party memory cases into the same runner without changing their event order. @@ -623,7 +635,22 @@ const result = await runAgentMemoryImprovement({ runProfileStep({ memory, step, context }), executeStepRef: 'support-profile@8f2c1a7', runDir: '.agent-knowledge/memory-runs/support-v3', - onPromote: ({ config }) => activateMemoryConfig(config), + activation: { + ref: 'config-store:support-memory:v1', + readCurrent: () => configStore.read('support-memory'), + compareAndSet: ({ + activationId, + expectedSurfaceHash, + config, + surfaceHash, + }) => configStore.compareAndSet({ + key: 'support-memory', + operationId: activationId, + expectedHash: expectedSurfaceHash, + nextHash: surfaceHash, + value: config, + }), + }, }) ``` @@ -632,16 +659,22 @@ const result = await runAgentMemoryImprovement({ Every experiment candidate must set `ref` to a version or commit that changes whenever its provider configuration or adapter behavior changes. The first seed is always the deployed baseline. Each seed starts an independent track with its own objective and optional proposer, and a branch inherits its parent track's proposer unless the governor chooses another one. -Set each candidate's `externalCostUsdPerSequence` to a conservative memory-provider charge; it is counted once provider adapter creation is attempted because a rejected factory may already have contacted the provider. +Candidate factories must be side-effect free. +Set each candidate's `externalCostUsdPerSequence` to a conservative memory-provider charge; it is counted once an adapter method attempts provider work, while a factory failure costs zero. Set `externalRecoveryCostUsdPerAttempt` when reconnecting and deleting an interrupted history adds a separate provider charge. Recovery uses the same durable run-wide cost account as candidate execution, proposers, profile steps, and the governor, so a resumed run cannot exceed `maxTotalCostUsd` by treating cleanup as free work. +If a process exits after paid cleanup but before recording its receipt, resume charges the reserved maximum, repeats the idempotent cleanup, and continues only after the account is complete. +The default `maxTotalCostUsd` is zero, so model and paid provider calls require an explicit limit. Model calls inside `executeStep` should use `context.cost.runPaidCall()` so `maxTotalCostUsd` counts both sources before starting more work. The same budget is supplied to named proposers as `context.costLedger` and to the governor as `context.costLedger`; the standard `agent-eval` proposers charge their model calls there. Steps inside one history remain ordered, and writes from one agent remain ordered while independent agents can write concurrently. Runtime callbacks may write only to scopes declared on a step, write, probe, or `sequence.cleanupScopes`; this lets an interrupted retry clear every actor and session it could have touched. Search never changes the live configuration. -Only `onPromote` can activate the fresh-history winner. -The callback receives a stable `activationId`; use it as the idempotency key when activation updates an external service. +Only `activation.compareAndSet` can activate the fresh-history winner. +The activation target must provide an exact current read and an atomic compare-and-set from the measured baseline to the measured winner. +The durable activation journal is written before that operation. +After a crash, the runner reads the live configuration and records the completed activation without applying it twice. +Change `activation.ref` whenever the target or activation behavior changes. The default filesystem path permits one controller and many parallel workers, persists candidate history, and resumes without adding more than `maxSteps` candidate nodes. Custom storage must set `controllerMode: 'process-local'` when all controllers share one process, or provide `acquireRunLease` for distributed controllers. Independent workers still use `candidateConcurrency` and `sequenceConcurrency`. @@ -653,11 +686,15 @@ The run waits for cleanup to succeed or for `cleanupTimeoutMs` to expire, and ei The runner appends a `started` event before adapter creation and a `cleaned` event only after provider cleanup and disposal succeed. On resume it recovers every unfinished attempt before starting new cells. Independent unfinished attempts recover up to `maxConcurrency` at once, and no new cell starts until every cleanup succeeds. +Recovery refuses more than `maxRecoveryAttempts`, which defaults to 1000. +Pass retired implementations through `recoveryCandidates` until their recorded attempts are cleaned. `createAdapter` receives `purpose: 'recovery'` during that pass so a dedicated-instance candidate can reconnect to the existing branch instead of provisioning another one. It may return `null` during recovery when the recorded attempt never created external state. Hosted Mem0 and Graphiti recovery waits for their configured ingestion timeout before deletion so an accepted asynchronous write can become visible first. +That visibility wait is bounded by `cleanupTimeoutMs`; a longer required delay fails before new provider work starts. The default filesystem storage survives process restarts and uses an OS lock. Custom durable storage must implement atomic `append`; process-local custom storage opts in with `controllerMode: 'process-local'`, and distributed controllers provide `acquireRunLease`. +Provider SDK calls should also have their own network timeout because a caller-side cleanup timeout cannot cancel an arbitrary third-party promise. The persisted run identity includes `budget.maxSteps`; start a new `runDir` to enlarge a completed search instead of mutating its history. Promotion is blocked when a configured critical dimension is missing or incomplete on either fresh-history arm. Improvement runs clear each scoped branch before and after a measured history by default, including timeout and thrown-error cleanup. diff --git a/package.json b/package.json index 626e0c8..e58b7f7 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@tangle-network/agent-knowledge", - "version": "3.2.1", + "version": "4.0.0", "description": "Source-grounded, eval-gated knowledge growth primitives for agents.", "homepage": "https://github.com/tangle-network/agent-knowledge#readme", "repository": { @@ -52,6 +52,7 @@ "dist", "docs", "AGENTS.md", + "CHANGELOG.md", "README.md" ], "publishConfig": { @@ -69,7 +70,7 @@ "verify:package": "node scripts/verify-package.mjs" }, "dependencies": { - "@tangle-network/agent-eval": "^0.122.7", + "@tangle-network/agent-eval": "^0.122.8", "@tangle-network/agent-interface": "^0.31.0", "proper-lockfile": "4.1.2", "zod": "^4.3.6" diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 3aa1d7d..b9f2f2d 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -13,8 +13,8 @@ importers: .: dependencies: '@tangle-network/agent-eval': - specifier: ^0.122.7 - version: 0.122.7(typescript@5.9.3) + specifier: ^0.122.8 + version: 0.122.8(typescript@5.9.3) '@tangle-network/agent-interface': specifier: ^0.31.0 version: 0.31.0 @@ -42,7 +42,7 @@ importers: version: 4.1.4 mem0ai: specifier: 3.1.0 - version: 3.1.0(@anthropic-ai/sdk@0.40.1)(@azure/identity@4.13.1)(@azure/search-documents@12.2.0)(@cloudflare/workers-types@4.20260702.1)(@google/genai@1.52.0)(@langchain/core@1.2.3(@opentelemetry/api@1.9.1)(ws@8.21.0))(@mistralai/mistralai@1.15.1)(@qdrant/js-client-rest@1.18.0(typescript@5.9.3))(@supabase/supabase-js@2.110.7)(@types/jest@29.5.14)(@types/pg@8.11.0)(better-sqlite3@12.11.1)(cloudflare@4.5.0)(compromise@14.16.0)(groq-sdk@0.3.0)(mongodb@7.2.0)(natural@8.1.1(@opentelemetry/api@1.9.1))(ollama@0.5.18)(pg@8.11.3)(redis@4.7.1)(ws@8.21.0) + version: 3.1.0(@anthropic-ai/sdk@0.40.1)(@azure/identity@4.13.1)(@azure/search-documents@12.2.0)(@cloudflare/workers-types@4.20260702.1)(@google/genai@1.52.0)(@langchain/core@1.2.3(@opentelemetry/api@1.9.1)(ws@8.21.0))(@mistralai/mistralai@1.15.1)(@qdrant/js-client-rest@1.18.0(typescript@5.9.3))(@supabase/supabase-js@2.110.7)(@types/jest@29.5.14)(@types/pg@8.11.0)(better-sqlite3@12.11.1)(cloudflare@4.5.0)(compromise@14.16.0)(groq-sdk@0.3.0)(mongodb@7.2.0)(natural@8.1.1(@opentelemetry/api@1.9.1))(ollama@0.5.18)(pg@8.22.0)(redis@5.12.1(@opentelemetry/api@1.9.1))(ws@8.21.0) tsup: specifier: ^8.0.0 version: 8.5.1(postcss@8.5.13)(tsx@4.22.4)(typescript@5.9.3)(yaml@2.8.4) @@ -631,21 +631,12 @@ packages: resolution: {integrity: sha512-oQG/FejNpItrxRHoyctYvT3rwGZOnK4jr3JdppO/c78ktDvkWiPXPHNsrDf33K9sZdRb6PR7gi4noIapu5q4HA==} engines: {node: '>=18.0.0', pnpm: '>=8'} - '@redis/bloom@1.2.0': - resolution: {integrity: sha512-HG2DFjYKbpNmVXsa0keLHp/3leGJz1mjh09f2RLGGLQZzSHpkmZWuwJbAvo3QcRY8p80m5+ZdXZdYOSBLlp7Cg==} - peerDependencies: - '@redis/client': ^1.0.0 - '@redis/bloom@5.12.1': resolution: {integrity: sha512-PUUfv+ms7jgPSBVoo/DN4AkPHj4D5TZSd6SbJX7egzBplkYUcKmHRE8RKia7UtZ8bSQbLguLvxVO+asKtQfZWA==} engines: {node: '>= 18.19.0'} peerDependencies: '@redis/client': ^5.12.1 - '@redis/client@1.6.1': - resolution: {integrity: sha512-/KCsg3xSlR+nCK8/8ZYSknYxvXHwubJrU82F3Lm1Fp6789VQ0/3RJKfsmRXjqfaTA++23CvC3hqmqe/2GEt6Kw==} - engines: {node: '>=14'} - '@redis/client@5.12.1': resolution: {integrity: sha512-7aPGWeqA3uFm43o19umzdl16CEjK/JQGtSXVPevplTaOU3VJA/rseBC1QvYUz9lLDIMBimc4SW/zrW4S89BaCA==} engines: {node: '>= 18.19.0'} @@ -658,38 +649,18 @@ packages: '@opentelemetry/api': optional: true - '@redis/graph@1.1.1': - resolution: {integrity: sha512-FEMTcTHZozZciLRl6GiiIB4zGm5z5F3F6a6FZCyrfxdKOhFlGkiAqlexWMBzCi4DcRoyiOsuLfW+cjlGWyExOw==} - peerDependencies: - '@redis/client': ^1.0.0 - - '@redis/json@1.0.7': - resolution: {integrity: sha512-6UyXfjVaTBTJtKNG4/9Z8PSpKE6XgSyEb8iwaqDcy+uKrd/DGYHTWkUdnQDyzm727V7p21WUMhsqz5oy65kPcQ==} - peerDependencies: - '@redis/client': ^1.0.0 - '@redis/json@5.12.1': resolution: {integrity: sha512-eOze75esLve4vfqDel7aMX08CNaiLLQS2fV8mpRN9NxPe1rVR4vQyYiW/OgtGUysF6QOr9ANhfxABKNOJfXdKg==} engines: {node: '>= 18.19.0'} peerDependencies: '@redis/client': ^5.12.1 - '@redis/search@1.2.0': - resolution: {integrity: sha512-tYoDBbtqOVigEDMAcTGsRlMycIIjwMCgD8eR2t0NANeQmgK/lvxNAvYyb6bZDD4frHRhIHkJu2TBRvB0ERkOmw==} - peerDependencies: - '@redis/client': ^1.0.0 - '@redis/search@5.12.1': resolution: {integrity: sha512-ItlxbxC9cKI6IU1TLWoczwJCRb6TdmkEpWv05UrPawqaAnWGRu3rcIqsc5vN483T2fSociuyV1UkWIL5I4//2w==} engines: {node: '>= 18.19.0'} peerDependencies: '@redis/client': ^5.12.1 - '@redis/time-series@1.1.0': - resolution: {integrity: sha512-c1Q99M5ljsIuc4YdaCwfUEXsofakb9c8+Zse2qxTadu8TalLXuAESzLvFAvNVbkmSlvlzIQOLpBCmWI9wTOt+g==} - peerDependencies: - '@redis/client': ^1.0.0 - '@redis/time-series@5.12.1': resolution: {integrity: sha512-c6JL6E3EcZJuNqKFz+KM+l9l5mpcQiKvTwgA3blt5glWJ8hjDk0yeHN3beE/MpqYIQ8UEX44ItQzgkE/gCBELQ==} engines: {node: '>= 18.19.0'} @@ -878,8 +849,8 @@ packages: '@tangle-network/agent-core@0.4.11': resolution: {integrity: sha512-5B1IjrJ8xDR7w8Hv/MSk2ixul6NEJQ5Ftzo+z6l7ipeFpc0yaf1+Kkml4HDUEmGW/rqdrNpBf9/MpT7i/SY0PA==} - '@tangle-network/agent-eval@0.122.7': - resolution: {integrity: sha512-uOYVynC/+BnlNqZsKRG8NXz/wqeGB/AUImOPb/juUDQMIDzbkmxw9CiZGDmWH7kjCX7Nb8Z5zfpzO3VCTpO74Q==} + '@tangle-network/agent-eval@0.122.8': + resolution: {integrity: sha512-7v0us+6zpcR+VKqt9olG7C7j4KlTctvwhlHgm2qHU+FBorEJFcJs6JDqWMw+3h7t9ba/07muIb0AfuvSsmJJ8w==} engines: {node: '>=20'} hasBin: true @@ -1107,10 +1078,6 @@ packages: buffer-equal-constant-time@1.0.1: resolution: {integrity: sha512-zRpUiDwd/xk6ADqPMATG8vc9VPrkck7T07OIx0gnjmJAnHnTVXNQG3vfvWNuiZIkwu9KrKdA1iJKfsfTVxE6NA==} - buffer-writer@2.0.0: - resolution: {integrity: sha512-a7ZpuTZU1TRtnwyCNW3I5dc0wWNC3VR9S++Ewyk2HHZdrO3CQJqSpd+95Us590V6AL7JqUAH2IwZ/398PmNFgw==} - engines: {node: '>=4'} - buffer@5.7.1: resolution: {integrity: sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==} @@ -1399,10 +1366,6 @@ packages: resolution: {integrity: sha512-zV/5HKTfCeKWnxG0Dmrw51hEWFGfcF2xiXqcA3+J90WDuP0SvoiSO5ORvcBsifmx/FoIjgQN3oNOGaQ5PhLFkg==} engines: {node: '>=18'} - generic-pool@3.9.0: - resolution: {integrity: sha512-hymDOu5B53XvN4QT9dBmZxPX4CWhBPPLguTZ9MMFeFa/Kg0xWVfylOVNlJji/E7yTZWFd/q9GO5TxDLq156D7g==} - engines: {node: '>= 4'} - get-intrinsic@1.3.0: resolution: {integrity: sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==} engines: {node: '>= 0.4'} @@ -1894,9 +1857,6 @@ packages: resolution: {integrity: sha512-rhIwUycgwwKcP9yTOOFK/AKsAopjjCakVqLHePO3CC6Mir1Z99xT+R63jZxAT5lFZLa2inS5h+ZS2GvR99/FBg==} engines: {node: '>=8'} - packet-reader@1.0.0: - resolution: {integrity: sha512-HAKu/fG3HpHFO0AA8WE8q2g+gBJaZ9MG7fcKk+IJPLTGAD6Psw4443l+9DGRbOIh3/aXr7Phy0TjilYivJo5XQ==} - pathe@2.0.3: resolution: {integrity: sha512-WUjGcAqP1gQacoQe+OBJsFA7Ld4DyXuUIjZ5cc75cLHvJ7dtNsTugphxIADwspS+AraAUePCKrSVtPLFj/F88w==} @@ -1934,15 +1894,6 @@ packages: resolution: {integrity: sha512-o2XFanIMy/3+mThw69O8d4n1E5zsLhdO+OPqswezu7Z5ekP4hYDqlDjlmOpYMbzY2Br0ufCwJLdDIXeNVwcWFg==} engines: {node: '>=10'} - pg@8.11.3: - resolution: {integrity: sha512-+9iuvG8QfaaUrrph+kpF24cXkH1YOOUeArRNYIxq1viYHZagBxrTno7cecY1Fa44tJeZvaoG+Djpkc3JwehN5g==} - engines: {node: '>= 8.0.0'} - peerDependencies: - pg-native: '>=3.0.1' - peerDependenciesMeta: - pg-native: - optional: true - pg@8.22.0: resolution: {integrity: sha512-8wih1vVIBMxoUM2oB4soJsD9tDnDpLv4OXBJ+EJzFsvycD+lfyIreC2gGHq78f8jbLLt+bvlPTFdFZfJkOuzAA==} engines: {node: '>= 16.0.0'} @@ -1955,9 +1906,6 @@ packages: pgpass@1.0.5: resolution: {integrity: sha512-FdW9r/jQZhSeohs1Z3sI1yxFQNFvMcnmfuj4WBMUTxOrAyLMaTcE1aAMBiTlbMNaXvBCQuVi0R7hd8udDSP7ug==} - pgpass@1.0.6: - resolution: {integrity: sha512-lqIfH7bdgsxHAY/ZnUOwm+aCFKrsHBDhSFuk9O0B9uCqJAIkrKTo/+LQqLPLUS4e04+jCmQVikxE3QipH5chPw==} - picocolors@1.1.1: resolution: {integrity: sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==} @@ -2076,9 +2024,6 @@ packages: resolution: {integrity: sha512-GDhwkLfywWL2s6vEjyhri+eXmfH6j1L7JE27WhqLeYzoh/A3DBaYGEj2H/HFZCn/kMfim73FXxEJTw06WtxQwg==} engines: {node: '>= 14.18.0'} - redis@4.7.1: - resolution: {integrity: sha512-S1bJDnqLftzHXHP8JsT5II/CtHWQrASX5K96REjWjlmWKrviSOLWmM7QnRLstAWsu1VBBV1ffV6DzCvxNP0UJQ==} - redis@5.12.1: resolution: {integrity: sha512-LDsoVvb/CpoV9EN3FXvgvSHNJWuCIzl9MiO3ppOevuGLpSGJhwfQjpEwfFJcQvNSddHADDdZaWx0HnmMxRXG7g==} engines: {node: '>= 18.19.0'} @@ -2444,9 +2389,6 @@ packages: resolution: {integrity: sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==} engines: {node: '>=0.4'} - yallist@4.0.0: - resolution: {integrity: sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==} - yaml@2.8.4: resolution: {integrity: sha512-ml/JPOj9fOQK8RNnWojA67GbZ0ApXAUlN2UQclwv2eVgTgn7O9gg9o7paZWKMp4g0H3nTLtS9LVzhkpOFIKzog==} engines: {node: '>= 14.6'} @@ -2924,50 +2866,24 @@ snapshots: '@qdrant/openapi-typescript-fetch@1.2.6': {} - '@redis/bloom@1.2.0(@redis/client@1.6.1)': - dependencies: - '@redis/client': 1.6.1 - '@redis/bloom@5.12.1(@redis/client@5.12.1(@opentelemetry/api@1.9.1))': dependencies: '@redis/client': 5.12.1(@opentelemetry/api@1.9.1) - '@redis/client@1.6.1': - dependencies: - cluster-key-slot: 1.1.2 - generic-pool: 3.9.0 - yallist: 4.0.0 - '@redis/client@5.12.1(@opentelemetry/api@1.9.1)': dependencies: cluster-key-slot: 1.1.2 optionalDependencies: '@opentelemetry/api': 1.9.1 - '@redis/graph@1.1.1(@redis/client@1.6.1)': - dependencies: - '@redis/client': 1.6.1 - - '@redis/json@1.0.7(@redis/client@1.6.1)': - dependencies: - '@redis/client': 1.6.1 - '@redis/json@5.12.1(@redis/client@5.12.1(@opentelemetry/api@1.9.1))': dependencies: '@redis/client': 5.12.1(@opentelemetry/api@1.9.1) - '@redis/search@1.2.0(@redis/client@1.6.1)': - dependencies: - '@redis/client': 1.6.1 - '@redis/search@5.12.1(@redis/client@5.12.1(@opentelemetry/api@1.9.1))': dependencies: '@redis/client': 5.12.1(@opentelemetry/api@1.9.1) - '@redis/time-series@1.1.0(@redis/client@1.6.1)': - dependencies: - '@redis/client': 1.6.1 - '@redis/time-series@5.12.1(@redis/client@5.12.1(@opentelemetry/api@1.9.1))': dependencies: '@redis/client': 5.12.1(@opentelemetry/api@1.9.1) @@ -3119,7 +3035,7 @@ snapshots: '@tangle-network/agent-interface': 0.26.0 zod: 4.4.3 - '@tangle-network/agent-eval@0.122.7(typescript@5.9.3)': + '@tangle-network/agent-eval@0.122.8(typescript@5.9.3)': dependencies: '@asteasolutions/zod-to-openapi': 8.5.0(zod@4.4.3) '@ax-llm/ax': 23.0.0(zod@4.4.3) @@ -3380,8 +3296,6 @@ snapshots: buffer-equal-constant-time@1.0.1: {} - buffer-writer@2.0.0: {} - buffer@5.7.1: dependencies: base64-js: 1.5.1 @@ -3688,8 +3602,6 @@ snapshots: transitivePeerDependencies: - supports-color - generic-pool@3.9.0: {} - get-intrinsic@1.3.0: dependencies: call-bind-apply-helpers: 1.0.2 @@ -3928,7 +3840,7 @@ snapshots: crypt: 0.0.2 is-buffer: 1.1.6 - mem0ai@3.1.0(@anthropic-ai/sdk@0.40.1)(@azure/identity@4.13.1)(@azure/search-documents@12.2.0)(@cloudflare/workers-types@4.20260702.1)(@google/genai@1.52.0)(@langchain/core@1.2.3(@opentelemetry/api@1.9.1)(ws@8.21.0))(@mistralai/mistralai@1.15.1)(@qdrant/js-client-rest@1.18.0(typescript@5.9.3))(@supabase/supabase-js@2.110.7)(@types/jest@29.5.14)(@types/pg@8.11.0)(better-sqlite3@12.11.1)(cloudflare@4.5.0)(compromise@14.16.0)(groq-sdk@0.3.0)(mongodb@7.2.0)(natural@8.1.1(@opentelemetry/api@1.9.1))(ollama@0.5.18)(pg@8.11.3)(redis@4.7.1)(ws@8.21.0): + mem0ai@3.1.0(@anthropic-ai/sdk@0.40.1)(@azure/identity@4.13.1)(@azure/search-documents@12.2.0)(@cloudflare/workers-types@4.20260702.1)(@google/genai@1.52.0)(@langchain/core@1.2.3(@opentelemetry/api@1.9.1)(ws@8.21.0))(@mistralai/mistralai@1.15.1)(@qdrant/js-client-rest@1.18.0(typescript@5.9.3))(@supabase/supabase-js@2.110.7)(@types/jest@29.5.14)(@types/pg@8.11.0)(better-sqlite3@12.11.1)(cloudflare@4.5.0)(compromise@14.16.0)(groq-sdk@0.3.0)(mongodb@7.2.0)(natural@8.1.1(@opentelemetry/api@1.9.1))(ollama@0.5.18)(pg@8.22.0)(redis@5.12.1(@opentelemetry/api@1.9.1))(ws@8.21.0): dependencies: '@anthropic-ai/sdk': 0.40.1 '@azure/identity': 4.13.1 @@ -3949,8 +3861,8 @@ snapshots: natural: 8.1.1(@opentelemetry/api@1.9.1) ollama: 0.5.18 openai: 4.104.0(ws@8.21.0)(zod@3.25.76) - pg: 8.11.3 - redis: 4.7.1 + pg: 8.22.0 + redis: 5.12.1(@opentelemetry/api@1.9.1) uuid: 11.1.1 zod: 3.25.76 optionalDependencies: @@ -4149,8 +4061,6 @@ snapshots: dependencies: p-finally: 1.0.0 - packet-reader@1.0.0: {} - pathe@2.0.3: {} pathval@2.0.1: {} @@ -4164,10 +4074,6 @@ snapshots: pg-numeric@1.0.2: {} - pg-pool@3.14.0(pg@8.11.3): - dependencies: - pg: 8.11.3 - pg-pool@3.14.0(pg@8.22.0): dependencies: pg: 8.22.0 @@ -4192,18 +4098,6 @@ snapshots: postgres-interval: 3.0.0 postgres-range: 1.1.4 - pg@8.11.3: - dependencies: - buffer-writer: 2.0.0 - packet-reader: 1.0.0 - pg-connection-string: 2.14.0 - pg-pool: 3.14.0(pg@8.11.3) - pg-protocol: 1.15.0 - pg-types: 2.2.0 - pgpass: 1.0.6 - optionalDependencies: - pg-cloudflare: 1.4.0 - pg@8.22.0: dependencies: pg-connection-string: 2.14.0 @@ -4218,8 +4112,6 @@ snapshots: dependencies: split2: 4.2.0 - pgpass@1.0.6: {} - picocolors@1.1.1: {} picomatch@2.3.2: {} @@ -4337,15 +4229,6 @@ snapshots: readdirp@4.1.2: {} - redis@4.7.1: - dependencies: - '@redis/bloom': 1.2.0(@redis/client@1.6.1) - '@redis/client': 1.6.1 - '@redis/graph': 1.1.1(@redis/client@1.6.1) - '@redis/json': 1.0.7(@redis/client@1.6.1) - '@redis/search': 1.2.0(@redis/client@1.6.1) - '@redis/time-series': 1.1.0(@redis/client@1.6.1) - redis@5.12.1(@opentelemetry/api@1.9.1): dependencies: '@redis/bloom': 5.12.1(@redis/client@5.12.1(@opentelemetry/api@1.9.1)) @@ -4706,8 +4589,6 @@ snapshots: xtend@4.0.2: {} - yallist@4.0.0: {} - yaml@2.8.4: {} zod-to-json-schema@3.25.2(zod@4.4.3): diff --git a/src/benchmarks/index.ts b/src/benchmarks/index.ts index 0fd3879..0801e7a 100644 --- a/src/benchmarks/index.ts +++ b/src/benchmarks/index.ts @@ -15,7 +15,13 @@ import { type Scenario, } from '@tangle-network/agent-eval/campaign' import { stableId } from '../ids' -import { appendAttemptJournalEvent, readActiveAttemptJournal } from '../memory/attempt-log' +import { + appendAttemptJournalEvent, + assertNoInterruptedPaidCalls, + hasSettledPaidCall, + readActiveAttemptJournal, + reconcileInterruptedMemoryPaidCalls, +} from '../memory/attempt-log' import { createMemoryExecutionPool, memoryRecoveryDelayMs, @@ -277,7 +283,7 @@ export interface MemoryAdapterBenchmarkCandidate { /** Versioned adapter and configuration identity used by resumable caches. */ ref: string label?: string - /** Construction must not start billable execute work; adapter methods run inside per-case accounting. */ + /** Construction must be side-effect free; billable provider work belongs in adapter methods. */ createAdapter: (input: { purpose: 'execute' | 'recovery' }) => AgentMemoryAdapter | Promise @@ -291,6 +297,8 @@ export interface MemoryAdapterBenchmarkCandidate { export interface RunMemoryAdapterBenchmarkOptions { cases: readonly KnowledgeMemoryBenchmarkCase[] candidates: readonly MemoryAdapterBenchmarkCandidate[] + /** Retired candidates retained only so interrupted scopes can be cleaned on resume. */ + recoveryCandidates?: readonly MemoryAdapterBenchmarkCandidate[] runDir: string storage?: CampaignStorage repo?: string @@ -304,6 +312,8 @@ export interface RunMemoryAdapterBenchmarkOptions { maxConcurrency?: number dispatchTimeoutMs?: number cleanupTimeoutMs?: number + /** Refuse a damaged run with more unfinished attempts than this. Default 1000. */ + maxRecoveryAttempts?: number expectUsage?: 'assert' | 'warn' | 'off' now?: () => Date /** Required with custom storage when all controllers are confined to one process. */ @@ -923,38 +933,46 @@ function createMemoryAdapterBenchmarkResponder(options: { throw new Error(`memory adapter costUsdPerCase must be non-negative finite, got ${costUsd}`) } + dispatchContext.signal.throwIfAborted() + await options.lease.assertOwned() + const startedAt = Date.now() + const attemptId = randomUUID() + const scope = benchmarkMemoryScope( + options.candidateId, + testCase, + dispatchContext.cellId, + attemptId, + options.scope, + ) + const attempt: MemoryAdapterBenchmarkAttemptEvent = { + schema: 1, + status: 'started', + attemptId, + candidateId: options.candidateId, + candidateRef: options.candidateRef, + adapterId: options.adapter.id, + caseId: testCase.id, + cellId: dispatchContext.cellId, + scope, + recordedAt: (options.now ?? (() => new Date()))().toISOString(), + recovery: false, + } + appendMemoryBenchmarkAttemptEvent(options.storage, options.attemptLogPath, attempt) + + let externalCallAttempted = false + let executeStarted = false + let cleanupConfirmed = false const execute = async (): Promise => { + executeStarted = true dispatchContext.signal.throwIfAborted() await options.lease.assertOwned() - const startedAt = Date.now() - const attemptId = randomUUID() - const scope = benchmarkMemoryScope( - options.candidateId, - testCase, - dispatchContext.cellId, - attemptId, - options.scope, - ) - const attempt: MemoryAdapterBenchmarkAttemptEvent = { - schema: 1, - status: 'started', - attemptId, - candidateId: options.candidateId, - candidateRef: options.candidateRef, - adapterId: options.adapter.id, - caseId: testCase.id, - cellId: dispatchContext.cellId, - scope, - recordedAt: (options.now ?? (() => new Date()))().toISOString(), - recovery: false, - } - appendMemoryBenchmarkAttemptEvent(options.storage, options.attemptLogPath, attempt) let artifact: KnowledgeBenchmarkArtifact | undefined let primaryError: unknown try { for (const event of testCase.events) { dispatchContext.signal.throwIfAborted() await options.lease.assertOwned() + externalCallAttempted = true await options.adapter.write({ id: event.id, kind: 'message', @@ -976,10 +994,12 @@ function createMemoryAdapterBenchmarkResponder(options: { dispatchContext.signal.throwIfAborted() await options.lease.assertOwned() } + externalCallAttempted = true await options.adapter.flush?.() dispatchContext.signal.throwIfAborted() await options.lease.assertOwned() + externalCallAttempted = true const adapterContext = await options.adapter.getContext(testCase.prompt, { scope, limit: options.searchLimit ?? 1, @@ -1024,7 +1044,10 @@ function createMemoryAdapterBenchmarkResponder(options: { await runBoundedMemoryLifecycle({ operation: `${options.candidateId}: benchmark attempt flush`, timeoutMs: options.cleanupTimeoutMs, - run: () => options.adapter.flush?.(), + run: () => { + externalCallAttempted = true + return options.adapter.flush?.() + }, }) } catch (error) { cleanupErrors.push(error) @@ -1033,23 +1056,16 @@ function createMemoryAdapterBenchmarkResponder(options: { await runBoundedMemoryLifecycle({ operation: `${options.candidateId}: benchmark attempt cleanup`, timeoutMs: options.cleanupTimeoutMs, - run: () => options.adapter.clear!(scope), - }) - } catch (error) { - cleanupErrors.push(error) - } - } - if (cleanupOwned && cleanupErrors.length === 0) { - try { - appendMemoryBenchmarkAttemptEvent(options.storage, options.attemptLogPath, { - ...attempt, - status: 'cleaned', - recordedAt: (options.now ?? (() => new Date()))().toISOString(), + run: () => { + externalCallAttempted = true + return options.adapter.clear!(scope) + }, }) } catch (error) { cleanupErrors.push(error) } } + if (cleanupOwned && cleanupErrors.length === 0) cleanupConfirmed = true if (cleanupErrors.length > 0) { const errors = [...(primaryError ? [primaryError] : []), ...cleanupErrors] throw new MemoryAdapterBenchmarkCleanupError( @@ -1063,22 +1079,56 @@ function createMemoryAdapterBenchmarkResponder(options: { return artifact } - if (costUsd === 0) return execute() + const appendCleanedAttempt = (priorError?: unknown): void => { + try { + appendMemoryBenchmarkAttemptEvent(options.storage, options.attemptLogPath, { + ...attempt, + status: 'cleaned', + recordedAt: (options.now ?? (() => new Date()))().toISOString(), + }) + } catch (error) { + throw new MemoryAdapterBenchmarkCleanupError( + [...(priorError ? [priorError] : []), error], + `${options.candidateId}: memory benchmark cleanup could not be recorded`, + ) + } + } + + if (costUsd === 0) { + let artifact: KnowledgeBenchmarkArtifact | undefined + let error: unknown + try { + artifact = await execute() + } catch (caught) { + error = caught + } + if (cleanupConfirmed) appendCleanedAttempt(error) + if (error) throw error + if (!artifact) + throw new Error(`${options.candidateId}: memory benchmark produced no artifact`) + return artifact + } const receipt = { model: options.adapter.id, inputTokens: 0, outputTokens: 0, - usageUnknown: true, actualCostUsd: costUsd, } as const const paid = await dispatchContext.cost.runPaidCall({ + callId: memoryBenchmarkCostCallId(attempt, 'execute', 0), actor: `agent-knowledge:memory-adapter:${options.adapter.id}`, model: options.adapter.id, maximumCharge: { externallyEnforcedMaximumUsd: costUsd }, execute, receipt: () => receipt, - receiptFromError: () => receipt, + receiptFromError: () => ({ + ...receipt, + actualCostUsd: externalCallAttempted ? costUsd : 0, + }), }) + if (!executeStarted || (cleanupConfirmed && (paid.succeeded || paid.receipt))) { + appendCleanedAttempt(paid.succeeded ? undefined : paid.error) + } if (!paid.succeeded) throw paid.error return paid.value } @@ -1089,11 +1139,12 @@ export async function runMemoryAdapterBenchmark( ): Promise { if (options.candidates.length === 0) throw new Error('memory adapter benchmark requires candidates') + const allCandidates = [...options.candidates, ...(options.recoveryCandidates ?? [])] assertUniqueNonEmptyStrings( - options.candidates.map((candidate) => candidate.id), + allCandidates.map((candidate) => candidate.id), 'memory adapter candidate id', ) - for (const candidate of options.candidates) { + for (const candidate of allCandidates) { assertNonEmptyBenchmarkString(candidate.ref, `memory adapter candidate ${candidate.id} ref`) if (!/^[A-Za-z0-9][A-Za-z0-9._-]*$/.test(candidate.id)) { throw new Error( @@ -1119,6 +1170,10 @@ export async function runMemoryAdapterBenchmark( options.cleanupTimeoutMs, 'memory adapter benchmark', ) + const maxRecoveryAttempts = options.maxRecoveryAttempts ?? 1_000 + if (!Number.isSafeInteger(maxRecoveryAttempts) || maxRecoveryAttempts <= 0) { + throw new Error('memory adapter benchmark maxRecoveryAttempts must be a positive safe integer') + } storage.ensureDir(runDir) const lease = await acquireAgentMemoryRunLease({ experimentId: `memory-adapter-benchmark:${runDir}`, @@ -1133,7 +1188,14 @@ export async function runMemoryAdapterBenchmark( let result: RunMemoryAdapterBenchmarkResult | undefined let primaryError: unknown try { - result = await runOwnedMemoryAdapterBenchmark(options, storage, runDir, lease, cleanupTimeoutMs) + result = await runOwnedMemoryAdapterBenchmark( + options, + storage, + runDir, + lease, + cleanupTimeoutMs, + maxRecoveryAttempts, + ) } catch (error) { primaryError = error } @@ -1161,26 +1223,28 @@ async function runOwnedMemoryAdapterBenchmark( runDir: string, lease: OwnedAgentMemoryRunLease, cleanupTimeoutMs: number, + maxRecoveryAttempts: number, ): Promise { const maxConcurrency = options.maxConcurrency ?? 2 if (!Number.isSafeInteger(maxConcurrency) || maxConcurrency <= 0) { throw new Error('memory adapter benchmark maxConcurrency must be a positive safe integer') } + const costCeiling = options.costCeiling ?? options.costLedger?.costCeilingUsd ?? 0 const costLedger = options.costLedger ?? createRunCostLedger({ storage, runDir, - costCeilingUsd: options.costCeiling, + costCeilingUsd: costCeiling, }) - if (options.costCeiling !== undefined && costLedger.costCeilingUsd !== options.costCeiling) { + if (costLedger.costCeilingUsd !== costCeiling) { throw new Error( 'memory adapter benchmark costCeiling must match the shared cost ledger ceiling', ) } const attemptLogPath = join(runDir, 'memory-adapter-attempts.jsonl') await recoverMemoryAdapterBenchmarkAttempts({ - candidates: options.candidates, + candidates: [...options.candidates, ...(options.recoveryCandidates ?? [])], storage, attemptLogPath, lease, @@ -1190,6 +1254,7 @@ async function runOwnedMemoryAdapterBenchmark( runDir, costLedger, costPhase: options.costPhase ?? 'memory.adapter-benchmark', + maxRecoveryAttempts, }) await lease.assertOwned() const rows: MemoryAdapterBenchmarkRankingRow[] = [] @@ -1238,7 +1303,7 @@ async function runOwnedMemoryAdapterBenchmark( seed: options.seed, reps: options.reps, resumable: options.resumable, - costCeiling: options.costCeiling, + costCeiling, costLedger, costPhase: `${options.costPhase ?? 'memory.adapter-benchmark'}.${candidate.id}`, maxConcurrency: options.maxConcurrency, @@ -1348,15 +1413,49 @@ async function recoverMemoryAdapterBenchmarkAttempts(input: { runDir: string costLedger: CostLedgerHandle costPhase: string + maxRecoveryAttempts: number }): Promise { - const attempts = readActiveMemoryBenchmarkAttempts(input.storage, input.attemptLogPath) + let attempts = readActiveMemoryBenchmarkAttempts(input.storage, input.attemptLogPath) + if (attempts.length > input.maxRecoveryAttempts) { + throw new Error( + `memory adapter benchmark has ${attempts.length} unfinished attempts; maxRecoveryAttempts is ${input.maxRecoveryAttempts}`, + ) + } const candidateById = new Map(input.candidates.map((candidate) => [candidate.id, candidate])) - const grouped = new Map() for (const attempt of attempts) { - const group = grouped.get(attempt.candidateId) ?? [] - group.push(attempt) - grouped.set(attempt.candidateId, group) + const candidate = candidateById.get(attempt.candidateId) + if (!candidate) { + throw new Error( + `cannot recover memory benchmark attempts: candidate '${attempt.candidateId}' is missing; pass it in recoveryCandidates`, + ) + } + if (attempt.candidateRef !== candidate.ref) { + throw new Error( + `cannot recover memory benchmark attempt '${attempt.attemptId}': candidate ref changed from '${attempt.candidateRef}' to '${candidate.ref}'`, + ) + } + } + + reconcileInterruptedMemoryPaidCalls(input.costLedger) + assertNoInterruptedPaidCalls(input.costLedger, 'memory adapter benchmark recovery') + + for (const attempt of attempts) { + const candidate = candidateById.get(attempt.candidateId)! + const costUsd = candidate.costUsdPerCase ?? 0 + if ( + costUsd > 0 && + !hasSettledPaidCall(input.costLedger, memoryBenchmarkCostCallId(attempt, 'execute', 0)) + ) { + appendMemoryBenchmarkAttemptEvent(input.storage, input.attemptLogPath, { + ...attempt, + status: 'cleaned', + recovery: true, + recordedAt: (input.now ?? (() => new Date()))().toISOString(), + }) + } } + attempts = readActiveMemoryBenchmarkAttempts(input.storage, input.attemptLogPath) + const grouped = groupMemoryBenchmarkAttempts(attempts) const pool = createMemoryExecutionPool(input.maxConcurrency) const settled = await Promise.allSettled( [...grouped] @@ -1367,7 +1466,7 @@ async function recoverMemoryAdapterBenchmarkAttempts(input: { const candidate = candidateById.get(candidateId) if (!candidate) { throw new Error( - `cannot recover memory benchmark attempts: candidate '${candidateId}' is missing`, + `cannot recover memory benchmark attempts: candidate '${candidateId}' is missing; pass it in recoveryCandidates`, ) } for (const attempt of candidateAttempts) { @@ -1378,10 +1477,12 @@ async function recoverMemoryAdapterBenchmarkAttempts(input: { } } + let recoveryAttemptsStarted = 0 + const cleared: MemoryAdapterBenchmarkAttemptEvent[] = [] + let cleanupCanBeRecorded = false const recover = async (): Promise => { let adapter: AgentMemoryAdapter | undefined let primaryError: unknown - const cleared: MemoryAdapterBenchmarkAttemptEvent[] = [] try { adapter = await runBoundedMemoryLifecycle({ operation: `${candidate.id}: benchmark recovery adapter creation`, @@ -1396,14 +1497,18 @@ async function recoverMemoryAdapterBenchmarkAttempts(input: { ) } } - await sleepForMemoryRecovery(memoryRecoveryDelayMs(adapter), () => - input.lease.assertOwned(), + await sleepForMemoryRecovery( + memoryRecoveryDelayMs(adapter), + () => input.lease.assertOwned(), + input.cleanupTimeoutMs, + `${candidate.id}: benchmark recovery visibility wait`, ) for (const attempt of candidateAttempts.sort((left, right) => left.attemptId.localeCompare(right.attemptId), )) { await input.lease.assertOwned() try { + recoveryAttemptsStarted += 1 await runBoundedMemoryLifecycle({ operation: `${candidate.id}: abandoned benchmark attempt cleanup`, timeoutMs: input.cleanupTimeoutMs, @@ -1441,17 +1546,7 @@ async function recoverMemoryAdapterBenchmarkAttempts(input: { closeError = error } } - if (!closeError) { - for (const attempt of cleared) { - await input.lease.assertOwned() - appendMemoryBenchmarkAttemptEvent(input.storage, input.attemptLogPath, { - ...attempt, - status: 'cleaned', - recovery: true, - recordedAt: (input.now ?? (() => new Date()))().toISOString(), - }) - } - } + cleanupCanBeRecorded = !closeError if (primaryError && closeError) { throw new AggregateError( [primaryError, closeError], @@ -1463,33 +1558,67 @@ async function recoverMemoryAdapterBenchmarkAttempts(input: { } const recoveryCostUsd = candidate.recoveryCostUsdPerAttempt ?? 0 + let recoveryError: unknown + let costReceiptCaptured = recoveryCostUsd === 0 if (recoveryCostUsd === 0) { - await recover() - return + try { + await recover() + } catch (error) { + recoveryError = error + } + } else { + const maximumCostUsd = recoveryCostUsd * candidateAttempts.length + const receipt = { + model: candidate.id, + inputTokens: 0, + outputTokens: 0, + actualCostUsd: maximumCostUsd, + } as const + const tags = memoryBenchmarkRecoveryCostTags(input.runDir, candidate.id) + const paid = await input.costLedger.runPaidCall({ + callId: memoryBenchmarkRecoveryCostCallId( + candidate, + candidateAttempts, + input.costLedger.list({ tags }).length, + ), + channel: 'driver', + phase: `${input.costPhase}.${candidate.id}.recovery`, + actor: `agent-knowledge:memory-adapter-recovery:${candidate.id}`, + model: candidate.id, + tags, + maximumCharge: { externallyEnforcedMaximumUsd: maximumCostUsd }, + execute: recover, + receipt: () => receipt, + receiptFromError: () => ({ + ...receipt, + actualCostUsd: recoveryCostUsd * recoveryAttemptsStarted, + }), + }) + costReceiptCaptured = paid.succeeded || Boolean(paid.receipt) + if (!paid.succeeded) recoveryError = paid.error } - const maximumCostUsd = recoveryCostUsd * candidateAttempts.length - const receipt = { - model: candidate.id, - inputTokens: 0, - outputTokens: 0, - usageUnknown: true, - actualCostUsd: maximumCostUsd, - } as const - const paid = await input.costLedger.runPaidCall({ - channel: 'driver', - phase: `${input.costPhase}.${candidate.id}.recovery`, - actor: `agent-knowledge:memory-adapter-recovery:${candidate.id}`, - model: candidate.id, - tags: { - runDir: join(input.runDir, candidate.id), - candidateId: candidate.id, - }, - maximumCharge: { externallyEnforcedMaximumUsd: maximumCostUsd }, - execute: recover, - receipt: () => receipt, - receiptFromError: () => receipt, - }) - if (!paid.succeeded) throw paid.error + + if (cleanupCanBeRecorded && costReceiptCaptured) { + try { + for (const attempt of cleared) { + await input.lease.assertOwned() + appendMemoryBenchmarkAttemptEvent(input.storage, input.attemptLogPath, { + ...attempt, + status: 'cleaned', + recovery: true, + recordedAt: (input.now ?? (() => new Date()))().toISOString(), + }) + } + } catch (error) { + recoveryError = recoveryError + ? new AggregateError( + [recoveryError, error], + `${candidate.id}: recovery result and cleanup journal both failed`, + ) + : error + } + } + if (recoveryError) throw recoveryError }), ), ) @@ -1502,6 +1631,65 @@ async function recoverMemoryAdapterBenchmarkAttempts(input: { } } +function groupMemoryBenchmarkAttempts( + attempts: readonly MemoryAdapterBenchmarkAttemptEvent[], +): Map { + const grouped = new Map() + for (const attempt of attempts) { + const group = grouped.get(attempt.candidateId) ?? [] + group.push(attempt) + grouped.set(attempt.candidateId, group) + } + return grouped +} + +function memoryBenchmarkCostCallId( + attempt: MemoryAdapterBenchmarkAttemptEvent, + purpose: 'execute' | 'recovery', + generation: number, +): string { + return stableId( + 'memory_benchmark_cost_call', + canonicalJson({ + purpose, + generation, + attemptId: attempt.attemptId, + candidateId: attempt.candidateId, + candidateRef: attempt.candidateRef, + adapterId: attempt.adapterId, + caseId: attempt.caseId, + cellId: attempt.cellId, + }), + ) +} + +function memoryBenchmarkRecoveryCostCallId( + candidate: MemoryAdapterBenchmarkCandidate, + attempts: readonly MemoryAdapterBenchmarkAttemptEvent[], + generation: number, +): string { + return stableId( + 'memory_benchmark_recovery_cost_call', + canonicalJson({ + generation, + candidateId: candidate.id, + candidateRef: candidate.ref, + attemptIds: attempts.map((attempt) => attempt.attemptId).sort(), + }), + ) +} + +function memoryBenchmarkRecoveryCostTags( + runDir: string, + candidateId: string, +): Record { + return { + runDir: join(runDir, candidateId), + candidateId, + memoryRecovery: 'benchmark', + } +} + function appendMemoryBenchmarkAttemptEvent( storage: CampaignStorage, path: string, @@ -1803,6 +1991,10 @@ export async function runKnowledgeBenchmarkSuite['dispatch'] = async ( scenario, @@ -1827,7 +2019,7 @@ export async function runKnowledgeBenchmarkSuite>() + +const MEMORY_PROVIDER_ACTOR_PREFIXES = [ + 'agent-knowledge:memory-adapter:', + 'agent-knowledge:memory-adapter-recovery:', + 'agent-knowledge:memory-experiment:', + 'agent-knowledge:memory-recovery:', +] as const + +export function reconcileInterruptedMemoryPaidCalls(costLedger: CostLedgerHandle): void { + reconcileInterruptedPaidCallsAtMaximum( + costLedger, + (actor) => isMemoryProviderActor(actor), + 'external memory provider', + ) +} + +export function reconcileInterruptedRunPaidCalls( + costLedger: CostLedgerHandle, + label: string, +): void { + reconcileInterruptedPaidCallsAtMaximum(costLedger, () => true, label) +} + +function reconcileInterruptedPaidCallsAtMaximum( + costLedger: CostLedgerHandle, + ownsActor: (actor: string) => boolean, + label: string, +): void { + if (costLedger.summary().unresolvedCalls === 0) return + const pending = requirePendingCostCallInspection(costLedger) + for (const call of pending) { + if (call.state !== 'interrupted' || !ownsActor(call.actor)) continue + if (call.maximumCostUsd === undefined) { + throw new Error( + `cannot recover unbounded interrupted ${label} call '${call.callId}' for '${call.actor}'`, + ) + } + try { + costLedger.reconcile( + call.callId, + { + model: call.model, + inputTokens: 0, + outputTokens: 0, + actualCostUsd: call.maximumCostUsd, + }, + { + error: `process exited before the ${label} receipt for '${call.actor}' was recorded; charged the reserved maximum`, + }, + ) + } catch (error) { + if (isMissingPendingCostCall(error, call.callId)) continue + throw error + } + } +} + +export function assertNoInterruptedPaidCalls(costLedger: CostLedgerHandle, label: string): void { + const unresolved = costLedger.summary().unresolvedCalls + if (unresolved === 0) return + const pending = requirePendingCostCallInspection(costLedger) + const blocked = pending.filter((call) => call.state !== 'active') + throw new Error( + `${label} has ${unresolved} unresolved paid call(s) that cannot be resumed: ${blocked + .map((call) => `${call.callId} (${call.actor}, ${call.state})`) + .join(', ')}`, + ) +} + +export function hasSettledPaidCall(costLedger: CostLedgerHandle, callId: string): boolean { + return costLedger.list().some((receipt) => receipt.callId === callId) +} + export function appendAttemptJournalEvent(input: { storage: CampaignStorage path: string event: TEvent label: string +}): void { + appendDurableJournalEvent(input) +} + +export function appendDurableJournalEvent(input: { + storage: CampaignStorage + path: string + event: TEvent + label: string }): void { const { storage, path, event, label } = input if (!storage.append) throw new Error(`${label} requires CampaignStorage.append`) const line = `${JSON.stringify(event)}\n` + const byteLengths = journalByteLengths.get(storage) ?? new Map() + journalByteLengths.set(storage, byteLengths) for (let retry = 0; retry < 100; retry += 1) { - const current = storage.read(path) - if (current === undefined && storage.exists(path)) { - throw new Error(`cannot read ${label} '${path}'`) + let expectedBytes = byteLengths.get(path) + if (expectedBytes === undefined) { + const current = storage.read(path) + if (current === undefined && storage.exists(path)) { + throw new Error(`cannot read ${label} '${path}'`) + } + expectedBytes = new TextEncoder().encode(current ?? '').byteLength + byteLengths.set(path, expectedBytes) } - const expectedBytes = new TextEncoder().encode(current ?? '').byteLength - if (storage.append(path, line, expectedBytes) !== undefined) return + const appendedBytes = storage.append(path, line, expectedBytes) + if (appendedBytes !== undefined) { + byteLengths.set(path, appendedBytes) + return + } + byteLengths.delete(path) } throw new Error(`${label} '${path}' remained contended after 100 retries`) } @@ -72,3 +166,20 @@ export function readActiveAttemptJournal(inp } return [...active.values()] } + +function isMissingPendingCostCall(error: unknown, callId: string): boolean { + return error instanceof Error && error.message === `CostLedger: no pending call '${callId}'` +} + +function requirePendingCostCallInspection(costLedger: CostLedgerHandle) { + if (!costLedger.listPending) { + throw new Error( + 'interrupted memory recovery requires CostLedger.listPending() from @tangle-network/agent-eval 0.122.8 or newer', + ) + } + return costLedger.listPending() +} + +function isMemoryProviderActor(actor: string): boolean { + return MEMORY_PROVIDER_ACTOR_PREFIXES.some((prefix) => actor.startsWith(prefix)) +} diff --git a/src/memory/branch.ts b/src/memory/branch.ts index 9ca2a1d..f75ac76 100644 --- a/src/memory/branch.ts +++ b/src/memory/branch.ts @@ -80,6 +80,7 @@ const DEFAULT_POLICY: AgentMemorySharingPolicy = { interface AdapterLeaseState { references: number closed: boolean + activeBranchIds: Set closePromise?: Promise } @@ -164,7 +165,7 @@ export function createAgentMemoryBranch( touchedScopes.set(canonicalJson(stripUndefined(scope)), scope) } let mutationBarrier = Promise.resolve() - const releaseAdapter = retainAdapter(options.adapter) + const releaseAdapter = retainAdapter(options.adapter, branchId) let lifecycle: 'open' | 'closing' | 'closed' = 'open' let closePromise: Promise | undefined @@ -562,16 +563,21 @@ function assertBranchIsolation( } } -function retainAdapter(adapter: AgentMemoryAdapter): () => Promise { +function retainAdapter(adapter: AgentMemoryAdapter, branchId: string): () => Promise { const existing = adapterLeases.get(adapter) if (existing?.closed) throw new Error(`${adapter.id}: adapter is already closed`) - const state = existing ?? { references: 0, closed: false } + const state = existing ?? { references: 0, closed: false, activeBranchIds: new Set() } + if (state.activeBranchIds.has(branchId)) { + throw new Error(`${adapter.id}: memory branch '${branchId}' already has an open handle`) + } + state.activeBranchIds.add(branchId) state.references += 1 adapterLeases.set(adapter, state) let released = false return async () => { if (released) return released = true + state.activeBranchIds.delete(branchId) state.references -= 1 if (state.references > 0) return if (!adapter.close) { diff --git a/src/memory/experiment.ts b/src/memory/experiment.ts index c12f6a9..0dee261 100644 --- a/src/memory/experiment.ts +++ b/src/memory/experiment.ts @@ -23,7 +23,13 @@ import { scoreMemoryBenchmarkArtifact, } from '../benchmarks/index' import { stableId } from '../ids' -import { appendAttemptJournalEvent, readActiveAttemptJournal } from './attempt-log' +import { + appendAttemptJournalEvent, + assertNoInterruptedPaidCalls, + hasSettledPaidCall, + readActiveAttemptJournal, + reconcileInterruptedMemoryPaidCalls, +} from './attempt-log' import { type AgentMemoryBranch, type AgentMemoryBranchSnapshot, @@ -98,6 +104,7 @@ export interface AgentMemoryExperimentCandidate { label?: string /** Change when provider configuration changes so cached cells cannot be reused. */ ref: string + /** Construction must be side-effect free; billable provider work belongs in adapter methods. */ createAdapter(input: { branchId: string sequence: AgentMemorySequence @@ -169,6 +176,8 @@ export interface RunAgentMemoryExperimentOptions { experimentRunId?: string sequences: readonly AgentMemorySequence[] candidates: readonly AgentMemoryExperimentCandidate[] + /** Retired candidates retained only so interrupted branches can be cleaned on resume. */ + recoveryCandidates?: readonly AgentMemoryExperimentCandidate[] runDir: string executeStep?: (input: { memory: AgentMemoryBranch @@ -199,6 +208,8 @@ export interface RunAgentMemoryExperimentOptions { dispatchTimeoutMs?: number /** Total deadline for each provider cleanup, close, or recovery operation. */ cleanupTimeoutMs?: number + /** Refuse a damaged run with more unfinished attempts than this. Default 1000. */ + maxRecoveryAttempts?: number now?: () => Date /** Required with custom storage when all controllers are confined to one process. */ controllerMode?: AgentMemoryControllerMode @@ -335,11 +346,11 @@ export async function runAgentMemoryExperiment( 'sequence', ) assertUnique( - options.candidates.map((candidate) => candidate.id), + [...options.candidates, ...(options.recoveryCandidates ?? [])].map((candidate) => candidate.id), 'candidate', ) assertMemorySequences(options.sequences) - for (const candidate of options.candidates) { + for (const candidate of [...options.candidates, ...(options.recoveryCandidates ?? [])]) { if (options.cleanupBranches === false && !candidate.disposeAdapter) { throw new Error( `${candidate.id}: cleanupBranches=false requires disposeAdapter to delete isolated external state`, @@ -372,6 +383,10 @@ export async function runAgentMemoryExperiment( throw new Error('memory experiment requires CampaignStorage.append for durable attempt state') } resolveMemoryCleanupTimeoutMs(options.cleanupTimeoutMs, 'memory experiment') + const maxRecoveryAttempts = options.maxRecoveryAttempts ?? 1_000 + if (!Number.isSafeInteger(maxRecoveryAttempts) || maxRecoveryAttempts <= 0) { + throw new Error('memory experiment maxRecoveryAttempts must be a positive safe integer') + } storage.ensureDir(runDir) const lease = await acquireAgentMemoryRunLease({ experimentId: options.experimentId, @@ -429,17 +444,24 @@ async function runOwnedAgentMemoryExperiment( const executionPool = createMemoryExecutionPool(maxConcurrency) const dispatchedExecutions: Promise[] = [] const candidateById = new Map(options.candidates.map((candidate) => [candidate.id, candidate])) + const recoveryCandidateById = new Map( + [...options.candidates, ...(options.recoveryCandidates ?? [])].map((candidate) => [ + candidate.id, + candidate, + ]), + ) const sequenceById = new Map(options.sequences.map((sequence) => [sequence.id, sequence])) const scenarios = buildAgentMemorySequenceScenarios(options.sequences, options.candidates) const attemptLogPath = join(runDir, 'memory-attempts.jsonl') + const costCeiling = options.costCeiling ?? options.costLedger?.costCeilingUsd ?? 0 const costLedger = options.costLedger ?? createRunCostLedger({ storage, runDir, - costCeilingUsd: options.costCeiling, + costCeilingUsd: costCeiling, }) - if (options.costCeiling !== undefined && costLedger.costCeilingUsd !== options.costCeiling) { + if (costLedger.costCeilingUsd !== costCeiling) { throw new Error('memory experiment costCeiling must match the shared cost ledger ceiling') } storage.ensureDir(runDir) @@ -448,11 +470,12 @@ async function runOwnedAgentMemoryExperiment( storage, runDir, attemptLogPath, - candidateById, + candidateById: recoveryCandidateById, sequenceById, lease, maxConcurrency, costLedger, + maxRecoveryAttempts: options.maxRecoveryAttempts ?? 1_000, }) await lease.assertOwned() let campaign: CampaignResult | undefined @@ -486,7 +509,7 @@ async function runOwnedAgentMemoryExperiment( seed: options.seed, reps: options.reps, resumable: options.resumable, - costCeiling: options.costCeiling, + costCeiling, costLedger, costPhase: options.costPhase, maxConcurrency, @@ -614,27 +637,34 @@ async function runSequenceCell(input: { `${candidate.id}: memory sequence`, ) + context.signal.throwIfAborted() + await lease.assertOwned() + const startedAt = Date.now() + const branchId = stableId( + 'memory_branch', + `${runIdentity}:${context.cellId}:${context.seed}:${randomUUID()}`, + ) + const attempt = memoryAttemptEvent({ + status: 'started', + branchId, + candidate, + sequence: scenario.sequence, + rep: context.rep, + seed: context.seed, + recovery: false, + cleanupBranches, + now: options.now, + }) + appendMemoryAttemptEvent(storage, attemptLogPath, attempt) + let externalCallAttempted = false + let executeStarted = false + let cleanupConfirmed = false const execute = async (): Promise => { + executeStarted = true context.signal.throwIfAborted() await lease.assertOwned() - const startedAt = Date.now() - const branchId = stableId( - 'memory_branch', - `${runIdentity}:${context.cellId}:${context.seed}:${randomUUID()}`, - ) - const attempt = memoryAttemptEvent({ - status: 'started', - branchId, - candidate, - sequence: scenario.sequence, - rep: context.rep, - seed: context.seed, - recovery: false, - cleanupBranches, - now: options.now, - }) - appendMemoryAttemptEvent(storage, attemptLogPath, attempt) + let rawAdapter: AgentMemoryAdapter | undefined let adapter: AgentMemoryAdapter | undefined let memory: AgentMemoryBranch | undefined let primaryError: unknown @@ -642,8 +672,6 @@ async function runSequenceCell(input: { let finalClearStarted = false let finalClearCompleted = false try { - // A factory can contact a provider before it rejects, so charge conservatively once invoked. - externalCallAttempted = true const created = await candidate.createAdapter({ branchId, sequence: scenario.sequence, @@ -652,7 +680,10 @@ async function runSequenceCell(input: { purpose: 'execute', }) if (!created) throw new Error(`${candidate.id}: createAdapter returned no execution adapter`) - adapter = created + rawAdapter = created + adapter = trackExternalMemoryCalls(created, () => { + externalCallAttempted = true + }) await lease.assertOwned() context.signal.throwIfAborted() if (cleanupBranches && !adapter.clear) { @@ -785,7 +816,10 @@ async function runSequenceCell(input: { await runBoundedMemoryLifecycle({ operation: `${candidate.id}: adapter disposal`, timeoutMs: cleanupTimeoutMs, - run: () => candidate.disposeAdapter?.(adapter!), + run: () => { + if (candidate.disposeAdapter) externalCallAttempted = true + return candidate.disposeAdapter?.(rawAdapter!) + }, }) } catch (error) { cleanupErrors.push(error) @@ -796,17 +830,7 @@ async function runSequenceCell(input: { new Error(`${candidate.id}: adapter creation failed before cleanup could be confirmed`), ) } - if (cleanupOwned && cleanupErrors.length === 0) { - try { - appendMemoryAttemptEvent(storage, attemptLogPath, { - ...attempt, - status: 'cleaned', - recordedAt: (options.now ?? (() => new Date()))().toISOString(), - }) - } catch (error) { - cleanupErrors.push(error) - } - } + if (cleanupOwned && cleanupErrors.length === 0) cleanupConfirmed = true if (!cleanupOwned && cleanupErrors.length === 0) { if (primaryError) throw primaryError throw ownershipError @@ -828,15 +852,42 @@ async function runSequenceCell(input: { return completedArtifact } - if (costUsd === 0) return execute() + const appendCleanedAttempt = (priorError?: unknown): void => { + try { + appendMemoryAttemptEvent(storage, attemptLogPath, { + ...attempt, + status: 'cleaned', + recordedAt: (options.now ?? (() => new Date()))().toISOString(), + }) + } catch (error) { + throw new AgentMemoryCleanupError( + [...(priorError ? [priorError] : []), error], + `${candidate.id}: memory branch cleanup could not be recorded`, + ) + } + } + + if (costUsd === 0) { + let artifact: AgentMemorySequenceArtifact | undefined + let error: unknown + try { + artifact = await execute() + } catch (caught) { + error = caught + } + if (cleanupConfirmed) appendCleanedAttempt(error) + if (error) throw error + if (!artifact) throw new Error(`${candidate.id}: memory sequence produced no result`) + return artifact + } const receipt = { model: candidate.id, inputTokens: 0, outputTokens: 0, - usageUnknown: true, actualCostUsd: costUsd, } as const const paid = await context.cost.runPaidCall({ + callId: memoryAttemptCostCallId(attempt, 'execute', 0), actor: `agent-knowledge:memory-experiment:${candidate.id}`, model: candidate.id, maximumCharge: { externallyEnforcedMaximumUsd: costUsd }, @@ -847,10 +898,59 @@ async function runSequenceCell(input: { actualCostUsd: externalCallAttempted ? costUsd : 0, }), }) + if (!executeStarted || (cleanupConfirmed && (paid.succeeded || paid.receipt))) { + appendCleanedAttempt(paid.succeeded ? undefined : paid.error) + } if (!paid.succeeded) throw paid.error return paid.value } +function trackExternalMemoryCalls( + adapter: AgentMemoryAdapter, + onExternalCall: () => void, +): AgentMemoryAdapter { + return { + id: adapter.id, + branchIsolation: adapter.branchIsolation, + search(query, options) { + onExternalCall() + return adapter.search.call(adapter, query, options) + }, + getContext(query, options) { + onExternalCall() + return adapter.getContext.call(adapter, query, options) + }, + write(input) { + onExternalCall() + return adapter.write.call(adapter, input) + }, + ...(adapter.clear + ? { + clear(scope?: AgentMemoryScope) { + onExternalCall() + return adapter.clear!.call(adapter, scope) + }, + } + : {}), + ...(adapter.flush + ? { + flush() { + onExternalCall() + return adapter.flush!.call(adapter) + }, + } + : {}), + ...(adapter.close + ? { + close() { + onExternalCall() + return adapter.close!.call(adapter) + }, + } + : {}), + } +} + async function recoverAbandonedMemoryAttempts(input: { options: RunAgentMemoryExperimentOptions storage: CampaignStorage @@ -861,8 +961,55 @@ async function recoverAbandonedMemoryAttempts(input: { lease: OwnedMemoryExperimentRunLease maxConcurrency: number costLedger: CostLedgerHandle + maxRecoveryAttempts: number }): Promise> { - const attempts = readActiveMemoryAttempts(input.storage, input.attemptLogPath) + let attempts = readActiveMemoryAttempts(input.storage, input.attemptLogPath) + if (attempts.length > input.maxRecoveryAttempts) { + throw new Error( + `memory experiment has ${attempts.length} unfinished attempts; maxRecoveryAttempts is ${input.maxRecoveryAttempts}`, + ) + } + for (const attempt of attempts) { + const candidate = input.candidateById.get(attempt.candidateId) + if (!candidate) { + throw new Error( + `cannot recover memory branch '${attempt.branchId}': candidate '${attempt.candidateId}' is missing; pass it in recoveryCandidates`, + ) + } + if (candidate.ref !== attempt.candidateRef) { + throw new Error( + `cannot recover memory branch '${attempt.branchId}': candidate ref changed from '${attempt.candidateRef}' to '${candidate.ref}'`, + ) + } + if (!input.sequenceById.has(attempt.sequenceId)) { + throw new Error( + `cannot recover memory branch '${attempt.branchId}': sequence '${attempt.sequenceId}' is missing`, + ) + } + if ((input.options.cleanupBranches ?? true) !== attempt.cleanupBranches) { + throw new Error(`cannot recover memory branch '${attempt.branchId}': cleanupBranches changed`) + } + } + + reconcileInterruptedMemoryPaidCalls(input.costLedger) + assertNoInterruptedPaidCalls(input.costLedger, 'memory experiment recovery') + + for (const attempt of attempts) { + const candidate = input.candidateById.get(attempt.candidateId)! + const executionCostUsd = candidate.externalCostUsdPerSequence ?? 0 + if ( + executionCostUsd > 0 && + !hasSettledPaidCall(input.costLedger, memoryAttemptCostCallId(attempt, 'execute', 0)) + ) { + appendMemoryAttemptEvent(input.storage, input.attemptLogPath, { + ...attempt, + status: 'cleaned', + recovery: true, + recordedAt: (input.options.now ?? (() => new Date()))().toISOString(), + }) + } + } + attempts = readActiveMemoryAttempts(input.storage, input.attemptLogPath) const pool = createMemoryExecutionPool(input.maxConcurrency) const settled = await Promise.allSettled( attempts @@ -873,7 +1020,7 @@ async function recoverAbandonedMemoryAttempts(input: { const candidate = input.candidateById.get(attempt.candidateId) if (!candidate) { throw new Error( - `cannot recover memory branch '${attempt.branchId}': candidate '${attempt.candidateId}' is missing`, + `cannot recover memory branch '${attempt.branchId}': candidate '${attempt.candidateId}' is missing; pass it in recoveryCandidates`, ) } if (candidate.ref !== attempt.candidateRef) { @@ -893,47 +1040,65 @@ async function recoverAbandonedMemoryAttempts(input: { ) } const recoveryCostUsd = candidate.externalRecoveryCostUsdPerAttempt ?? 0 - const recover = () => - recoverMemoryAttempt({ + let externalRecoveryAttempted = false + const recover = async (): Promise => { + await recoverMemoryAttempt({ options: input.options, candidate, sequence, attempt, lease: input.lease, + onExternalCall: () => { + externalRecoveryAttempted = true + }, }) + } if (recoveryCostUsd === 0) { await recover() + appendMemoryAttemptEvent(input.storage, input.attemptLogPath, { + ...attempt, + status: 'cleaned', + recovery: true, + recordedAt: (input.options.now ?? (() => new Date()))().toISOString(), + }) } else { + const tags = memoryRecoveryCostTags(input.runDir, candidate.id, attempt.branchId) const receipt = { model: candidate.id, inputTokens: 0, outputTokens: 0, - usageUnknown: true, actualCostUsd: recoveryCostUsd, } as const const paid = await input.costLedger.runPaidCall({ + callId: memoryAttemptCostCallId( + attempt, + 'recovery', + input.costLedger.list({ tags }).length, + ), channel: 'driver', phase: `${input.options.costPhase ?? 'memory.experiment'}.recovery`, actor: `agent-knowledge:memory-recovery:${candidate.id}`, model: candidate.id, - tags: { - runDir: input.runDir, - candidateId: candidate.id, - branchId: attempt.branchId, - }, + tags, maximumCharge: { externallyEnforcedMaximumUsd: recoveryCostUsd }, execute: recover, - receipt: () => receipt, - receiptFromError: () => receipt, + receipt: () => ({ + ...receipt, + actualCostUsd: externalRecoveryAttempted ? recoveryCostUsd : 0, + }), + receiptFromError: () => ({ + ...receipt, + actualCostUsd: externalRecoveryAttempted ? recoveryCostUsd : 0, + }), }) if (!paid.succeeded) throw paid.error + appendMemoryAttemptEvent(input.storage, input.attemptLogPath, { + ...attempt, + status: 'cleaned', + recovery: true, + recordedAt: (input.options.now ?? (() => new Date()))().toISOString(), + }) } - appendMemoryAttemptEvent(input.storage, input.attemptLogPath, { - ...attempt, - status: 'cleaned', - recovery: true, - recordedAt: (input.options.now ?? (() => new Date()))().toISOString(), - }) }), ), ) @@ -958,13 +1123,15 @@ async function recoverMemoryAttempt(input: { sequence: AgentMemorySequence attempt: AgentMemoryAttemptEvent lease: OwnedMemoryExperimentRunLease + onExternalCall(): void }): Promise { - const { options, candidate, sequence, attempt, lease } = input + const { options, candidate, sequence, attempt, lease, onExternalCall } = input const cleanupBranches = attempt.cleanupBranches const cleanupTimeoutMs = resolveMemoryCleanupTimeoutMs( options.cleanupTimeoutMs, `${candidate.id}: abandoned branch recovery`, ) + let rawAdapter: AgentMemoryAdapter | undefined let adapter: AgentMemoryAdapter | undefined let memory: AgentMemoryBranch | undefined let primaryError: unknown @@ -983,9 +1150,15 @@ async function recoverMemoryAttempt(input: { }) await lease.assertOwned() if (recovered === null) return - adapter = recovered + rawAdapter = recovered + adapter = trackExternalMemoryCalls(recovered, onExternalCall) const recoveryDelayMs = memoryRecoveryDelayMs(adapter) - await sleepForMemoryRecovery(recoveryDelayMs, () => lease.assertOwned()) + await sleepForMemoryRecovery( + recoveryDelayMs, + () => lease.assertOwned(), + cleanupTimeoutMs, + `${candidate.id}: abandoned branch recovery visibility wait`, + ) if (cleanupBranches) { if (!adapter.clear) { throw new Error( @@ -1025,9 +1198,12 @@ async function recoverMemoryAttempt(input: { operation: `${candidate.id}: recovery adapter close`, timeoutMs: cleanupTimeoutMs, run: async () => { - if (memory && cleanupOwned) await memory.close?.() - else { - if (cleanupOwned) await adapter!.flush?.() + if (memory && cleanupOwned) { + await memory.close?.() + } else { + if (cleanupOwned && adapter!.flush) { + await adapter!.flush() + } await adapter!.close?.() } }, @@ -1037,10 +1213,11 @@ async function recoverMemoryAttempt(input: { } if (cleanupOwned) { try { + if (candidate.disposeAdapter) onExternalCall() await runBoundedMemoryLifecycle({ operation: `${candidate.id}: recovery adapter disposal`, timeoutMs: cleanupTimeoutMs, - run: () => candidate.disposeAdapter?.(adapter!), + run: () => candidate.disposeAdapter?.(rawAdapter!), }) } catch (error) { cleanupErrors.push(error) @@ -1061,6 +1238,39 @@ async function recoverMemoryAttempt(input: { } } +function memoryAttemptCostCallId( + attempt: AgentMemoryAttemptEvent, + purpose: 'execute' | 'recovery', + generation: number, +): string { + return stableId( + 'memory_cost_call', + canonicalJson({ + purpose, + generation, + branchId: attempt.branchId, + candidateId: attempt.candidateId, + candidateRef: attempt.candidateRef, + sequenceId: attempt.sequenceId, + rep: attempt.rep, + seed: attempt.seed, + }), + ) +} + +function memoryRecoveryCostTags( + runDir: string, + candidateId: string, + branchId: string, +): Record { + return { + runDir, + candidateId, + branchId, + memoryRecovery: 'attempt', + } +} + function memoryAttemptEvent(input: { status: AgentMemoryAttemptEvent['status'] branchId: string diff --git a/src/memory/graphiti.ts b/src/memory/graphiti.ts index b3f92df..ea9c69a 100644 --- a/src/memory/graphiti.ts +++ b/src/memory/graphiti.ts @@ -13,6 +13,8 @@ export interface GraphitiMcpClientLike { export interface GraphitiMemoryAdapterOptions { client: GraphitiMcpClientLike id?: string + /** Stable server/deployment identity for cache-key helpers. Never put credentials here. */ + backendRef?: string defaultScope?: AgentMemoryScope consistency?: 'queued' | 'visible' ingestionTimeoutMs?: number @@ -454,6 +456,12 @@ function assertGraphitiOptions(options: GraphitiMemoryAdapterOptions): void { if (options.id !== undefined && (typeof options.id !== 'string' || !options.id.trim())) { throw new Error('Graphiti id must be a non-empty string') } + if ( + options.backendRef !== undefined && + (typeof options.backendRef !== 'string' || !options.backendRef.trim()) + ) { + throw new Error('Graphiti backendRef must be a non-empty string') + } if (options.consistency !== undefined && !['queued', 'visible'].includes(options.consistency)) { throw new Error('Graphiti consistency must be queued or visible') } @@ -552,7 +560,10 @@ export function graphitiMemoryAdapterIdentity( | 'search' | 'toolNames' | 'defaultScope' - >, + > & { backendRef: string }, ): string { + if (typeof options.backendRef !== 'string' || !options.backendRef.trim()) { + throw new Error('Graphiti backendRef must be a non-empty string') + } return stableId('graphiti', canonicalJson(stripUndefined(options))) } diff --git a/src/memory/improvement.ts b/src/memory/improvement.ts index 462e5b6..346edb1 100644 --- a/src/memory/improvement.ts +++ b/src/memory/improvement.ts @@ -23,6 +23,11 @@ import { type SurfaceProposer, surfaceHash, } from '@tangle-network/agent-eval/campaign' +import { + appendDurableJournalEvent, + assertNoInterruptedPaidCalls, + reconcileInterruptedRunPaidCalls, +} from './attempt-log' import type { AgentMemoryExperimentCandidate, AgentMemorySequence, @@ -32,6 +37,7 @@ import type { RunAgentMemoryExperimentResult, } from './experiment' import { runAgentMemoryExperiment } from './experiment' +import { runBoundedMemoryLifecycle } from './lifecycle' import { type AgentMemoryAcquireRunLease, type AgentMemoryControllerMode, @@ -71,8 +77,50 @@ export interface AgentMemoryPromotionDecision { export interface AgentMemoryActivation { id: string - status: 'not-eligible' | 'not-configured' | 'pending' | 'activated' | 'already-activated' - receiptPath: string + status: + | 'not-eligible' + | 'not-configured' + | 'pending' + | 'activated' + | 'recovered' + | 'already-activated' + journalPath: string +} + +export interface AgentMemoryActivationDriver { + /** Change whenever activation behavior or the external target changes. */ + ref: string + /** Return the exact currently active configuration. */ + readCurrent(): Promise + /** Atomically replace expectedConfig with config, or fail on a concurrent change. */ + compareAndSet(input: { + activationId: string + expectedConfig: TConfig + expectedSurfaceHash: string + config: TConfig + surfaceHash: string + decision: AgentMemoryPromotionDecision + lineage: Lineage + holdout: RunAgentMemoryExperimentResult + }): Promise +} + +interface AgentMemoryActivationEvent { + schema: 1 + status: 'prepared' | 'activated' + activationId: string + experimentId: string + activationRef: string + baselineSurfaceHash: string + winnerSurfaceHash: string + holdoutManifestHash: string + recordedAt: string + outcome?: 'applied' | 'recovered' | 'already-current' +} + +interface AgentMemoryActivationJournalState { + prepared: boolean + activated?: AgentMemoryActivationEvent } export type AgentMemoryImprovementRunLease = AgentMemoryRunLease @@ -122,6 +170,7 @@ export interface RunAgentMemoryImprovementOptions { resumable?: boolean dispatchTimeoutMs?: number cleanupTimeoutMs?: number + maxRecoveryAttempts?: number maxTotalCostUsd?: number executeStep?: RunAgentMemoryExperimentOptions['executeStep'] executeStepRef?: string @@ -133,13 +182,8 @@ export interface RunAgentMemoryImprovementOptions { criticalDimensions?: readonly string[] criticalDimensionTolerance?: number minHoldoutScore?: number - onPromote?: (input: { - activationId: string - config: TConfig - decision: AgentMemoryPromotionDecision - lineage: Lineage - holdout: RunAgentMemoryExperimentResult - }) => Promise + activation?: AgentMemoryActivationDriver + activationTimeoutMs?: number now?: () => Date } @@ -173,6 +217,11 @@ const DEFAULT_CRITICAL_DIMENSIONS = [ export async function runAgentMemoryImprovement( options: RunAgentMemoryImprovementOptions, ): Promise> { + if ('onPromote' in options) { + throw new Error( + 'memory improvement onPromote was removed; use activation.readCurrent and activation.compareAndSet', + ) + } if (options.seeds.length === 0) throw new Error('memory improvement requires seed configs') if (options.trainSequences.length === 0) { throw new Error('memory improvement requires training sequences') @@ -253,6 +302,9 @@ async function runAgentMemoryImprovementOwned( const serialize = (config: TConfig): string => serializeMemoryConfig(serializeRaw, config) const parse = (surface: string): TConfig => parseMemoryConfig(parseRaw, surface) for (const seed of options.seeds) assertMemoryConfigRoundTrip(seed.config, serialize, parse) + if (options.activation && !storage.append) { + throw new Error('memory activation requires CampaignStorage.append') + } if (options.resumable !== false && !options.lineageStore && !storage.append) { throw new Error('resumable memory improvement requires CampaignStorage.append') } @@ -260,8 +312,10 @@ async function runAgentMemoryImprovementOwned( const costLedger = createRunCostLedger({ storage, runDir, - costCeilingUsd: options.maxTotalCostUsd, + costCeilingUsd: options.maxTotalCostUsd ?? 0, }) + reconcileInterruptedRunPaidCalls(costLedger, 'memory improvement run') + assertNoInterruptedPaidCalls(costLedger, 'memory improvement recovery') const trainScenarios: MemoryConfigScenario[] = options.trainSequences.map((sequence) => ({ id: sequence.id, kind: 'agent-memory-config-search', @@ -390,32 +444,46 @@ async function runAgentMemoryImprovementOwned( } const resultJsonPath = join(runDir, 'memory-improvement-result.json') + const activationRef = options.activation?.ref ?? 'not-configured' const activationId = `memory-activation-${surfaceHash( canonicalJson({ experimentId: options.experimentId, improvementRef: options.improvementRef, + activationRef, baselineSurfaceHash: baselineHash, winnerSurfaceHash: winnerHash, holdoutManifestHash: holdout?.campaign.manifestHash ?? null, promotionPolicy: normalizedPromotionPolicy(options), }), )}` - const activationReceiptDir = join(runDir, 'activations') - const activationReceiptPath = join(activationReceiptDir, `${activationId}.json`) + const activationJournalDir = join(runDir, 'activations') + const activationJournalPath = join(activationJournalDir, `${activationId}.jsonl`) const activationEligible = decision.status === 'promote' && holdout !== undefined - const alreadyActivated = - activationEligible && - hasCompletedActivation(storage, activationReceiptPath, activationId, winnerHash) + const activationEventIdentity = holdout + ? { + schema: 1 as const, + activationId, + experimentId: options.experimentId, + activationRef, + baselineSurfaceHash: baselineHash, + winnerSurfaceHash: winnerHash, + holdoutManifestHash: holdout.campaign.manifestHash, + } + : undefined + const activationJournal = + activationEligible && activationEventIdentity + ? readMemoryActivationJournal(storage, activationJournalPath, activationEventIdentity) + : { prepared: false } const activation: AgentMemoryActivation = { id: activationId, status: !activationEligible ? 'not-eligible' - : alreadyActivated + : activationJournal.activated ? 'already-activated' - : options.onPromote + : options.activation ? 'pending' : 'not-configured', - receiptPath: activationReceiptPath, + journalPath: activationJournalPath, } const result = { lineage: search.lineage, @@ -433,33 +501,108 @@ async function runAgentMemoryImprovementOwned( } satisfies RunAgentMemoryImprovementResult await lease.assertOwned() writeMemoryImprovementResult(storage, options.experimentId, result) - if (activationEligible && !alreadyActivated && holdout && options.onPromote) { + if ( + activationEligible && + !activationJournal.activated && + activationEventIdentity && + holdout && + options.activation + ) { + const activationDriver = options.activation + const activationTimeoutMs = options.activationTimeoutMs ?? 60_000 + const hadPreparedEvent = activationJournal.prepared + if (!hadPreparedEvent) { + await lease.assertOwned() + storage.ensureDir(activationJournalDir) + appendMemoryActivationEvent(storage, activationJournalPath, { + ...activationEventIdentity, + status: 'prepared', + recordedAt: (options.now ?? (() => new Date()))().toISOString(), + }) + } + await lease.assertOwned() - await options.onPromote({ - activationId, - config: winnerConfig, - decision, - lineage: search.lineage, - holdout, + const currentConfig = await runBoundedMemoryLifecycle({ + operation: `${activationDriver.ref}: read current memory configuration`, + timeoutMs: activationTimeoutMs, + run: () => activationDriver.readCurrent(), }) await lease.assertOwned() - storage.ensureDir(activationReceiptDir) - storage.write( - activationReceiptPath, - `${JSON.stringify( - { - activationId, - experimentId: options.experimentId, - winnerSurfaceHash: winnerHash, - holdoutManifestHash: holdout.campaign.manifestHash, - status: 'activated', - activatedAt: (options.now ?? (() => new Date()))().toISOString(), - }, - null, - 2, - )}\n`, - ) - activation.status = 'activated' + const currentHash = surfaceHash(serialize(currentConfig)) + if (currentHash !== baselineHash && currentHash !== winnerHash) { + throw new Error( + `memory activation target '${activationDriver.ref}' changed concurrently; expected '${baselineHash}' or '${winnerHash}', found '${currentHash}'`, + ) + } + + let outcome: NonNullable + if (currentHash === winnerHash) { + outcome = hadPreparedEvent ? 'recovered' : 'already-current' + activation.status = 'recovered' + } else { + let compareError: unknown + try { + await runBoundedMemoryLifecycle({ + operation: `${activationDriver.ref}: activate memory configuration`, + timeoutMs: activationTimeoutMs, + run: () => + activationDriver.compareAndSet({ + activationId, + expectedConfig: baselineConfig, + expectedSurfaceHash: baselineHash, + config: winnerConfig, + surfaceHash: winnerHash, + decision, + lineage: search.lineage, + holdout, + }), + }) + } catch (error) { + compareError = error + } + await lease.assertOwned() + + let observedConfig: TConfig + try { + observedConfig = await runBoundedMemoryLifecycle({ + operation: `${activationDriver.ref}: confirm memory configuration`, + timeoutMs: activationTimeoutMs, + run: () => activationDriver.readCurrent(), + }) + } catch (error) { + if (compareError) { + throw new AggregateError( + [compareError, error], + `memory activation '${activationId}' failed and its live state could not be confirmed`, + ) + } + throw error + } + await lease.assertOwned() + const observedHash = surfaceHash(serialize(observedConfig)) + if (observedHash !== winnerHash) { + const mismatch = new Error( + `memory activation '${activationId}' did not install the measured winner; found '${observedHash}'`, + ) + if (compareError) { + throw new AggregateError( + [compareError, mismatch], + `memory activation '${activationId}' failed without applying the measured winner`, + ) + } + throw mismatch + } + outcome = compareError ? 'recovered' : 'applied' + activation.status = compareError ? 'recovered' : 'activated' + } + + await lease.assertOwned() + appendMemoryActivationEvent(storage, activationJournalPath, { + ...activationEventIdentity, + status: 'activated', + outcome, + recordedAt: (options.now ?? (() => new Date()))().toISOString(), + }) writeMemoryImprovementResult(storage, options.experimentId, result) } return result @@ -538,6 +681,7 @@ function experimentOptions( | 'maxConcurrency' | 'dispatchTimeoutMs' | 'cleanupTimeoutMs' + | 'maxRecoveryAttempts' | 'executeStep' | 'executeStepRef' | 'onBranchSnapshot' @@ -554,6 +698,7 @@ function experimentOptions( maxConcurrency: options.sequenceConcurrency, dispatchTimeoutMs: options.dispatchTimeoutMs, cleanupTimeoutMs: options.cleanupTimeoutMs, + maxRecoveryAttempts: options.maxRecoveryAttempts, executeStep: options.executeStep, executeStepRef: options.executeStepRef, onBranchSnapshot: options.onBranchSnapshot, @@ -577,9 +722,10 @@ function assertMemoryImprovementIdentity( ): void { const path = join(runDir, 'memory-improvement-manifest.json') const identity = { - schema: 4, + schema: 5, experimentId: options.experimentId, improvementRef: options.improvementRef, + activationRef: options.activation?.ref ?? null, proposerKind: options.proposer.kind, proposerKinds: Object.fromEntries( Object.entries(options.proposers ?? {}) @@ -633,33 +779,92 @@ function assertMemoryImprovementIdentity( } } -function hasCompletedActivation( +function appendMemoryActivationEvent( storage: CampaignStorage, path: string, - activationId: string, - winnerSurfaceHash: string, -): boolean { + event: AgentMemoryActivationEvent, +): void { + appendDurableJournalEvent({ + storage, + path, + event, + label: 'memory activation journal', + }) +} + +function readMemoryActivationJournal( + storage: CampaignStorage, + path: string, + expected: Omit, +): AgentMemoryActivationJournalState { const stored = storage.read(path) if (stored === undefined) { - if (storage.exists(path)) throw new Error(`cannot read activation receipt '${path}'`) - return false + if (storage.exists(path)) throw new Error(`cannot read memory activation journal '${path}'`) + return { prepared: false } } - let receipt: unknown - try { - receipt = JSON.parse(stored) - } catch (error) { - throw new Error(`invalid activation receipt '${path}'`, { cause: error }) + let prepared = false + let activated: AgentMemoryActivationEvent | undefined + for (const [index, line] of stored.split('\n').entries()) { + if (!line) continue + let raw: unknown + try { + raw = JSON.parse(line) + } catch (error) { + throw new Error(`invalid memory activation journal '${path}' line ${index + 1}`, { + cause: error, + }) + } + const event = parseMemoryActivationEvent(raw, path, index + 1, expected) + if (event.status === 'prepared') { + if (prepared || activated) { + throw new Error(`memory activation journal '${path}' repeats its prepared event`) + } + prepared = true + continue + } + if (!prepared || activated) { + throw new Error(`memory activation journal '${path}' has an out-of-order activated event`) + } + activated = event + } + return { prepared, ...(activated ? { activated } : {}) } +} + +function parseMemoryActivationEvent( + value: unknown, + path: string, + line: number, + expected: Omit, +): AgentMemoryActivationEvent { + if (!value || typeof value !== 'object' || Array.isArray(value)) { + throw new Error(`invalid memory activation journal '${path}' line ${line}`) + } + const event = value as Record + for (const [key, expectedValue] of Object.entries(expected)) { + if (event[key] !== expectedValue) { + throw new Error( + `memory activation journal '${path}' line ${line} does not match the measured winner`, + ) + } + } + if (event.status !== 'prepared' && event.status !== 'activated') { + throw new Error(`invalid memory activation journal '${path}' line ${line} status`) + } + if (typeof event.recordedAt !== 'string' || !Number.isFinite(Date.parse(event.recordedAt))) { + throw new Error(`invalid memory activation journal '${path}' line ${line} recordedAt`) } if ( - !receipt || - typeof receipt !== 'object' || - (receipt as Record).activationId !== activationId || - (receipt as Record).winnerSurfaceHash !== winnerSurfaceHash || - (receipt as Record).status !== 'activated' + event.status === 'activated' && + event.outcome !== 'applied' && + event.outcome !== 'recovered' && + event.outcome !== 'already-current' ) { - throw new Error(`activation receipt '${path}' does not match the measured winner`) + throw new Error(`invalid memory activation journal '${path}' line ${line} outcome`) + } + if (event.status === 'prepared' && event.outcome !== undefined) { + throw new Error(`invalid memory activation journal '${path}' line ${line} prepared outcome`) } - return true + return value as AgentMemoryActivationEvent } function writeMemoryImprovementResult( @@ -930,6 +1135,17 @@ function assertMemoryImprovementOptions( if (options.governor !== undefined && typeof options.governor.decide !== 'function') { throw new Error('memory improvement governor.decide must be a function') } + if (options.activation !== undefined) { + if (typeof options.activation.ref !== 'string' || !options.activation.ref.trim()) { + throw new Error('memory improvement activation.ref must be a non-empty string') + } + if (typeof options.activation.readCurrent !== 'function') { + throw new Error('memory improvement activation.readCurrent must be a function') + } + if (typeof options.activation.compareAndSet !== 'function') { + throw new Error('memory improvement activation.compareAndSet must be a function') + } + } for (const [name, proposer] of Object.entries(options.proposers ?? {})) { if (!name.trim()) throw new Error('memory improvement proposer labels must be non-empty') if ( @@ -955,6 +1171,7 @@ function assertMemoryImprovementOptions( ['candidateConcurrency', options.candidateConcurrency], ['sequenceConcurrency', options.sequenceConcurrency], ['reps', options.reps], + ['maxRecoveryAttempts', options.maxRecoveryAttempts], ] as const) { if (value !== undefined && (!Number.isSafeInteger(value) || value <= 0)) { throw new Error(`memory improvement ${name} must be a positive safe integer`) @@ -966,6 +1183,12 @@ function assertMemoryImprovementOptions( ) { throw new Error('memory improvement maxTotalCostUsd must be a non-negative finite number') } + if ( + options.activationTimeoutMs !== undefined && + (!Number.isSafeInteger(options.activationTimeoutMs) || options.activationTimeoutMs <= 0) + ) { + throw new Error('memory improvement activationTimeoutMs must be a positive safe integer') + } const tolerance = options.criticalDimensionTolerance if (tolerance !== undefined && (!Number.isFinite(tolerance) || tolerance < 0 || tolerance > 1)) { throw new Error('memory improvement criticalDimensionTolerance must be between 0 and 1') diff --git a/src/memory/lifecycle.ts b/src/memory/lifecycle.ts index 9195151..4f907a1 100644 --- a/src/memory/lifecycle.ts +++ b/src/memory/lifecycle.ts @@ -55,10 +55,16 @@ export function memoryRecoveryDelayMs(adapter: AgentMemoryAdapter): number { export async function sleepForMemoryRecovery( delayMs: number, assertOwned: () => Promise, + timeoutMs = delayMs, + operation = 'memory recovery visibility wait', ): Promise { if (delayMs <= 0) return - await new Promise((resolve) => setTimeout(resolve, delayMs)) + if (!Number.isSafeInteger(timeoutMs) || timeoutMs <= 0) { + throw new Error(`${operation} timeout must be a positive safe integer`) + } + await new Promise((resolve) => setTimeout(resolve, Math.min(delayMs, timeoutMs))) await assertOwned() + if (delayMs > timeoutMs) throw new AgentMemoryLifecycleTimeoutError(operation, timeoutMs) } export function createMemoryExecutionPool(limit: number): { diff --git a/src/memory/mem0.ts b/src/memory/mem0.ts index ca0f0f2..f6e2124 100644 --- a/src/memory/mem0.ts +++ b/src/memory/mem0.ts @@ -23,7 +23,6 @@ export interface Mem0ClientLike { search(query: string, options?: Record): Promise getAll?(options?: Record): Promise delete?(memoryId: string, options?: Record): Promise - deleteAll?(options?: Record): Promise } export interface Mem0MemoryAdapterOptions { @@ -39,6 +38,8 @@ export interface Mem0MemoryAdapterOptions { getEvent?: (eventId: string) => Promise ingestionTimeoutMs?: number pollIntervalMs?: number + /** Stable deployment/account identity for cache-key helpers. Never put credentials here. */ + backendRef?: string defaultScope?: AgentMemoryScope } @@ -159,9 +160,14 @@ export function createMem0MemoryAdapter(options: Mem0MemoryAdapterOptions): Agen }, async clear(scope) { const mergedScope = mergeScopes(options.defaultScope, scope) + const clearFilters = mem0Filters(mergedScope, options.appId) + if (Object.keys(clearFilters).length === 0) { + throw new Error(`${id}: refusing an unscoped Mem0 clear`) + } await settlePendingMem0Writes(options, pendingWrites, mergedScope, false) if (options.client.getAll && options.client.delete) { const deletedIds = new Set() + const deletedSearchProbes = new Map>() const visibilityTimeoutMs = options.ingestionTimeoutMs ?? 30_000 const pollIntervalMs = options.pollIntervalMs ?? 250 let visibilityDeadline = Date.now() + visibilityTimeoutMs @@ -182,12 +188,41 @@ export function createMem0MemoryAdapter(options: Mem0MemoryAdapterOptions): Agen }, ) const items = extractMem0Items(raw) - if (items.length === 0) return + if (items.length === 0) { + if ( + !(await deletedMem0IdsRemainSearchable( + options, + clearFilters, + deletedIds, + deletedSearchProbes, + )) + ) { + return + } + const remainingMs = visibilityDeadline - Date.now() + if (remainingMs <= 0) { + throw new Error( + `${id}: deleted Mem0 memories remained searchable after ${visibilityTimeoutMs}ms`, + ) + } + await sleep(Math.min(pollIntervalMs, remainingMs)) + continue + } const rawIds = items.map((item) => stringField(item, ['id'])) if (!rawIds.every((memoryId): memoryId is string => memoryId !== undefined)) { throw new Error(`${id}: Mem0 returned memories without ids during scoped clear`) } const ids = [...new Set(rawIds)] + for (const item of items) { + const memoryId = stringField(item, ['id'])! + const text = + stringField(item, ['memory', 'text', 'content']) ?? + stringField(recordField(item, 'data'), ['memory', 'text', 'content']) + if (!text) continue + const idsForText = deletedSearchProbes.get(text) ?? new Set() + idsForText.add(memoryId) + deletedSearchProbes.set(text, idsForText) + } const unseenIds = ids.filter((memoryId) => !deletedIds.has(memoryId)) if (unseenIds.length === 0) { const remainingMs = visibilityDeadline - Date.now() @@ -210,44 +245,7 @@ export function createMem0MemoryAdapter(options: Mem0MemoryAdapterOptions): Agen `${id}: refused to clear more than ${deletedIds.size} Mem0 memories in one call`, ) } - if ( - !options.client.deleteAll || - !options.client.getAll || - !canDeleteAllExactly(mergedScope) - ) { - throw new Error( - `${id}: exact scoped clear requires Mem0 getAll plus delete or an exact deleteAll`, - ) - } - await options.client.deleteAll(mem0Entity(mergedScope, options.appId)) - const visibilityTimeoutMs = options.ingestionTimeoutMs ?? 30_000 - const pollIntervalMs = options.pollIntervalMs ?? 250 - const visibilityDeadline = Date.now() + visibilityTimeoutMs - for (;;) { - const raw = await options.client.getAll( - options.mode === 'hosted' - ? { - filters: mem0Filters(mergedScope, options.appId), - page: 1, - pageSize: 1, - latestOnly: false, - showExpired: true, - } - : { - filters: mem0Filters(mergedScope, options.appId), - topK: 1, - showExpired: true, - }, - ) - if (extractMem0Items(raw).length === 0) return - const remainingMs = visibilityDeadline - Date.now() - if (remainingMs <= 0) { - throw new Error( - `${id}: bulk-deleted Mem0 memories remained visible after ${visibilityTimeoutMs}ms`, - ) - } - await sleep(Math.min(pollIntervalMs, remainingMs)) - } + throw new Error(`${id}: exact scoped clear requires Mem0 getAll plus per-memory delete`) }, async flush() { await settlePendingMem0Writes(options, pendingWrites, undefined, true) @@ -263,6 +261,12 @@ function assertMem0Options(options: Mem0MemoryAdapterOptions): void { if (options.appId !== undefined && (typeof options.appId !== 'string' || !options.appId.trim())) { throw new Error('Mem0 appId must be a non-empty string') } + if ( + options.backendRef !== undefined && + (typeof options.backendRef !== 'string' || !options.backendRef.trim()) + ) { + throw new Error('Mem0 backendRef must be a non-empty string') + } if (options.mode === 'oss' && options.consistency === 'queued') { throw new Error('Mem0 OSS writes are synchronous and do not support consistency="queued"') } @@ -507,14 +511,35 @@ function mem0CustomScopeMetadata(scope: AgentMemoryScope): Record, + deletedIds: ReadonlySet, + probes: ReadonlyMap>, +): Promise { + const entries = [...probes] + for (let offset = 0; offset < entries.length; offset += 8) { + const results = await Promise.all( + entries.slice(offset, offset + 8).map(async ([query, expectedIds]) => { + const raw = await options.client.search(query, { + filters, + topK: Math.min(100, Math.max(10, expectedIds.size)), + ...(options.mode === 'hosted' ? { latestOnly: false, showExpired: true } : {}), + }) + return extractMem0Items(raw).map((item) => { + const memoryId = stringField(item, ['id']) + if (!memoryId) { + throw new Error( + `${options.id ?? `mem0-${options.mode}`}: Mem0 search returned a memory without an id during clear verification`, + ) + } + return memoryId + }) + }), + ) + if (results.some((ids) => ids.some((memoryId) => deletedIds.has(memoryId)))) return true + } + return false } function mem0Role(role: AgentMemoryWriteInput['role']): 'user' | 'assistant' { @@ -639,7 +664,10 @@ export function mem0MemoryAdapterIdentity( | 'ingestionTimeoutMs' | 'pollIntervalMs' | 'defaultScope' - >, + > & { backendRef: string }, ): string { + if (typeof options.backendRef !== 'string' || !options.backendRef.trim()) { + throw new Error('Mem0 backendRef must be a non-empty string') + } return stableId('mem0', canonicalJson(compactRecord(options))) } diff --git a/src/memory/neo4j.ts b/src/memory/neo4j.ts index 0d90166..a8c91b7 100644 --- a/src/memory/neo4j.ts +++ b/src/memory/neo4j.ts @@ -236,20 +236,38 @@ async function writeNeo4jMemory( stringMetadata(input.metadata, 'task') ?? input.title ?? input.text, ], ) + assertNeo4jWriteSucceeded(trace, adapterId, 'startTrace') const traceId = idFromResult(trace) if (!traceId) throw new Error(`${adapterId}: Neo4j startTrace returned no trace id`) - await callRequired( - reasoning, - ['addStep'], - [ - traceId, - { - thought: input.text, - observation: stringMetadata(input.metadata, 'observation'), - action: stringMetadata(input.metadata, 'action'), - }, - ], - ) + try { + const step = await callRequired( + reasoning, + ['addStep'], + [ + traceId, + { + thought: input.text, + observation: stringMetadata(input.metadata, 'observation'), + action: stringMetadata(input.metadata, 'action'), + }, + ], + ) + assertNeo4jWriteSucceeded(step, adapterId, 'addStep') + } catch (error) { + try { + await callRequired( + reasoning, + ['completeTrace'], + [traceId, { outcome: 'agent-knowledge addStep failed', success: false }], + ) + } catch (cleanupError) { + throw new AggregateError( + [error, cleanupError], + `${adapterId}: Neo4j reasoning step and trace cleanup failed`, + ) + } + throw error + } result = await callRequired( reasoning, ['completeTrace'], @@ -276,6 +294,8 @@ async function writeNeo4jMemory( ) } + assertNeo4jWriteSucceeded(result, adapterId, input.kind) + const id = idFromResult(result) ?? input.id ?? @@ -292,6 +312,25 @@ async function writeNeo4jMemory( } } +function assertNeo4jWriteSucceeded(value: unknown, adapterId: string, operation: string): void { + const response = record(value) + if (!response) return + const status = stringField(response, ['status'])?.toLowerCase() + if ( + response.success === false || + response.ok === false || + response.isError === true || + status === 'failed' || + status === 'error' + ) { + const nestedError = record(response.error) + const detail = + stringField(response, ['error', 'message', 'detail']) ?? + (nestedError ? stringField(nestedError, ['message', 'detail']) : undefined) + throw new Error(`${adapterId}: Neo4j ${operation} failed${detail ? `: ${detail}` : ''}`) + } +} + async function searchNeo4jMemory( client: Record, query: string, diff --git a/tests/benchmarks.test.ts b/tests/benchmarks.test.ts index 98ebe83..dd4ca63 100644 --- a/tests/benchmarks.test.ts +++ b/tests/benchmarks.test.ts @@ -1,4 +1,4 @@ -import { inMemoryCampaignStorage } from '@tangle-network/agent-eval/campaign' +import { createRunCostLedger, inMemoryCampaignStorage } from '@tangle-network/agent-eval/campaign' import { describe, expect, it } from 'vitest' import { buildFirstPartyMemoryLifecycleBenchmarkCases, @@ -104,6 +104,7 @@ describe('knowledge benchmark adapters', () => { runDir: '/runs/knowledge-benchmark-smoke', storage, respondRef: 'retriever-fixture:v1', + costCeiling: 1, respond: async ({ case: testCase, context }) => { const paid = await context.cost.runPaidCall({ actor: 'benchmark-retriever', @@ -426,6 +427,7 @@ describe('knowledge benchmark adapters', () => { runDir: '/runs/direct-benchmark-recovery', storage, candidates: [candidate], + costCeiling: 1, }) await expect(run()).rejects.toThrow('memory benchmark attempt cleanup failed') @@ -451,6 +453,78 @@ describe('knowledge benchmark adapters', () => { ]) }) + it('reconciles a crash after direct recovery but before its cost receipt', async () => { + const storage = inMemoryCampaignStorage() + const append = storage.append!.bind(storage) + let failRecoveryReceipt = false + storage.append = (path, value, expectedBytes) => { + if ( + failRecoveryReceipt && + path.endsWith('/cost-ledger.jsonl') && + value.includes('"status":"settled"') && + value.includes('memory-adapter-recovery') + ) { + failRecoveryReceipt = false + throw new Error('simulated process exit before benchmark recovery receipt') + } + return append(path, value, expectedBytes) + } + + let firstExecution = true + let recoveryClears = 0 + const candidate = { + id: 'receipt-crash', + ref: 'receipt-crash:v1', + costUsdPerCase: 0.1, + recoveryCostUsdPerAttempt: 0.1, + createAdapter({ purpose }: { purpose: 'execute' | 'recovery' }) { + const adapter = createInMemoryBenchmarkAdapter({ id: 'receipt-crash-provider' }) + const clear = adapter.clear! + const failThisExecution = purpose === 'execute' && firstExecution + if (purpose === 'execute') firstExecution = false + adapter.clear = async (scope) => { + if (purpose === 'recovery') recoveryClears += 1 + if (failThisExecution) throw new Error('leave direct benchmark state active') + await clear(scope) + } + return adapter + }, + } + const run = () => + runMemoryAdapterBenchmark({ + cases: buildFirstPartyMemoryLifecycleBenchmarkCases().slice(0, 1), + runDir: '/runs/direct-recovery-receipt-crash', + storage, + costCeiling: 1, + candidates: [candidate], + }) + + await expect(run()).rejects.toThrow('memory benchmark attempt cleanup failed') + failRecoveryReceipt = true + await expect(run()).rejects.toThrow('failed to persist') + expect( + storage + .read('/runs/direct-recovery-receipt-crash/memory-adapter-attempts.jsonl') + ?.trim() + .split('\n'), + ).toHaveLength(1) + + const result = await run() + const costLedger = createRunCostLedger({ + storage, + runDir: '/runs/direct-recovery-receipt-crash', + costCeilingUsd: 1, + }) + + expect(result.rows[0]).toMatchObject({ candidateId: 'receipt-crash', cellsFailed: 0 }) + expect(recoveryClears).toBe(2) + expect(costLedger.summary()).toMatchObject({ + unresolvedCalls: 0, + totalCostUsd: 0.4, + accountingComplete: true, + }) + }) + it('bounds direct benchmark cleanup and preserves its recovery record', async () => { const storage = inMemoryCampaignStorage() const adapter = createInMemoryBenchmarkAdapter({ id: 'hung-cleanup' }) @@ -508,6 +582,54 @@ describe('knowledge benchmark adapters', () => { }) }) + it('runs sequential paid adapter cases after exact external-cost receipts', async () => { + const result = await runMemoryAdapterBenchmark({ + cases: buildFirstPartyMemoryLifecycleBenchmarkCases().slice(0, 2), + runDir: '/runs/sequential-adapter-cost-accounting', + storage: inMemoryCampaignStorage(), + costCeiling: 0.2, + maxConcurrency: 1, + candidates: [ + { + id: 'sequential-paid', + ref: 'sequential-paid:v1', + costUsdPerCase: 0.1, + createAdapter: () => createInMemoryBenchmarkAdapter({ id: 'sequential-paid' }), + }, + ], + }) + + expect(result.rows[0]).toMatchObject({ cellsFailed: 0, totalCostUsd: 0.2 }) + expect(result.totalCostUsd).toBe(0.2) + }) + + it('refuses paid adapter calls when no dollar limit is configured', async () => { + let providerCalls = 0 + const adapter = createInMemoryBenchmarkAdapter({ id: 'paid-by-default' }) + const write = adapter.write.bind(adapter) + adapter.write = async (input) => { + providerCalls += 1 + return write(input) + } + + const result = await runMemoryAdapterBenchmark({ + cases: buildFirstPartyMemoryLifecycleBenchmarkCases().slice(0, 1), + runDir: '/runs/default-zero-cost-ceiling', + storage: inMemoryCampaignStorage(), + candidates: [ + { + id: 'paid-by-default', + ref: 'paid-by-default:v1', + costUsdPerCase: 0.01, + createAdapter: () => adapter, + }, + ], + }) + + expect(result.rows[0]).toMatchObject({ cellsFailed: 1, totalCostUsd: 0 }) + expect(providerCalls).toBe(0) + }) + it('does not reuse resumable rows when adapter benchmark options change', async () => { const storage = inMemoryCampaignStorage() const run = (searchLimit: number) => diff --git a/tests/memory-systems.test.ts b/tests/memory-systems.test.ts index 83bdc61..7a2e542 100644 --- a/tests/memory-systems.test.ts +++ b/tests/memory-systems.test.ts @@ -397,7 +397,7 @@ describe('Mem0 adapter', () => { }) it('includes the default scope in the Mem0 adapter identity', () => { - const base = { mode: 'hosted' as const, id: 'mem0' } + const base = { mode: 'hosted' as const, id: 'mem0', backendRef: 'mem0-account-a' } expect(mem0MemoryAdapterIdentity({ ...base, defaultScope: { tenantId: 'tenant-a' } })).not.toBe( mem0MemoryAdapterIdentity({ ...base, defaultScope: { tenantId: 'tenant-b' } }), ) @@ -525,10 +525,11 @@ describe('Mem0 adapter', () => { expect(getAllCalls).toBeGreaterThan(2) }) - it('waits for bulk-deleted memories to disappear before reporting exact cleanup', async () => { + it('waits for list and search indexes to drop deleted memories before reporting cleanup', async () => { let deletedAt: number | undefined let getAllCalls = 0 - let deleteAllCalls = 0 + let searchCalls = 0 + let deleteCalls = 0 const adapter = createMem0MemoryAdapter({ mode: 'hosted', ingestionTimeoutMs: 50, @@ -538,15 +539,17 @@ describe('Mem0 adapter', () => { return [] }, async search() { - return { results: [] } + searchCalls += 1 + const visible = deletedAt === undefined || Date.now() - deletedAt < 8 + return { results: visible ? [{ id: 'memory-1', memory: 'stale listing' }] : [] } }, async getAll() { getAllCalls += 1 const visible = deletedAt === undefined || Date.now() - deletedAt < 5 return { results: visible ? [{ id: 'memory-1', memory: 'stale listing' }] : [] } }, - async deleteAll() { - deleteAllCalls += 1 + async delete() { + deleteCalls += 1 deletedAt = Date.now() return { message: 'deletion accepted' } }, @@ -555,8 +558,36 @@ describe('Mem0 adapter', () => { await adapter.clear?.({ userId: 'user-1', namespace: 'branch-1' }) - expect(deleteAllCalls).toBe(1) + expect(deleteCalls).toBe(1) expect(getAllCalls).toBeGreaterThan(1) + expect(searchCalls).toBeGreaterThan(1) + }) + + it('refuses to clear Mem0 without an exact identity scope', async () => { + let providerCalls = 0 + const adapter = createMem0MemoryAdapter({ + mode: 'hosted', + client: { + async add() { + return [] + }, + async search() { + providerCalls += 1 + return { results: [] } + }, + async getAll() { + providerCalls += 1 + return { results: [] } + }, + async delete() { + providerCalls += 1 + return { message: 'deleted' } + }, + }, + }) + + await expect(adapter.clear?.()).rejects.toThrow('refusing an unscoped Mem0 clear') + expect(providerCalls).toBe(0) }) }) @@ -993,9 +1024,17 @@ describe('Graphiti adapter', () => { it('includes the default scope in the Graphiti adapter identity', () => { expect( - graphitiMemoryAdapterIdentity({ id: 'graphiti', defaultScope: { namespace: 'one' } }), + graphitiMemoryAdapterIdentity({ + id: 'graphiti', + backendRef: 'graphiti-cluster-a', + defaultScope: { namespace: 'one' }, + }), ).not.toBe( - graphitiMemoryAdapterIdentity({ id: 'graphiti', defaultScope: { namespace: 'two' } }), + graphitiMemoryAdapterIdentity({ + id: 'graphiti', + backendRef: 'graphiti-cluster-a', + defaultScope: { namespace: 'two' }, + }), ) }) }) @@ -1089,6 +1128,27 @@ describe('Neo4j Agent Memory adapter', () => { ) }) + it('rejects non-throwing Neo4j write errors', async () => { + const adapter = createNeo4jAgentMemoryAdapter({ + transport: 'rest', + client: { + shortTerm: { + async addMessage() { + return { success: false, error: { message: 'database unavailable' } } + }, + }, + }, + }) + + await expect( + adapter.write({ + kind: 'message', + text: 'Remember this.', + scope: { sessionId: 'session-1' }, + }), + ).rejects.toThrow('database unavailable') + }) + it('maps hosted reasoning memories to recordStep', async () => { let recorded: Record | undefined const adapter = createNeo4jAgentMemoryAdapter({ @@ -1181,6 +1241,19 @@ describe('Neo4j Agent Memory adapter', () => { }) describe('memory branches', () => { + it('rejects two live handles for the same adapter branch', async () => { + const adapter = createScopedTestAdapter('duplicate-handle') + const first = createAgentMemoryBranch({ adapter, branchId: 'same-branch' }) + + expect(() => createAgentMemoryBranch({ adapter, branchId: 'same-branch' })).toThrow( + "memory branch 'same-branch' already has an open handle", + ) + + await first.close?.() + const resumed = createAgentMemoryBranch({ adapter, branchId: 'same-branch' }) + await resumed.close?.() + }) + it('rejects adapters that do not declare how branches are isolated', () => { const { branchIsolation: _branchIsolation, ...legacy } = createScopedTestAdapter('legacy') @@ -1742,6 +1815,7 @@ describe('memory branches', () => { const snapshot = await child.snapshot() expect(snapshot.journal.every((entry) => Boolean(entry.input.id))).toBe(true) expect(new Set(snapshot.journal.map((entry) => entry.input.id)).size).toBe(2) + await child.close?.() const resumed = createAgentMemoryBranch({ adapter: childStorage, branchId: 'child', @@ -1860,7 +1934,7 @@ describe('agent memory experiments', () => { ).toHaveLength(1) }) - it('charges conservative external cost once adapter creation is attempted', async () => { + it('does not charge provider cost when side-effect-free adapter construction fails', async () => { const storage = inMemoryCampaignStorage() const costLedger = createRunCostLedger({ storage, @@ -1894,7 +1968,225 @@ describe('agent memory experiments', () => { }), ).rejects.toThrow('memory experiment cleanup failed after dispatch') - expect(costLedger.summary().totalCostUsd).toBe(0.25) + expect(costLedger.summary().totalCostUsd).toBe(0) + }) + + it('accounts for parallel candidates against one shared dollar limit', async () => { + let releaseFirstCalls: (() => void) | undefined + const firstCallsReady = new Promise((resolve) => { + releaseFirstCalls = resolve + }) + let activeCalls = 0 + let firstCalls = 0 + let maxActiveCalls = 0 + const createAdapter = (id: string): AgentMemoryAdapter => { + const adapter = createScopedTestAdapter(id) + const clear = adapter.clear! + let entered = false + adapter.clear = async (scope) => { + if (!entered) { + entered = true + firstCalls += 1 + activeCalls += 1 + maxActiveCalls = Math.max(maxActiveCalls, activeCalls) + if (firstCalls === 2) releaseFirstCalls?.() + await firstCallsReady + activeCalls -= 1 + } + await clear(scope) + } + return adapter + } + + const result = await runAgentMemoryExperiment({ + experimentId: 'parallel-shared-cost', + sequences: [ + { + id: 'history', + family: 'first-party', + steps: [ + { + id: 'remember', + scope: { agentId: 'worker' }, + writes: [{ kind: 'fact', text: 'parallel fact' }], + probes: [ + { + id: 'recall', + query: 'parallel', + scope: { agentId: 'worker' }, + referenceAnswer: 'parallel fact', + }, + ], + }, + ], + }, + ], + candidates: ['first', 'second'].map((id) => ({ + id, + ref: `${id}:v1`, + externalCostUsdPerSequence: 0.1, + createAdapter: () => createAdapter(id), + })), + runDir: '/runs/parallel-shared-cost', + storage: inMemoryCampaignStorage(), + costCeiling: 0.2, + maxConcurrency: 2, + }) + + expect(maxActiveCalls).toBe(2) + expect(result.rows.map((row) => row.totalCostUsd).sort()).toEqual([0.1, 0.1]) + expect(result.campaign.aggregates.totalCostUsd).toBe(0.2) + }) + + it('runs sequential paid histories after exact external-cost receipts', async () => { + const storage = inMemoryCampaignStorage() + const costLedger = createRunCostLedger({ + storage, + runDir: '/runs/sequential-shared-cost', + costCeilingUsd: 0.2, + }) + const sequences = ['first', 'second'].map((id) => ({ + id, + family: 'first-party', + steps: [ + { + id: 'remember', + scope: { agentId: 'worker' }, + writes: [{ kind: 'fact' as const, text: `${id} fact` }], + probes: [ + { + id: 'recall', + query: id, + scope: { agentId: 'worker' }, + referenceAnswer: `${id} fact`, + }, + ], + }, + ], + })) + + const result = await runAgentMemoryExperiment({ + experimentId: 'sequential-shared-cost', + sequences, + candidates: [ + { + id: 'memory', + ref: 'memory:v1', + externalCostUsdPerSequence: 0.1, + createAdapter: ({ sequence }) => createScopedTestAdapter(sequence.id), + }, + ], + runDir: '/runs/sequential-shared-cost', + storage, + costLedger, + maxConcurrency: 1, + }) + + expect(result.rows[0]).toMatchObject({ cellsFailed: 0, totalCostUsd: 0.2 }) + expect(costLedger.summary()).toMatchObject({ + totalCalls: 2, + totalCostUsd: 0.2, + accountingComplete: true, + }) + }) + + it('reconciles interrupted paid calls from every parallel memory branch before resuming', async () => { + const storage = inMemoryCampaignStorage() + const runDir = '/runs/parallel-interrupted-cost-recovery' + const abandonedLedger = createRunCostLedger({ storage, runDir, costCeilingUsd: 1 }) + const controllers = [new AbortController(), new AbortController()] + const releases: Array<() => void> = [] + let started = 0 + let reportStarted: (() => void) | undefined + const bothStarted = new Promise((resolve) => { + reportStarted = resolve + }) + const abandonedCalls = ['one', 'two'].map((branch, index) => + abandonedLedger.runPaidCall({ + callId: `abandoned-${branch}`, + channel: 'agent', + phase: 'memory.train', + actor: `agent-knowledge:memory-experiment:${branch}`, + model: `provider-${branch}`, + signal: controllers[index]!.signal, + maximumCharge: { externallyEnforcedMaximumUsd: 0.1 }, + async execute() { + started += 1 + if (started === 2) reportStarted?.() + return await new Promise((resolve) => { + releases[index] = () => resolve('late provider result') + }) + }, + receipt: () => ({ + model: `provider-${branch}`, + inputTokens: 0, + outputTokens: 0, + actualCostUsd: 0.05, + }), + }), + ) + + await bothStarted + for (const controller of controllers) { + controller.abort(new Error('simulated process exit')) + } + await Promise.all(abandonedCalls) + expect(abandonedLedger.listPending().map((call) => call.state)).toEqual(['late', 'late']) + + const resumedLedger = createRunCostLedger({ storage, runDir, costCeilingUsd: 1 }) + expect(resumedLedger.listPending().map((call) => call.state)).toEqual([ + 'interrupted', + 'interrupted', + ]) + + const result = await runAgentMemoryExperiment({ + experimentId: 'parallel-interrupted-cost-recovery', + sequences: [ + { + id: 'history', + family: 'first-party', + steps: [ + { + id: 'remember', + scope: { agentId: 'worker' }, + writes: [{ kind: 'fact', text: 'resumed fact' }], + probes: [ + { + id: 'recall', + query: 'resumed', + referenceAnswer: 'resumed fact', + }, + ], + }, + ], + }, + ], + candidates: [ + { + id: 'resumed', + ref: 'resumed:v1', + createAdapter: () => createScopedTestAdapter('resumed'), + }, + ], + runDir, + storage, + costLedger: resumedLedger, + }) + + expect(result.rows[0]).toMatchObject({ candidateId: 'resumed', cellsFailed: 0 }) + expect(resumedLedger.summary()).toMatchObject({ + totalCalls: 2, + unresolvedCalls: 0, + totalCostUsd: 0.2, + accountingComplete: true, + }) + expect(resumedLedger.list().map((receipt) => receipt.error)).toEqual([ + expect.stringContaining('charged the reserved maximum'), + expect.stringContaining('charged the reserved maximum'), + ]) + + for (const release of releases) release() + await abandonedLedger.waitForIdle() }) it('does not reuse cells after the candidate implementation reference changes', async () => { @@ -2405,6 +2697,7 @@ describe('agent memory experiments', () => { candidates: [candidate], runDir: '/runs/restart-recovery', storage, + costCeiling: 1, }) await expect(run()).rejects.toThrow('memory experiment cleanup failed after dispatch') @@ -2432,6 +2725,99 @@ describe('agent memory experiments', () => { ]) }) + it('reconciles a crash after provider recovery but before its cost receipt', async () => { + const storage = inMemoryCampaignStorage() + const append = storage.append!.bind(storage) + let failRecoveryReceipt = false + storage.append = (path, value, expectedBytes) => { + if ( + failRecoveryReceipt && + path.endsWith('/cost-ledger.jsonl') && + value.includes('"status":"settled"') && + value.includes('agent-knowledge:memory-recovery') + ) { + failRecoveryReceipt = false + throw new Error('simulated process exit before recovery receipt') + } + return append(path, value, expectedBytes) + } + + let firstExecution = true + let recoveryClears = 0 + const candidate = { + id: 'crash-recoverable', + ref: 'crash-recoverable:v1', + externalCostUsdPerSequence: 0.1, + externalRecoveryCostUsdPerAttempt: 0.1, + createAdapter({ purpose }: { purpose: 'execute' | 'recovery' }) { + const adapter = createScopedTestAdapter(`crash-recoverable:${purpose}`) + const clear = adapter.clear! + let clearCalls = 0 + const failThisExecution = purpose === 'execute' && firstExecution + if (purpose === 'execute') firstExecution = false + adapter.clear = async (scope) => { + clearCalls += 1 + if (purpose === 'recovery') recoveryClears += 1 + if (failThisExecution && clearCalls === 2) { + throw new Error('leave the first branch active') + } + await clear(scope) + } + return adapter + }, + } + const run = () => + runAgentMemoryExperiment({ + experimentId: 'recovery-receipt-crash', + sequences: [ + { + id: 'history', + family: 'first-party', + steps: [ + { + id: 'remember', + scope: { agentId: 'worker' }, + writes: [{ kind: 'fact', text: 'durable recovery fact' }], + probes: [ + { + id: 'recall', + query: 'durable', + referenceAnswer: 'durable recovery fact', + }, + ], + }, + ], + }, + ], + candidates: [candidate], + runDir: '/runs/recovery-receipt-crash', + storage, + costCeiling: 1, + }) + + await expect(run()).rejects.toThrow('memory experiment cleanup failed after dispatch') + failRecoveryReceipt = true + await expect(run()).rejects.toThrow('failed to persist') + expect( + storage.read('/runs/recovery-receipt-crash/memory-attempts.jsonl')?.trim().split('\n'), + ).toHaveLength(1) + + const result = await run() + const costLedger = createRunCostLedger({ + storage, + runDir: '/runs/recovery-receipt-crash', + costCeilingUsd: 1, + }) + + expect(result.rows[0]).toMatchObject({ candidateId: 'crash-recoverable', cellsFailed: 0 }) + expect(recoveryClears).toBe(2) + expect(costLedger.summary()).toMatchObject({ + unresolvedCalls: 0, + totalCostUsd: 0.4, + accountingComplete: true, + }) + }) + it('recovers independent abandoned branches in parallel before retrying them', async () => { const storage = inMemoryCampaignStorage() let phase: 'leave-active' | 'recover' = 'leave-active' @@ -2525,6 +2911,7 @@ describe('agent memory experiments', () => { { id: 'sometimes-created', ref: 'sometimes-created:v1', + externalRecoveryCostUsdPerAttempt: 0.1, createAdapter({ purpose }) { purposes.push(purpose) if (purpose === 'recovery') return null @@ -2536,12 +2923,14 @@ describe('agent memory experiments', () => { ], runDir: '/runs/provider-creation-recovery', storage, + costCeiling: 1, }) await expect(run()).rejects.toThrow('memory experiment cleanup failed after dispatch') const result = await run() expect(result.rows[0]).toMatchObject({ candidateId: 'sometimes-created', cellsFailed: 0 }) + expect(result.rows[0]?.totalCostUsd).toBe(0) expect(purposes).toEqual(['execute', 'recovery', 'execute']) const attemptEvents = storage .read(result.attemptLogPath)! @@ -2556,6 +2945,148 @@ describe('agent memory experiments', () => { ]) }) + it('uses a retired candidate only to clean its unfinished branch', async () => { + const storage = inMemoryCampaignStorage() + const runDir = '/runs/retired-recovery-candidate' + const sequence = { + id: 'history', + family: 'first-party' as const, + steps: [ + { + id: 'remember', + scope: { agentId: 'worker' }, + writes: [{ kind: 'fact' as const, text: 'active fact' }], + probes: [{ id: 'recall', query: 'active', referenceAnswer: 'active fact' }], + }, + ], + } + storage.write( + `${runDir}/memory-attempts.jsonl`, + `${JSON.stringify({ + schema: 1, + status: 'started', + branchId: 'retired-branch', + candidateId: 'retired', + candidateRef: 'retired:v1', + sequenceId: sequence.id, + rep: 0, + seed: 1, + cleanupBranches: true, + recordedAt: '2026-01-01T00:00:00.000Z', + recovery: false, + })}\n`, + ) + const purposes: string[] = [] + let retiredClears = 0 + const result = await runAgentMemoryExperiment({ + experimentId: 'retired-recovery-candidate', + sequences: [sequence], + candidates: [ + { + id: 'active', + ref: 'active:v1', + createAdapter({ purpose }) { + purposes.push(`active:${purpose}`) + return createScopedTestAdapter('active') + }, + }, + ], + recoveryCandidates: [ + { + id: 'retired', + ref: 'retired:v1', + createAdapter({ purpose }) { + purposes.push(`retired:${purpose}`) + const adapter = createScopedTestAdapter('retired') + adapter.clear = async () => { + retiredClears += 1 + } + return adapter + }, + }, + ], + runDir, + storage, + resumable: false, + }) + + expect(result.rows).toHaveLength(1) + expect(result.rows[0]?.candidateId).toBe('active') + expect(purposes).toEqual(['retired:recovery', 'active:execute']) + expect(retiredClears).toBeGreaterThan(0) + }) + + it('refuses a recovery backlog larger than maxRecoveryAttempts', async () => { + const storage = inMemoryCampaignStorage() + const runDir = '/runs/recovery-backlog-limit' + const sequence = { + id: 'history', + family: 'first-party' as const, + steps: [ + { + id: 'probe', + scope: { agentId: 'worker' }, + probes: [{ id: 'recall', query: 'fact', referenceAnswer: 'fact' }], + }, + ], + } + storage.write( + `${runDir}/memory-attempts.jsonl`, + `${[ + { + schema: 1, + status: 'started', + branchId: 'branch-1', + candidateId: 'memory', + candidateRef: 'memory:v1', + sequenceId: sequence.id, + rep: 0, + seed: 1, + cleanupBranches: true, + recordedAt: '2026-01-01T00:00:00.000Z', + recovery: false, + }, + { + schema: 1, + status: 'started', + branchId: 'branch-2', + candidateId: 'memory', + candidateRef: 'memory:v1', + sequenceId: sequence.id, + rep: 0, + seed: 2, + cleanupBranches: true, + recordedAt: '2026-01-01T00:00:00.000Z', + recovery: false, + }, + ] + .map((event) => JSON.stringify(event)) + .join('\n')}\n`, + ) + let providerCreates = 0 + + await expect( + runAgentMemoryExperiment({ + experimentId: 'recovery-backlog-limit', + sequences: [sequence], + candidates: [ + { + id: 'memory', + ref: 'memory:v1', + createAdapter() { + providerCreates += 1 + return createScopedTestAdapter('memory') + }, + }, + ], + runDir, + storage, + maxRecoveryAttempts: 1, + }), + ).rejects.toThrow('2 unfinished attempts; maxRecoveryAttempts is 1') + expect(providerCreates).toBe(0) + }) + it('allows one controller per run while its history workers run in parallel', async () => { const storage = inMemoryCampaignStorage() let reportStarted: (() => void) | undefined @@ -2731,8 +3262,15 @@ describe('agent memory experiments', () => { expect(storage.append!(result.attemptLogPath, `${last}\n`, Buffer.byteLength(journal))).toBe( Buffer.byteLength(journal) + Buffer.byteLength(`${last}\n`), ) + const callsBeforeCachedRun = { + purposes: purposes.length, + searches, + clears, + closes, + } const cached = await run() expect(cached.campaign.aggregates.cellsCached).toBe(1) + expect({ purposes: purposes.length, searches, clears, closes }).toEqual(callsBeforeCachedRun) }) it('preserves both the run failure and controller release failure', async () => { @@ -3142,6 +3680,16 @@ describe('agent memory experiments', () => { }) describe('agent memory improvement', () => { + it('fails fast when a JavaScript caller uses the removed onPromote option', async () => { + await expect( + runAgentMemoryImprovementRaw({ + onPromote() {}, + } as unknown as RunAgentMemoryImprovementOptions), + ).rejects.toThrow( + 'onPromote was removed; use activation.readCurrent and activation.compareAndSet', + ) + }) + it('requires an explicit controller policy for custom improvement storage', async () => { await expect( runAgentMemoryImprovementRaw({ @@ -3172,6 +3720,7 @@ describe('agent memory improvement', () => { type Config = { visibility: 'private' | 'team' | 'shared' } const storage = inMemoryCampaignStorage() const promoted: Config[] = [] + let activeConfig: Config = { visibility: 'private' } const activationIds: string[] = [] const contenderIds = new Set() const activeContenderCalls = new Map() @@ -3253,9 +3802,17 @@ describe('agent memory improvement', () => { else activeContenderCalls.set(candidateId, remaining) } }, - onPromote: async ({ activationId, config }) => { - activationIds.push(activationId) - promoted.push(config) + activation: { + ref: 'memory-policy/live:v1', + async readCurrent() { + return structuredClone(activeConfig) + }, + async compareAndSet({ activationId, expectedConfig, config }) { + expect(activeConfig).toEqual(expectedConfig) + activationIds.push(activationId) + activeConfig = structuredClone(config) + promoted.push(structuredClone(config)) + }, }, } const firstRun = runAgentMemoryImprovement(options) @@ -3281,7 +3838,7 @@ describe('agent memory improvement', () => { expect( JSON.parse(storage.read('/runs/improve-team-memory/memory-improvement-manifest.json') ?? '{}') .identity?.schema, - ).toBe(4) + ).toBe(5) const resumed = await runAgentMemoryImprovement(options) expect(proposalCalls).toBe(1) @@ -3291,11 +3848,7 @@ describe('agent memory improvement', () => { ...result.activation, status: 'already-activated', }) - const resumedWithoutCallback = await runAgentMemoryImprovement({ - ...options, - onPromote: undefined, - }) - expect(resumedWithoutCallback.activation.status).toBe('already-activated') + expect(activeConfig).toEqual({ visibility: 'team' }) await expect( runAgentMemoryImprovement({ ...options, budget: { maxSteps: 2 } }), ).rejects.toThrow('does not match its persisted inputs or implementationRef') @@ -3307,6 +3860,90 @@ describe('agent memory improvement', () => { ).rejects.toThrow('does not match its persisted inputs or implementationRef') }) + it('recovers when the live config changes before the activation event is persisted', async () => { + type Config = { visibility: 'private' | 'team' } + const storage = inMemoryCampaignStorage() + const append = storage.append!.bind(storage) + let rejectActivatedEvent = true + storage.append = (path, value, expectedBytes) => { + if ( + rejectActivatedEvent && + path.includes('/activations/') && + value.includes('"status":"activated"') + ) { + rejectActivatedEvent = false + throw new Error('activation journal unavailable') + } + return append(path, value, expectedBytes) + } + let activeConfig: Config = { visibility: 'private' } + let compareAndSetCalls = 0 + const options: RunAgentMemoryImprovementOptions = { + experimentId: 'recover-memory-activation', + trainSequences: [ + improvementSequence('activation-train-a', 'train'), + improvementSequence('activation-train-b', 'train'), + ], + holdoutSequences: [ + improvementSequence('activation-holdout-a', 'holdout'), + improvementSequence('activation-holdout-b', 'holdout'), + ], + seeds: [ + { + config: { visibility: 'private' }, + track: 'baseline', + proposer: 'seed', + }, + ], + proposer: { + kind: 'team-sharing-proposer', + async propose() { + return [ + { + surface: JSON.stringify({ visibility: 'team' }), + label: 'share with team', + rationale: 'the second agent needs the accepted fact', + }, + ] + }, + }, + improvementRef: 'recover-memory-activation:v1', + budget: { maxSteps: 1 }, + runDir: '/runs/recover-memory-activation', + storage, + significance: { minProductiveRuns: 2, resamples: 200, seed: 11 }, + createCandidate: ({ config, candidateId }) => ({ + ref: `visibility:${config.visibility}:v1`, + policy: { read: [config.visibility], write: config.visibility }, + createAdapter: ({ branchId }) => createScopedTestAdapter(`${candidateId}:${branchId}`), + }), + activation: { + ref: 'memory-policy/live:v1', + async readCurrent() { + return structuredClone(activeConfig) + }, + async compareAndSet({ expectedConfig, config }) { + expect(activeConfig).toEqual(expectedConfig) + compareAndSetCalls += 1 + activeConfig = structuredClone(config) + }, + }, + } + + await expect(runAgentMemoryImprovement(options)).rejects.toThrow( + 'activation journal unavailable', + ) + expect(activeConfig).toEqual({ visibility: 'team' }) + expect(compareAndSetCalls).toBe(1) + + const recovered = await runAgentMemoryImprovement(options) + expect(recovered.activation.status).toBe('recovered') + expect(compareAndSetCalls).toBe(1) + const resumed = await runAgentMemoryImprovement(options) + expect(resumed.activation.status).toBe('already-activated') + expect(compareAndSetCalls).toBe(1) + }) + it('routes independent tracks to their named proposers with track context', async () => { type Config = { visibility: 'private' | 'team' } const trackContexts: Array<{ @@ -3488,6 +4125,99 @@ describe('agent memory improvement', () => { expect(proposerExecuted).toBe(false) }) + it('charges an interrupted proposer reservation before resuming the search', async () => { + type Config = { visibility: 'private' | 'team' } + const storage = inMemoryCampaignStorage() + const runDir = '/runs/interrupted-proposer-recovery' + const append = storage.append!.bind(storage) + let failFirstProposerReceipt = true + storage.append = (path, value, expectedBytes) => { + if ( + failFirstProposerReceipt && + path.endsWith('/cost-ledger.jsonl') && + value.includes('"status":"settled"') && + value.includes('memory-config-proposer') + ) { + failFirstProposerReceipt = false + throw new Error('simulated process exit before proposer receipt') + } + return append(path, value, expectedBytes) + } + let proposerCalls = 0 + const options: RunAgentMemoryImprovementOptions = { + experimentId: 'interrupted-proposer-recovery', + trainSequences: [improvementSequence('proposer-train', 'train')], + holdoutSequences: [improvementSequence('proposer-holdout', 'holdout')], + seeds: [ + { + config: { visibility: 'private' }, + track: 'baseline', + proposer: 'costed', + }, + ], + proposer: { + kind: 'costed', + async propose(context) { + proposerCalls += 1 + if (!context.costLedger) throw new Error('missing run cost ledger') + const paid = await context.costLedger.runPaidCall({ + actor: 'memory-config-proposer', + channel: 'agent', + phase: context.costPhase, + model: 'fixture-model', + maximumCharge: { externallyEnforcedMaximumUsd: 0.1 }, + execute: async () => JSON.stringify({ visibility: 'team' }), + receipt: () => ({ + model: 'fixture-model', + inputTokens: 0, + outputTokens: 0, + actualCostUsd: 0.1, + }), + }) + if (!paid.succeeded) throw paid.error + return [paid.value] + }, + }, + improvementRef: 'interrupted-proposer-recovery/v1', + budget: { maxSteps: 1 }, + maxTotalCostUsd: 0.2, + runDir, + storage, + createCandidate: ({ config, candidateId }) => ({ + ref: `visibility:${config.visibility}:v1`, + policy: { read: [config.visibility], write: config.visibility }, + createAdapter: ({ branchId }) => createScopedTestAdapter(`${candidateId}:${branchId}`), + }), + } + + await expect(runAgentMemoryImprovement(options)).rejects.toThrow('failed to persist') + const interruptedLedger = createRunCostLedger({ + storage, + runDir, + costCeilingUsd: 0.2, + }) + expect(interruptedLedger.listPending()).toEqual([ + expect.objectContaining({ actor: 'memory-config-proposer', state: 'interrupted' }), + ]) + + const result = await runAgentMemoryImprovement(options) + const resumedLedger = createRunCostLedger({ storage, runDir, costCeilingUsd: 0.2 }) + + expect(proposerCalls).toBe(2) + expect(result.totalCostUsd).toBe(0.2) + expect(resumedLedger.summary()).toMatchObject({ + totalCalls: 2, + unresolvedCalls: 0, + totalCostUsd: 0.2, + accountingComplete: true, + }) + expect(resumedLedger.list()[0]).toMatchObject({ + actor: 'memory-config-proposer', + costUsd: 0.1, + error: expect.stringContaining('charged the reserved maximum'), + }) + }) + it('rejects train and holdout histories with the same id', async () => { const sequence = improvementSequence('duplicate', 'train') await expect( From c3cc7b4e467e1d0aaa525ae4917b5a9746220baf Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Sat, 18 Jul 2026 00:45:36 -0600 Subject: [PATCH 4/4] fix(memory): close provider isolation and recovery gaps --- CHANGELOG.md | 19 + README.md | 84 ++-- package.json | 6 +- pnpm-lock.yaml | 153 +++---- src/benchmarks/index.ts | 522 +++++++++++++++++++----- src/memory/attempt-log.ts | 107 +++++ src/memory/experiment.ts | 286 ++++++++++---- src/memory/improvement.ts | 8 +- src/memory/index.ts | 11 +- src/memory/lifecycle.ts | 66 +++- src/memory/mem0.ts | 349 ++++++++-------- src/memory/neo4j.ts | 80 +++- src/memory/rank.ts | 2 +- tests/benchmarks.test.ts | 427 +++++++++++++++++++- tests/memory-adapter.test.ts | 66 +++- tests/memory-holdout.test.ts | 1 + tests/memory-systems.test.ts | 747 +++++++++++++++++++++++++++++------ 17 files changed, 2309 insertions(+), 625 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 188041f..ec74c2f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,7 +7,9 @@ - `runAgentMemoryImprovement()` now activates a measured winner through `activation.readCurrent()` and atomic `activation.compareAndSet()` instead of `onPromote`. - `AgentMemoryActivation.receiptPath` is now `journalPath` because the file is an append-only activation record. - `mem0MemoryAdapterIdentity()` and `graphitiMemoryAdapterIdentity()` require a stable, non-secret `backendRef` so two deployments cannot share a cache identity. +- Mem0 hosted mode now follows the synchronous array response from `mem0ai` 3.x; the unsupported queued-event options were removed. - Paid benchmark and improvement work now defaults to a zero dollar limit and requires an explicit `costCeiling` or `maxTotalCostUsd`. +- In-flight run directories from releases before 4.0 are not migrated; archive or clear them before upgrading because 4.0 rejects older attempt records. ### Added @@ -15,4 +17,21 @@ - Isolated memory branches with snapshots, replayable forks, private, team, and shared visibility, and ordered writes per agent. - Parallel multi-track memory experiments and configuration search on `agent-eval`, with fresh-history comparison before activation. - Durable controller ownership, interrupted-attempt cleanup, retired-candidate recovery, bounded cleanup work, and conservative recovery cost reconciliation. +- Complete candidate cost attribution across interrupted retries, plus explicit unranked recovery spend for retired benchmark candidates. - Exact scoped Mem0 deletion with list and search convergence checks. + +### Fixed + +- Mem0 and direct Neo4j operations reject provider scopes they cannot enforce before any provider call. +- Hosted Mem0 `appId` is an additional filter and cannot authorize an unscoped whole-application read or delete. +- Mem0 cleanup tracks fresh writes until they become visible and confirms deletion from both list and search indexes. +- Broader Mem0 cleanup scopes wait for delayed writes created under matching narrower scopes. +- Mem0 pending-write probes expire after the configured visibility window instead of accumulating for the adapter lifetime. +- Direct Neo4j reasoning writes reject combined session and run scopes because the SDK can enforce only one conversation identifier. +- Timed-out provider work blocks close or reuse of the same adapter until the original operation settles. +- Recovery retries are reserved durably before provider work and stop after three failures per attempt by default. +- Direct memory benchmarks account for billable adapter provisioning and reconnects in the shared dollar limit. +- Fully cached benchmark resumes skip adapter creation and add no provider charge. +- Execute and recovery adapter factories receive abort signals and are bounded by the configured timeout. +- Adapters returned after a timed-out factory call are closed, and experiment adapters also run their configured disposal callback. +- Reported dollar totals are normalized to twelve decimal places instead of exposing binary floating-point artifacts. diff --git a/README.md b/README.md index b7e5564..bbdd55e 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ This package turns raw sources and generated markdown knowledge into a versionab - [Common uses](#common-uses): wiki, RAG, memory, new KBs, existing KBs, and parallel agents - [CLI](#cli): `init` → `source-add` → `index` → `search` → `lint` - [Design](#design): the invariants, including immutable sources, cited claims, and deterministic graph output -- [Benchmark harness](#benchmark-harness): BEIR/MTEB/qrels, RAG answer, hallucination, KB-improvement cases +- [Benchmark runner](#benchmark-runner): BEIR/MTEB/qrels, RAG answer, hallucination, KB-improvement cases - [Agent-Eval integration](#agent-eval-integration): retrieval eval, readiness bundles, and release reports - [Memory adapters](#memory-adapters): Mem0, Graphiti, Neo4j, branching, and automatic improvement - [Research loop](#research-loop): `runKnowledgeResearchLoop` plus the control-loop adapter @@ -39,8 +39,8 @@ Two ways in, depending on what you're doing: - *"Expose the lower-level RAG lifecycle phases"* → `runRagKnowledgeImprovementLoop` in the [Agent-Eval integration](#agent-eval-integration) section. It exposes retrieval tuning, gap diagnosis, knowledge acquisition/update, answer-quality checks, and promotion as one typed lifecycle. - *"Evaluate RAG answers or a wiki/KB"* → `ragAnswerQualityJudge`, `createRagAnswerQualityHook`, and `scoreKnowledgeBaseIndex` in the [Agent-Eval integration](#agent-eval-integration) section. - - *"Run standard RAG/KB benchmarks"* → `runKnowledgeBenchmarkSuite` in the [Benchmark harness](#benchmark-harness) section. - - *"Does this candidate KB actually improve task success?"* → run an [agent-eval improvement loop](#agent-eval-integration) over KB variants, then `knowledgeReleaseReport` for the promotion decision. + - *"Run standard RAG/KB benchmarks"* → `runKnowledgeBenchmarkSuite` in the [Benchmark runner](#benchmark-runner) section. + - *"Does this candidate KB improve task success?"* → run an [agent-eval improvement loop](#agent-eval-integration) over KB variants, then `knowledgeReleaseReport` for the promotion decision. - *"Keep live authorities fresh"* → [pluggable sources](#pluggable-knowledge-sources) + `detectChanges` → eval re-runs. Storage stays consumer-owned via `KbStore` (`MemoryKbStore`, `FileSystemKbStore`, or your own D1/Postgres). Every primitive below is source-grounded: claims cite immutable source records, and lint fails on un-grounded citations. @@ -233,14 +233,22 @@ console.log(result.report.score.mean) `runMemoryAdapterBenchmark()` is for adapters that enforce every supplied scope and support exact scoped deletion. Every cell records its fresh provider namespace before the first write and deletes it after measurement, so concurrent repetitions and retries cannot share memory. On restart, unfinished scopes are deleted before any new benchmark cell runs. -The candidate factory receives `purpose: 'execute' | 'recovery'`; both paths must reconnect to the same provider and adapter identity. -The execute factory only constructs the adapter; normal provider work belongs in adapter methods so it runs inside per-case cost accounting. +The candidate factory receives `purpose: 'execute' | 'recovery'`, `signal`, and `markExternalCall()`; both paths must reconnect to the same provider and adapter identity. +Pass `signal` to provider connection and provisioning calls so timeout cancellation reaches the SDK. +Set `adapterId` when the returned adapter ID differs from the candidate ID. +The declared ID lets a fully cached resume skip adapter creation entirely; a returned mismatch fails before any case runs. +Keep local construction free. +When construction provisions or reconnects billable provider state, set `adapterCreationCostUsd` and call `markExternalCall()` immediately before the external action. +Normal read and write work belongs in adapter methods so it runs inside per-case cost accounting. Use `runAgentMemoryExperiment()` below when an agent profile needs ordered multi-step histories, branch snapshots, runtime callbacks, or automatic configuration improvement. Set `cleanupTimeoutMs` to bound each provider cleanup and close operation; the default is 180 seconds. -Set `costUsdPerCase` for normal provider work and `recoveryCostUsdPerAttempt` when reconnecting and deleting an interrupted case can add provider charges. +Set `costUsdPerCase` for normal provider work, `adapterCreationCostUsd` for billable construction or reconnects, and `recoveryCostUsdPerAttempt` when deleting an interrupted case can add another provider charge. One `costCeiling` covers every candidate and recovery call in the comparison, and `totalCostUsd` reports the complete comparison cost. +`unrankedRecoveryCostUsd` identifies cleanup spend from retired candidates that cannot appear in ranking rows. The default `costCeiling` is zero, so paid work is refused until the caller sets a limit. Keep removed implementations in `recoveryCandidates` until their unfinished scopes are gone, and use `maxRecoveryAttempts` to reject an unexpectedly large backlog before provider work starts. +Each failed recovery is recorded before its provider call. +`maxRecoveryRetriesPerAttempt` defaults to three, and `recoveryLogPath` identifies the durable retry history returned by the run. The default filesystem storage uses an OS lock; custom storage requires `controllerMode: 'process-local'` or a distributed `acquireRunLease` implementation. Billable responders must execute paid work through `context.cost.runPaidCall()`; `costUsd` on an artifact is display-only. @@ -379,7 +387,7 @@ That keeps the public API from reporting a fake “RAG improved” result when t Use retrieval eval when the question is whether a retrieval/RAG config can find the right knowledge before an agent reasons over it. The labels should name stable pages, source records, anchors, or source spans, not ephemeral chunk IDs. -The completion roadmap is in [`docs/eval/rag-eval-roadmap.md`](docs/eval/rag-eval-roadmap.md). +Benchmark coverage and completion criteria are documented in [`docs/eval/rag-eval-roadmap.md`](docs/eval/rag-eval-roadmap.md). ```ts import { runRetrievalImprovementLoop } from '@tangle-network/agent-knowledge' @@ -407,7 +415,7 @@ Pass a custom `retrieve` function to `buildRetrievalEvalDispatch` when the confi The built-in fallback uses `searchKnowledge` over the local deterministic index. Use `buildRetrievalEvalDispatch`, `retrievalRecallJudge`, and `retrievalParameterSweepProposer` directly only when you need custom `agent-eval` wiring. -To answer whether a candidate knowledge base actually improves agent task success, run an `@tangle-network/agent-eval` improvement loop (`runImprovementLoop`) over your KB variants on a real task corpus; each run is scored into a `RunRecord`. +To answer whether a candidate knowledge base improves agent task success, run an `@tangle-network/agent-eval` improvement loop (`runImprovementLoop`) over your KB variants on a real task corpus; each run is scored into a `RunRecord`. Use `knowledgeReleaseReport()` before promotion: pass the candidate and baseline `RunRecord[]` (plus optional `ReleaseTraceEvidence` and the gate decision) and it folds them into a `ReleaseConfidenceScorecard` and a `KnowledgeRelease` using `agent-eval`'s release gates and `RunRecord` validation. @@ -456,22 +464,10 @@ import { createNeo4jAgentMemoryAdapter, } from '@tangle-network/agent-knowledge/memory' -const mem0ApiKey = process.env.MEM0_API_KEY -if (!mem0ApiKey) throw new Error('MEM0_API_KEY is required') - const mem0Hosted = createMem0MemoryAdapter({ client: mem0Client, mode: 'hosted', backendRef: 'mem0:production-account', - consistency: 'visible', - async getEvent(eventId) { - const response = await fetch( - `https://api.mem0.ai/v1/event/${encodeURIComponent(eventId)}/`, - { headers: { Authorization: `Token ${mem0ApiKey}` } }, - ) - if (!response.ok) throw new Error(`Mem0 event request failed: ${response.status}`) - return response.json() - }, }) const mem0Oss = createMem0MemoryAdapter({ @@ -496,9 +492,11 @@ The Graphiti defaults match the current official server tools: `add_memory`, `se Set `toolNames` explicitly when your server lists different names. The adapter never retries a write under another name because that could enqueue it twice. Visible writes expand the episode scan when Graphiti truncates a long group and fail explicitly at `episodeScanLimit` instead of reporting a false success. -Hosted Mem0 also defaults to visible writes and polls the official event endpoint until the add succeeds or fails. -The current JavaScript SDK does not expose that endpoint, so hosted mode accepts the small `getEvent` callback shown above. +Hosted Mem0 follows the synchronous memory-array response from `mem0ai` 3.x and rejects legacy asynchronous event responses instead of treating them as successful writes. Mem0 cleanup requires `getAll` plus per-memory `delete`, refuses an empty identity scope, and waits until both list and search results stop returning deleted IDs. +Every Mem0 operation requires `userId`, `agentId`, `runId`, or `namespace`. +`appId`, tenant, team, session, and tag filters refine that identity but cannot replace it. +Fresh-write probes expire after `ingestionTimeoutMs`, so a long-lived adapter retains only its current visibility window rather than its lifetime write history. The adapter does not call Mem0's account-wide `deleteAll` method. Use a stable, non-secret `backendRef` in Mem0 and Graphiti adapter identities so caches and dispatch keys cannot alias two accounts or clusters. @@ -513,9 +511,11 @@ Each adapter uses the same `search`, `getContext`, and `write` method shape, but Neo4j bridge facts are graph writes. The official `@neo4j-labs/agent-memory` 0.4 API has no corresponding fact-search method, so fact-only retrieval experiments should use another provider or a custom Neo4j query adapter. -Mem0 and Graphiti apply tenant, user, agent, team, run, session, namespace, and tag scopes per request. +Mem0 and Graphiti apply tenant, user, agent, team, run, session, namespace, and tag filters per request, subject to Mem0's required provider identity above. Neo4j Agent Memory experiments require disposable external state per branch because the SDK cannot clear every memory kind atomically. -Pass `branchId` only after the caller has provisioned that isolation. +On a shared Neo4j client, the adapter rejects scope fields the selected SDK method cannot enforce before making a provider call. +Conversation messages can use `sessionId`, and reasoning writes can use either `sessionId` or `runId`; other direct operations are unscoped unless a dedicated client is used. +Pass `branchId` only after the caller has provisioned one physically isolated instance for that exact branch. Provider-specific identifiers remain internal to the adapter. Custom adapters must declare `branchIsolation: { mode: 'scoped' }` only when every operation enforces the supplied scope; undeclared adapters are rejected by branch execution. @@ -526,13 +526,16 @@ const neo4jCandidate = { id: 'neo4j', ref: 'neo4j-agent-memory:0.4.0:rest', policy: { read: ['shared'], write: 'shared' }, - async createAdapter({ branchId, purpose }: { + async createAdapter({ branchId, purpose, signal, markExternalCall }: { branchId: string purpose: 'execute' | 'recovery' + signal: AbortSignal + markExternalCall(): void }) { + markExternalCall() const client = purpose === 'recovery' - ? await openDedicatedNeo4jMemory({ branchId, transport: 'rest' }) - : await provisionDedicatedNeo4jMemory({ branchId, transport: 'rest' }) + ? await openDedicatedNeo4jMemory({ branchId, transport: 'rest', signal }) + : await provisionDedicatedNeo4jMemory({ branchId, transport: 'rest', signal }) if (!client) return null const adapter = createNeo4jAgentMemoryAdapter({ client, transport: 'rest', branchId }) neo4jInstances.set(adapter, branchId) @@ -659,11 +662,14 @@ const result = await runAgentMemoryImprovement({ Every experiment candidate must set `ref` to a version or commit that changes whenever its provider configuration or adapter behavior changes. The first seed is always the deployed baseline. Each seed starts an independent track with its own objective and optional proposer, and a branch inherits its parent track's proposer unless the governor chooses another one. -Candidate factories must be side-effect free. -Set each candidate's `externalCostUsdPerSequence` to a conservative memory-provider charge; it is counted once an adapter method attempts provider work, while a factory failure costs zero. +Candidate factories should perform local construction only when possible. +When a dedicated instance must be provisioned or reconnected, call `markExternalCall()` immediately before that action. +Set each candidate's `externalCostUsdPerSequence` to a conservative memory-provider charge; it is counted once an adapter method or a marked factory action attempts provider work, while a local factory failure costs zero. Set `externalRecoveryCostUsdPerAttempt` when reconnecting and deleting an interrupted history adds a separate provider charge. Recovery uses the same durable run-wide cost account as candidate execution, proposers, profile steps, and the governor, so a resumed run cannot exceed `maxTotalCostUsd` by treating cleanup as free work. -If a process exits after paid cleanup but before recording its receipt, resume charges the reserved maximum, repeats the idempotent cleanup, and continues only after the account is complete. +`runAgentMemoryExperiment()` reports all attempt and recovery spend in `totalCostUsd`, with retired-candidate cleanup separated as `unrankedRecoveryCostUsd`. +Cleanup completion is journaled before a paid-call receipt. +If a process exits between those writes, resume charges the reserved maximum without repeating provider cleanup and continues only after the account is complete. The default `maxTotalCostUsd` is zero, so model and paid provider calls require an explicit limit. Model calls inside `executeStep` should use `context.cost.runPaidCall()` so `maxTotalCostUsd` counts both sources before starting more work. The same budget is supplied to named proposers as `context.costLedger` and to the governor as `context.costLedger`; the standard `agent-eval` proposers charge their model calls there. @@ -687,6 +693,9 @@ The runner appends a `started` event before adapter creation and a `cleaned` eve On resume it recovers every unfinished attempt before starting new cells. Independent unfinished attempts recover up to `maxConcurrency` at once, and no new cell starts until every cleanup succeeds. Recovery refuses more than `maxRecoveryAttempts`, which defaults to 1000. +Recovery also refuses a fourth failed cleanup for the same attempt by default. +Set `maxRecoveryRetriesPerAttempt` to change that limit; its durable retry record is returned as `recoveryLogPath`. +`runAgentMemoryImprovement()` forwards the same option to every training and fresh-history experiment. Pass retired implementations through `recoveryCandidates` until their recorded attempts are cleaned. `createAdapter` receives `purpose: 'recovery'` during that pass so a dedicated-instance candidate can reconnect to the existing branch instead of provisioning another one. It may return `null` during recovery when the recorded attempt never created external state. @@ -695,6 +704,7 @@ That visibility wait is bounded by `cleanupTimeoutMs`; a longer required delay f The default filesystem storage survives process restarts and uses an OS lock. Custom durable storage must implement atomic `append`; process-local custom storage opts in with `controllerMode: 'process-local'`, and distributed controllers provide `acquireRunLease`. Provider SDK calls should also have their own network timeout because a caller-side cleanup timeout cannot cancel an arbitrary third-party promise. +Candidate factories receive an `AbortSignal` and must pass it into provider connection or provisioning calls. The persisted run identity includes `budget.maxSteps`; start a new `runDir` to enlarge a completed search instead of mutating its history. Promotion is blocked when a configured critical dimension is missing or incomplete on either fresh-history arm. Improvement runs clear each scoped branch before and after a measured history by default, including timeout and thrown-error cleanup. @@ -707,13 +717,13 @@ The `executeStep` callback is where a runtime profile, coding agent, research ag This keeps provider and measurement code reusable while the profile owns what agents do and what they choose to remember. The Neo4j adapter accepts the real `@neo4j-labs/agent-memory` client and rejects unsafe branch reuse unless `branchId` identifies caller-provisioned isolated state. -The Mem0 adapter typechecks against `mem0ai` hosted and open-source clients and waits for the hosted event result by default. +The Mem0 adapter typechecks against `mem0ai` hosted and open-source clients and accepts only the current hosted SDK response shape. The Graphiti adapter calls the official MCP tools and waits for queued episodes to become visible before evaluating them by default. Hosted Mem0 and Graphiti visible writes are safe only in `lifetime: 'attempt'` branches because their writes may outlive a worker process. `runAgentMemoryExperiment()` creates those fresh attempt branches automatically. Attempt snapshots are audit and fork inputs, not restart points; use `forkAgentMemoryBranchSnapshot()` to replay one into a fresh branch. Mem0 OSS and caller-provisioned disposable instances can use resumable branches. -For hosted Mem0 and Graphiti, `consistency: 'queued'` is available only for direct ingestion and is rejected by branch execution. +Graphiti `consistency: 'queued'` is available only for direct ingestion and is rejected by branch execution. ## Research Loop @@ -805,11 +815,11 @@ with its own pass (`driverResearches: true`), and gates on the readiness check. Both are yours (no creds). The loop owns the deterministic mechanics (indexing, applying write blocks, scoring readiness) and stops once no blocking gap remains. -Does the verifying driver actually earn its keep? See -[docs/two-agent-research-ab.md](docs/two-agent-research-ab.md) for an equal-compute -A/B (9 ML topics, `glm-5.2`): the two-agent loop admits ~2.33 fewer sources per topic -at identical coverage, though most of that win is de-duplication, not relevance -filtering. Honest caveats and how to reproduce included. +An equal-compute comparison across 9 ML topics using `glm-5.2` is documented in +[docs/two-agent-research-ab.md](docs/two-agent-research-ab.md). +The two-agent loop admitted about 2.33 fewer sources per topic at identical coverage. +Most of that difference came from de-duplication rather than relevance filtering. +The document includes limitations and reproduction steps. ```ts import { diff --git a/package.json b/package.json index e58b7f7..548577d 100644 --- a/package.json +++ b/package.json @@ -73,17 +73,17 @@ "@tangle-network/agent-eval": "^0.122.8", "@tangle-network/agent-interface": "^0.31.0", "proper-lockfile": "4.1.2", - "zod": "^4.3.6" + "zod": "^4.4.3" }, "devDependencies": { - "@biomejs/biome": "^2.4.15", + "@biomejs/biome": "^2.5.4", "@neo4j-labs/agent-memory": "0.4.0", "@tangle-network/sandbox": "^0.9.7", "@types/node": "^25.6.0", "@types/proper-lockfile": "4.1.4", "mem0ai": "3.1.0", "tsup": "^8.0.0", - "tsx": "^4.22.4", + "tsx": "^4.23.1", "typescript": "^5.7.0", "vitest": "^3.0.0" }, diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index b9f2f2d..78194ea 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -22,18 +22,18 @@ importers: specifier: 4.1.2 version: 4.1.2 zod: - specifier: ^4.3.6 - version: 4.4.2 + specifier: ^4.4.3 + version: 4.4.3 devDependencies: '@biomejs/biome': - specifier: ^2.4.15 - version: 2.4.15 + specifier: ^2.5.4 + version: 2.5.4 '@neo4j-labs/agent-memory': specifier: 0.4.0 version: 0.4.0 '@tangle-network/sandbox': specifier: ^0.9.7 - version: 0.9.7(viem@2.48.8(typescript@5.9.3)(zod@4.4.2)) + version: 0.9.7(viem@2.48.8(typescript@5.9.3)(zod@4.4.3)) '@types/node': specifier: ^25.6.0 version: 25.6.0 @@ -45,16 +45,16 @@ importers: version: 3.1.0(@anthropic-ai/sdk@0.40.1)(@azure/identity@4.13.1)(@azure/search-documents@12.2.0)(@cloudflare/workers-types@4.20260702.1)(@google/genai@1.52.0)(@langchain/core@1.2.3(@opentelemetry/api@1.9.1)(ws@8.21.0))(@mistralai/mistralai@1.15.1)(@qdrant/js-client-rest@1.18.0(typescript@5.9.3))(@supabase/supabase-js@2.110.7)(@types/jest@29.5.14)(@types/pg@8.11.0)(better-sqlite3@12.11.1)(cloudflare@4.5.0)(compromise@14.16.0)(groq-sdk@0.3.0)(mongodb@7.2.0)(natural@8.1.1(@opentelemetry/api@1.9.1))(ollama@0.5.18)(pg@8.22.0)(redis@5.12.1(@opentelemetry/api@1.9.1))(ws@8.21.0) tsup: specifier: ^8.0.0 - version: 8.5.1(postcss@8.5.13)(tsx@4.22.4)(typescript@5.9.3)(yaml@2.8.4) + version: 8.5.1(postcss@8.5.13)(tsx@4.23.1)(typescript@5.9.3)(yaml@2.8.4) tsx: - specifier: ^4.22.4 - version: 4.22.4 + specifier: ^4.23.1 + version: 4.23.1 typescript: specifier: ^5.7.0 version: 5.9.3 vitest: specifier: ^3.0.0 - version: 3.2.4(@types/node@25.6.0)(tsx@4.22.4)(yaml@2.8.4) + version: 3.2.4(@types/node@25.6.0)(tsx@4.23.1)(yaml@2.8.4) packages: @@ -145,55 +145,55 @@ packages: resolution: {integrity: sha512-qehxGkRj55h/ff8EMaJ+cYhyaKlHIxqYDn682wQD7RNp9UujOQsHog2uS0r2vzr4pW+sXf90NeeayjcNaX3fFg==} engines: {node: '>=6.9.0'} - '@biomejs/biome@2.4.15': - resolution: {integrity: sha512-j5VH3a/h/HXTKBM50MDMxRCzkeLv9S2XJcW2WgnZT1+xyisi+0bISrXR82gCX+8S9lvK0skEvHJRN+3Ktr2hlw==} + '@biomejs/biome@2.5.4': + resolution: {integrity: sha512-xy5FNE5kQJKyK5MR1gJy6ztXYx4WBAbYGlK04lMEgmyPRWKybY9NFwiG9yo0XdzOU8Xvhj41u034J1ywfoWfMw==} engines: {node: '>=14.21.3'} hasBin: true - '@biomejs/cli-darwin-arm64@2.4.15': - resolution: {integrity: sha512-rF3PPqLq1yoST79zaQbDjVJwsuIeci/O+9bgNmC5QpgOqz6aqYuzA4abyAGx+mgyiDXn4A049xAN8gijbuR1Qg==} + '@biomejs/cli-darwin-arm64@2.5.4': + resolution: {integrity: sha512-4o3NFRobXHynkgcFVrlZsoDAFtF2ldlEGN8sORSws5ZQqyY4PXnPUIylu4ksfyHuwkfvDREuWh3JK+niRwGq3w==} engines: {node: '>=14.21.3'} cpu: [arm64] os: [darwin] - '@biomejs/cli-darwin-x64@2.4.15': - resolution: {integrity: sha512-/5KHXYMfSJs1fNXiX30xFtI8JcCFV6zaVVLxOa0M2sfqBKHkpQhRTv94yxQWxeTY2lzo2OuTlNvPC+hDQt2wcQ==} + '@biomejs/cli-darwin-x64@2.5.4': + resolution: {integrity: sha512-D32P5HkU2Y6PySuC/WsVDTOgsDwVFmujzhhhOQjajtATpVWFDXuVd3oRbsWNSEA+aaFzyzZm22szsyydBYlSyQ==} engines: {node: '>=14.21.3'} cpu: [x64] os: [darwin] - '@biomejs/cli-linux-arm64-musl@2.4.15': - resolution: {integrity: sha512-ZPcxznxm0pogHBLZhYntyR3sR+MrZjqJIKEr7ZqVen0Rl+P/4upVmfYXjftizi9RoqZntg33fv/1fbdhbYXpEQ==} + '@biomejs/cli-linux-arm64-musl@2.5.4': + resolution: {integrity: sha512-Rpm5/AT1m+DlJmUoYvS4/vXc+0tXJPJ2NQz25TGPyHVF5JrWy75PE0GH6kVxsKtQDuCH4OgzquZq0R4kj/wCVg==} engines: {node: '>=14.21.3'} cpu: [arm64] os: [linux] - '@biomejs/cli-linux-arm64@2.4.15': - resolution: {integrity: sha512-owaAMZD/T4LrD0ELNCk0Km3qrRHuM0X6EAyVE1FSqGY0rbLoiDLrO4Us2tllm6cAeB2Ioa9C2C08NZPdr8+0Ug==} + '@biomejs/cli-linux-arm64@2.5.4': + resolution: {integrity: sha512-pSEfW7B8kTsXUjUxC1xVVK+y85Ht3C5XxZ9gclmC7/3Ku9Vqz8jmI7k0p/BNIjQ6t4sFERI2sFeH73ybiZl6YQ==} engines: {node: '>=14.21.3'} cpu: [arm64] os: [linux] - '@biomejs/cli-linux-x64-musl@2.4.15': - resolution: {integrity: sha512-CNq/9W38SYSH023lfcQ4KKU8K0YX8T//FZUhcgtMMRABDojx5XsMV7jlweAvGSl389wJQB29Qo6Zb/a+jdvt+w==} + '@biomejs/cli-linux-x64-musl@2.5.4': + resolution: {integrity: sha512-aby/PohmmgbShcHqFsZVzG8H6D98+P+A6xRWRrQcLW1pCjabcov5UUlke4UqNQBYTkDQav+jB4zyyDDeKB2GaA==} engines: {node: '>=14.21.3'} cpu: [x64] os: [linux] - '@biomejs/cli-linux-x64@2.4.15': - resolution: {integrity: sha512-0jj7THz12GbUOLmMibktK6DZjqz2zV64KFxyBtcFTKPiiOIY0a7vns1elpO1dERvxpsZ5ik0oFfz0oGwFde1+g==} + '@biomejs/cli-linux-x64@2.5.4': + resolution: {integrity: sha512-FNxojWJkL7EajAuzBgoLe0T2G0y112M4lBrDIFl/DomFTx8yqenYOIdsRLNXvOvBBofE8hJi85LjzLmBDpY7/Q==} engines: {node: '>=14.21.3'} cpu: [x64] os: [linux] - '@biomejs/cli-win32-arm64@2.4.15': - resolution: {integrity: sha512-ouhkYdlhp/1GghEJPdWwD/Vi3gQ1nFxuSpMolWsbq3Lsq3QUR4jl6UdhhscdCugKU5vOEuMiJhvKj66O0OCq+w==} + '@biomejs/cli-win32-arm64@2.5.4': + resolution: {integrity: sha512-emoXexPZIPAZkz2RKmA95WJUqK3I5MJNYtwEbL5ESciRzhmFMMyekDhNG8hpeOaK+ZGRDxAU4wvGuA5IHQ0h0w==} engines: {node: '>=14.21.3'} cpu: [arm64] os: [win32] - '@biomejs/cli-win32-x64@2.4.15': - resolution: {integrity: sha512-zBrGq5mx5wwpnow4+2BxUvleDM+GNd4sLbPaMapsSLQLD0NGRCquqPBTgN+7XkUteHvj7M+BstuI8tmnV7+HgQ==} + '@biomejs/cli-win32-x64@2.5.4': + resolution: {integrity: sha512-U1jaluLw1qQc2Tx7/CeSoL9N5XcqIH+GWjpUAy1ouB5nVjSCMNO+NNHdY3RAs8zxNurLWAdj6pehQdCA2zyU+Q==} engines: {node: '>=14.21.3'} cpu: [x64] os: [win32] @@ -925,6 +925,9 @@ packages: '@types/node@25.6.0': resolution: {integrity: sha512-+qIYRKdNYJwY3vRCZMdJbPLJAtGjQBudzZzdzwQYkEPQd+PJGixUL5QfvCLDaULoLv+RhT3LDkwEfKaAkgSmNQ==} + '@types/node@25.9.5': + resolution: {integrity: sha512-OScDchr2fwuUmWdf4kZ9h7PcJiYDVInhJizG/biAq3cAvqwYktuy/TYGGdZNMtNTFUP7rnb0NU4TUdm82kt4Rg==} + '@types/pg@8.11.0': resolution: {integrity: sha512-sDAlRiBNthGjNFfvt0k6mtotoVYVQ63pA8R4EMWka7crawSR60waVYR0HAgmPRs/e2YaeJTD/43OoZ3PFw80pw==} @@ -2211,8 +2214,8 @@ packages: typescript: optional: true - tsx@4.22.4: - resolution: {integrity: sha512-X8EX+XV4QR5xCsrgxaED954zTDfY8KqlDtskKEL0cHhyS/P8b4IFOvGDQpsC9Q1XnLq915wEfwwY/zzskCtmhg==} + tsx@4.23.1: + resolution: {integrity: sha512-GQHnkIfxyx1wYCOS/wonik5MVRZU9hi1TEZmzGZSCJB1y9YgoZ8H6itNE/u4suE+yLmOzuE4E5S4TZ/ZX2wcWQ==} engines: {node: '>=18.0.0'} hasBin: true @@ -2236,6 +2239,9 @@ packages: undici-types@7.19.2: resolution: {integrity: sha512-qYVnV5OEm2AW8cJMCpdV20CDyaN3g0AjDlOGf1OW4iaDEx8MwdtChUp4zu4H0VP3nDRF/8RKWH+IPp9uW0YGZg==} + undici-types@7.24.6: + resolution: {integrity: sha512-WRNW+sJgj5OBN4/0JpHFqtqzhpbnV0GuB+OozA9gCL7a993SmU+1JBZCzLNxYsbMfIeDL+lTsphD5jN5N+n0zg==} + undici@6.27.0: resolution: {integrity: sha512-YmfV3YnEDzXRC5lZ2jWtWWHKGUm1zIt8AhesR1tens+HTNv+YZlN/dp6G727LOvMJ8xjP9Be7Y2Sdr96LDm+pg==} engines: {node: '>=18.17'} @@ -2402,9 +2408,6 @@ packages: zod@3.25.76: resolution: {integrity: sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ==} - zod@4.4.2: - resolution: {integrity: sha512-IynmDyxsEsb9RKzO3J9+4SxXnl2FTFSzNBaKKaMV6tsSk0rw9gYw9gs+JFCq/qk2LCZ78KDwyj+Z289TijSkUw==} - zod@4.4.3: resolution: {integrity: sha512-ytENFjIJFl2UwYglde2jchW2Hwm4GJFLDiSXWdTrJQBIN9Fcyp7n4DhxJEiWNAJMV1/BqWfW/kkg71UDcHJyTQ==} @@ -2550,39 +2553,39 @@ snapshots: '@babel/helper-validator-identifier@7.29.7': {} - '@biomejs/biome@2.4.15': + '@biomejs/biome@2.5.4': optionalDependencies: - '@biomejs/cli-darwin-arm64': 2.4.15 - '@biomejs/cli-darwin-x64': 2.4.15 - '@biomejs/cli-linux-arm64': 2.4.15 - '@biomejs/cli-linux-arm64-musl': 2.4.15 - '@biomejs/cli-linux-x64': 2.4.15 - '@biomejs/cli-linux-x64-musl': 2.4.15 - '@biomejs/cli-win32-arm64': 2.4.15 - '@biomejs/cli-win32-x64': 2.4.15 - - '@biomejs/cli-darwin-arm64@2.4.15': + '@biomejs/cli-darwin-arm64': 2.5.4 + '@biomejs/cli-darwin-x64': 2.5.4 + '@biomejs/cli-linux-arm64': 2.5.4 + '@biomejs/cli-linux-arm64-musl': 2.5.4 + '@biomejs/cli-linux-x64': 2.5.4 + '@biomejs/cli-linux-x64-musl': 2.5.4 + '@biomejs/cli-win32-arm64': 2.5.4 + '@biomejs/cli-win32-x64': 2.5.4 + + '@biomejs/cli-darwin-arm64@2.5.4': optional: true - '@biomejs/cli-darwin-x64@2.4.15': + '@biomejs/cli-darwin-x64@2.5.4': optional: true - '@biomejs/cli-linux-arm64-musl@2.4.15': + '@biomejs/cli-linux-arm64-musl@2.5.4': optional: true - '@biomejs/cli-linux-arm64@2.4.15': + '@biomejs/cli-linux-arm64@2.5.4': optional: true - '@biomejs/cli-linux-x64-musl@2.4.15': + '@biomejs/cli-linux-x64-musl@2.5.4': optional: true - '@biomejs/cli-linux-x64@2.4.15': + '@biomejs/cli-linux-x64@2.5.4': optional: true - '@biomejs/cli-win32-arm64@2.4.15': + '@biomejs/cli-win32-arm64@2.5.4': optional: true - '@biomejs/cli-win32-x64@2.4.15': + '@biomejs/cli-win32-x64@2.5.4': optional: true '@cfworker/json-schema@4.1.1': {} @@ -2773,7 +2776,7 @@ snapshots: '@jest/schemas': 29.6.3 '@types/istanbul-lib-coverage': 2.0.6 '@types/istanbul-reports': 3.0.4 - '@types/node': 25.6.0 + '@types/node': 25.9.5 '@types/yargs': 17.0.35 chalk: 4.1.2 @@ -3072,7 +3075,7 @@ snapshots: '@noble/hashes': 1.8.0 zod: 4.4.3 - '@tangle-network/sandbox@0.9.7(viem@2.48.8(typescript@5.9.3)(zod@4.4.2))': + '@tangle-network/sandbox@0.9.7(viem@2.48.8(typescript@5.9.3)(zod@4.4.3))': dependencies: '@tangle-network/agent-core': 0.3.8 '@tangle-network/agent-interface': 0.13.0 @@ -3085,7 +3088,7 @@ snapshots: dependencies: '@scure/bip32': 2.2.0 '@scure/bip39': 2.2.0 - '@tangle-network/sandbox': 0.9.7(viem@2.48.8(typescript@5.9.3)(zod@4.4.2)) + '@tangle-network/sandbox': 0.9.7(viem@2.48.8(typescript@5.9.3)(zod@4.4.3)) '@tangle-network/tcloud-attestation': 0.1.1 commander: 14.0.3 viem: 2.48.8(typescript@5.9.3)(zod@4.4.3) @@ -3136,9 +3139,13 @@ snapshots: dependencies: undici-types: 7.19.2 + '@types/node@25.9.5': + dependencies: + undici-types: 7.24.6 + '@types/pg@8.11.0': dependencies: - '@types/node': 25.6.0 + '@types/node': 25.9.5 pg-protocol: 1.15.0 pg-types: 4.1.0 @@ -3180,13 +3187,13 @@ snapshots: chai: 5.3.3 tinyrainbow: 2.0.0 - '@vitest/mocker@3.2.4(vite@7.3.2(@types/node@25.6.0)(tsx@4.22.4)(yaml@2.8.4))': + '@vitest/mocker@3.2.4(vite@7.3.2(@types/node@25.6.0)(tsx@4.23.1)(yaml@2.8.4))': dependencies: '@vitest/spy': 3.2.4 estree-walker: 3.0.3 magic-string: 0.30.21 optionalDependencies: - vite: 7.3.2(@types/node@25.6.0)(tsx@4.22.4)(yaml@2.8.4) + vite: 7.3.2(@types/node@25.6.0)(tsx@4.23.1)(yaml@2.8.4) '@vitest/pretty-format@3.2.4': dependencies: @@ -3751,7 +3758,7 @@ snapshots: jest-util@29.7.0: dependencies: '@jest/types': 29.6.3 - '@types/node': 25.6.0 + '@types/node': 25.9.5 chalk: 4.1.2 ci-info: 3.9.0 graceful-fs: 4.2.11 @@ -4126,12 +4133,12 @@ snapshots: mlly: 1.8.2 pathe: 2.0.3 - postcss-load-config@6.0.1(postcss@8.5.13)(tsx@4.22.4)(yaml@2.8.4): + postcss-load-config@6.0.1(postcss@8.5.13)(tsx@4.23.1)(yaml@2.8.4): dependencies: lilconfig: 3.1.3 optionalDependencies: postcss: 8.5.13 - tsx: 4.22.4 + tsx: 4.23.1 yaml: 2.8.4 postcss@8.5.13: @@ -4200,7 +4207,7 @@ snapshots: '@protobufjs/path': 1.1.2 '@protobufjs/pool': 1.1.0 '@protobufjs/utf8': 1.1.2 - '@types/node': 25.6.0 + '@types/node': 25.9.5 long: 5.3.2 proxy-from-env@2.1.0: {} @@ -4403,7 +4410,7 @@ snapshots: tslib@2.8.1: {} - tsup@8.5.1(postcss@8.5.13)(tsx@4.22.4)(typescript@5.9.3)(yaml@2.8.4): + tsup@8.5.1(postcss@8.5.13)(tsx@4.23.1)(typescript@5.9.3)(yaml@2.8.4): dependencies: bundle-require: 5.1.0(esbuild@0.27.7) cac: 6.7.14 @@ -4414,7 +4421,7 @@ snapshots: fix-dts-default-cjs-exports: 1.0.1 joycon: 3.1.1 picocolors: 1.1.1 - postcss-load-config: 6.0.1(postcss@8.5.13)(tsx@4.22.4)(yaml@2.8.4) + postcss-load-config: 6.0.1(postcss@8.5.13)(tsx@4.23.1)(yaml@2.8.4) resolve-from: 5.0.0 rollup: 4.60.2 source-map: 0.7.6 @@ -4431,7 +4438,7 @@ snapshots: - tsx - yaml - tsx@4.22.4: + tsx@4.23.1: dependencies: esbuild: 0.28.1 optionalDependencies: @@ -4451,6 +4458,8 @@ snapshots: undici-types@7.19.2: {} + undici-types@7.24.6: {} + undici@6.27.0: {} util-deprecate@1.0.2: {} @@ -4476,13 +4485,13 @@ snapshots: - utf-8-validate - zod - vite-node@3.2.4(@types/node@25.6.0)(tsx@4.22.4)(yaml@2.8.4): + vite-node@3.2.4(@types/node@25.6.0)(tsx@4.23.1)(yaml@2.8.4): dependencies: cac: 6.7.14 debug: 4.4.3 es-module-lexer: 1.7.0 pathe: 2.0.3 - vite: 7.3.2(@types/node@25.6.0)(tsx@4.22.4)(yaml@2.8.4) + vite: 7.3.2(@types/node@25.6.0)(tsx@4.23.1)(yaml@2.8.4) transitivePeerDependencies: - '@types/node' - jiti @@ -4497,7 +4506,7 @@ snapshots: - tsx - yaml - vite@7.3.2(@types/node@25.6.0)(tsx@4.22.4)(yaml@2.8.4): + vite@7.3.2(@types/node@25.6.0)(tsx@4.23.1)(yaml@2.8.4): dependencies: esbuild: 0.27.7 fdir: 6.5.0(picomatch@4.0.4) @@ -4508,14 +4517,14 @@ snapshots: optionalDependencies: '@types/node': 25.6.0 fsevents: 2.3.3 - tsx: 4.22.4 + tsx: 4.23.1 yaml: 2.8.4 - vitest@3.2.4(@types/node@25.6.0)(tsx@4.22.4)(yaml@2.8.4): + vitest@3.2.4(@types/node@25.6.0)(tsx@4.23.1)(yaml@2.8.4): dependencies: '@types/chai': 5.2.3 '@vitest/expect': 3.2.4 - '@vitest/mocker': 3.2.4(vite@7.3.2(@types/node@25.6.0)(tsx@4.22.4)(yaml@2.8.4)) + '@vitest/mocker': 3.2.4(vite@7.3.2(@types/node@25.6.0)(tsx@4.23.1)(yaml@2.8.4)) '@vitest/pretty-format': 3.2.4 '@vitest/runner': 3.2.4 '@vitest/snapshot': 3.2.4 @@ -4533,8 +4542,8 @@ snapshots: tinyglobby: 0.2.16 tinypool: 1.1.1 tinyrainbow: 2.0.0 - vite: 7.3.2(@types/node@25.6.0)(tsx@4.22.4)(yaml@2.8.4) - vite-node: 3.2.4(@types/node@25.6.0)(tsx@4.22.4)(yaml@2.8.4) + vite: 7.3.2(@types/node@25.6.0)(tsx@4.23.1)(yaml@2.8.4) + vite-node: 3.2.4(@types/node@25.6.0)(tsx@4.23.1)(yaml@2.8.4) why-is-node-running: 2.3.0 optionalDependencies: '@types/node': 25.6.0 @@ -4597,6 +4606,4 @@ snapshots: zod@3.25.76: {} - zod@4.4.2: {} - zod@4.4.3: {} diff --git a/src/benchmarks/index.ts b/src/benchmarks/index.ts index 0801e7a..9e86aec 100644 --- a/src/benchmarks/index.ts +++ b/src/benchmarks/index.ts @@ -18,13 +18,16 @@ import { stableId } from '../ids' import { appendAttemptJournalEvent, assertNoInterruptedPaidCalls, + DEFAULT_MEMORY_RECOVERY_RETRIES_PER_ATTEMPT, hasSettledPaidCall, readActiveAttemptJournal, reconcileInterruptedMemoryPaidCalls, + reserveRecoveryAttempts, } from '../memory/attempt-log' import { createMemoryExecutionPool, memoryRecoveryDelayMs, + releaseMemoryAdapterCreatedAfterAbort, resolveMemoryCleanupTimeoutMs, runBoundedMemoryLifecycle, sleepForMemoryRecovery, @@ -51,7 +54,7 @@ import { } from '../retrieval-eval' const KNOWLEDGE_BENCHMARK_IMPLEMENTATION_REF = 'agent-knowledge:benchmark-suite:v2' -const MEMORY_ADAPTER_BENCHMARK_IMPLEMENTATION_REF = 'agent-knowledge:memory-adapter-benchmark:v2' +const MEMORY_ADAPTER_BENCHMARK_IMPLEMENTATION_REF = 'agent-knowledge:memory-adapter-benchmark:v7' export type KnowledgeBenchmarkTaskKind = | 'retrieval' @@ -282,11 +285,17 @@ export interface MemoryAdapterBenchmarkCandidate { id: string /** Versioned adapter and configuration identity used by resumable caches. */ ref: string + /** Expected adapter.id. Defaults to candidate id and permits lazy no-work resume. */ + adapterId?: string label?: string - /** Construction must be side-effect free; billable provider work belongs in adapter methods. */ + /** Local construction is free; call markExternalCall before billable provisioning or reconnects. */ createAdapter: (input: { purpose: 'execute' | 'recovery' + signal: AbortSignal + markExternalCall(): void }) => AgentMemoryAdapter | Promise + /** Conservative charge for one billable adapter provisioning or reconnect call. */ + adapterCreationCostUsd?: number searchLimit?: number costUsdPerCase?: number /** Conservative extra provider charge for recovering one interrupted case. */ @@ -314,6 +323,8 @@ export interface RunMemoryAdapterBenchmarkOptions { cleanupTimeoutMs?: number /** Refuse a damaged run with more unfinished attempts than this. Default 1000. */ maxRecoveryAttempts?: number + /** Bound repeated provider cleanup after process crashes. Default 3 per attempt. */ + maxRecoveryRetriesPerAttempt?: number expectUsage?: 'assert' | 'warn' | 'off' now?: () => Date /** Required with custom storage when all controllers are confined to one process. */ @@ -341,15 +352,18 @@ export interface MemoryAdapterBenchmarkRankingRow { export interface RunMemoryAdapterBenchmarkResult { rows: readonly MemoryAdapterBenchmarkRankingRow[] totalCostUsd: number + /** Recovery spend for retired candidates, excluded from ranking rows but included in totalCostUsd. */ + unrankedRecoveryCostUsd: number rankingJsonPath: string rankingMarkdownPath: string attemptLogPath: string + recoveryLogPath: string } class MemoryAdapterBenchmarkCleanupError extends AggregateError {} interface MemoryAdapterBenchmarkAttemptEvent { - schema: 1 + schema: 3 status: 'started' | 'cleaned' attemptId: string candidateId: string @@ -358,6 +372,9 @@ interface MemoryAdapterBenchmarkAttemptEvent { caseId: string cellId: string scope: AgentMemoryScope + adapterCreationCostUsd: number + costUsdPerCase: number + recoveryCostUsdPerAttempt: number recordedAt: string recovery: boolean } @@ -921,6 +938,8 @@ function createMemoryAdapterBenchmarkResponder(options: { searchLimit?: number scope?: AgentMemoryScope costUsdPerCase?: number + adapterCreationCostUsd?: number + recoveryCostUsdPerAttempt?: number now?: () => Date }): KnowledgeBenchmarkResponder { assertScopedMemoryBenchmarkAdapter(options.adapter) @@ -945,7 +964,7 @@ function createMemoryAdapterBenchmarkResponder(options: { options.scope, ) const attempt: MemoryAdapterBenchmarkAttemptEvent = { - schema: 1, + schema: 3, status: 'started', attemptId, candidateId: options.candidateId, @@ -954,16 +973,30 @@ function createMemoryAdapterBenchmarkResponder(options: { caseId: testCase.id, cellId: dispatchContext.cellId, scope, + adapterCreationCostUsd: options.adapterCreationCostUsd ?? 0, + costUsdPerCase: costUsd, + recoveryCostUsdPerAttempt: options.recoveryCostUsdPerAttempt ?? 0, recordedAt: (options.now ?? (() => new Date()))().toISOString(), recovery: false, } appendMemoryBenchmarkAttemptEvent(options.storage, options.attemptLogPath, attempt) let externalCallAttempted = false - let executeStarted = false - let cleanupConfirmed = false + const appendCleanedAttempt = (priorError?: unknown): void => { + try { + appendMemoryBenchmarkAttemptEvent(options.storage, options.attemptLogPath, { + ...attempt, + status: 'cleaned', + recordedAt: (options.now ?? (() => new Date()))().toISOString(), + }) + } catch (error) { + throw new MemoryAdapterBenchmarkCleanupError( + [...(priorError ? [priorError] : []), error], + `${options.candidateId}: memory benchmark cleanup could not be recorded`, + ) + } + } const execute = async (): Promise => { - executeStarted = true dispatchContext.signal.throwIfAborted() await options.lease.assertOwned() let artifact: KnowledgeBenchmarkArtifact | undefined @@ -1044,6 +1077,7 @@ function createMemoryAdapterBenchmarkResponder(options: { await runBoundedMemoryLifecycle({ operation: `${options.candidateId}: benchmark attempt flush`, timeoutMs: options.cleanupTimeoutMs, + resource: options.adapter, run: () => { externalCallAttempted = true return options.adapter.flush?.() @@ -1056,6 +1090,7 @@ function createMemoryAdapterBenchmarkResponder(options: { await runBoundedMemoryLifecycle({ operation: `${options.candidateId}: benchmark attempt cleanup`, timeoutMs: options.cleanupTimeoutMs, + resource: options.adapter, run: () => { externalCallAttempted = true return options.adapter.clear!(scope) @@ -1065,7 +1100,7 @@ function createMemoryAdapterBenchmarkResponder(options: { cleanupErrors.push(error) } } - if (cleanupOwned && cleanupErrors.length === 0) cleanupConfirmed = true + if (cleanupOwned && cleanupErrors.length === 0) appendCleanedAttempt(primaryError) if (cleanupErrors.length > 0) { const errors = [...(primaryError ? [primaryError] : []), ...cleanupErrors] throw new MemoryAdapterBenchmarkCleanupError( @@ -1079,21 +1114,6 @@ function createMemoryAdapterBenchmarkResponder(options: { return artifact } - const appendCleanedAttempt = (priorError?: unknown): void => { - try { - appendMemoryBenchmarkAttemptEvent(options.storage, options.attemptLogPath, { - ...attempt, - status: 'cleaned', - recordedAt: (options.now ?? (() => new Date()))().toISOString(), - }) - } catch (error) { - throw new MemoryAdapterBenchmarkCleanupError( - [...(priorError ? [priorError] : []), error], - `${options.candidateId}: memory benchmark cleanup could not be recorded`, - ) - } - } - if (costUsd === 0) { let artifact: KnowledgeBenchmarkArtifact | undefined let error: unknown @@ -1102,7 +1122,6 @@ function createMemoryAdapterBenchmarkResponder(options: { } catch (caught) { error = caught } - if (cleanupConfirmed) appendCleanedAttempt(error) if (error) throw error if (!artifact) throw new Error(`${options.candidateId}: memory benchmark produced no artifact`) @@ -1126,9 +1145,6 @@ function createMemoryAdapterBenchmarkResponder(options: { actualCostUsd: externalCallAttempted ? costUsd : 0, }), }) - if (!executeStarted || (cleanupConfirmed && (paid.succeeded || paid.receipt))) { - appendCleanedAttempt(paid.succeeded ? undefined : paid.error) - } if (!paid.succeeded) throw paid.error return paid.value } @@ -1146,11 +1162,31 @@ export async function runMemoryAdapterBenchmark( ) for (const candidate of allCandidates) { assertNonEmptyBenchmarkString(candidate.ref, `memory adapter candidate ${candidate.id} ref`) + if (candidate.adapterId !== undefined) { + assertNonEmptyBenchmarkString( + candidate.adapterId, + `memory adapter candidate ${candidate.id} adapterId`, + ) + } if (!/^[A-Za-z0-9][A-Za-z0-9._-]*$/.test(candidate.id)) { throw new Error( `memory adapter candidate id '${candidate.id}' must be a safe directory segment`, ) } + if ( + candidate.adapterCreationCostUsd !== undefined && + (!Number.isFinite(candidate.adapterCreationCostUsd) || candidate.adapterCreationCostUsd < 0) + ) { + throw new Error( + `${candidate.id}: adapterCreationCostUsd must be a non-negative finite number`, + ) + } + if ( + candidate.costUsdPerCase !== undefined && + (!Number.isFinite(candidate.costUsdPerCase) || candidate.costUsdPerCase < 0) + ) { + throw new Error(`${candidate.id}: costUsdPerCase must be a non-negative finite number`) + } if ( candidate.recoveryCostUsdPerAttempt !== undefined && (!Number.isFinite(candidate.recoveryCostUsdPerAttempt) || @@ -1174,6 +1210,13 @@ export async function runMemoryAdapterBenchmark( if (!Number.isSafeInteger(maxRecoveryAttempts) || maxRecoveryAttempts <= 0) { throw new Error('memory adapter benchmark maxRecoveryAttempts must be a positive safe integer') } + const maxRecoveryRetriesPerAttempt = + options.maxRecoveryRetriesPerAttempt ?? DEFAULT_MEMORY_RECOVERY_RETRIES_PER_ATTEMPT + if (!Number.isSafeInteger(maxRecoveryRetriesPerAttempt) || maxRecoveryRetriesPerAttempt <= 0) { + throw new Error( + 'memory adapter benchmark maxRecoveryRetriesPerAttempt must be a positive safe integer', + ) + } storage.ensureDir(runDir) const lease = await acquireAgentMemoryRunLease({ experimentId: `memory-adapter-benchmark:${runDir}`, @@ -1195,6 +1238,7 @@ export async function runMemoryAdapterBenchmark( lease, cleanupTimeoutMs, maxRecoveryAttempts, + maxRecoveryRetriesPerAttempt, ) } catch (error) { primaryError = error @@ -1224,6 +1268,7 @@ async function runOwnedMemoryAdapterBenchmark( lease: OwnedAgentMemoryRunLease, cleanupTimeoutMs: number, maxRecoveryAttempts: number, + maxRecoveryRetriesPerAttempt: number, ): Promise { const maxConcurrency = options.maxConcurrency ?? 2 if (!Number.isSafeInteger(maxConcurrency) || maxConcurrency <= 0) { @@ -1243,6 +1288,7 @@ async function runOwnedMemoryAdapterBenchmark( ) } const attemptLogPath = join(runDir, 'memory-adapter-attempts.jsonl') + const recoveryLogPath = join(runDir, 'memory-adapter-recovery-attempts.jsonl') await recoverMemoryAdapterBenchmarkAttempts({ candidates: [...options.candidates, ...(options.recoveryCandidates ?? [])], storage, @@ -1255,34 +1301,79 @@ async function runOwnedMemoryAdapterBenchmark( costLedger, costPhase: options.costPhase ?? 'memory.adapter-benchmark', maxRecoveryAttempts, + recoveryLogPath, + maxRecoveryRetriesPerAttempt, }) await lease.assertOwned() const rows: MemoryAdapterBenchmarkRankingRow[] = [] for (const candidate of options.candidates) { await lease.assertOwned() - const adapter = await candidate.createAdapter({ purpose: 'execute' }) - await lease.assertOwned() + const expectedAdapterId = memoryAdapterBenchmarkExpectedId(candidate) + let adapter: AgentMemoryAdapter | undefined + let adapterPromise: Promise | undefined + let adapterCreationError: unknown + const getAdapter = (): Promise => { + if (!adapterPromise) { + const abortController = new AbortController() + const creation = createMemoryAdapterBenchmarkAdapter({ + candidate, + purpose: 'execute', + signal: abortController.signal, + costLedger, + runDir, + costPhase: options.costPhase ?? 'memory.adapter-benchmark', + }) + releaseMemoryAdapterCreatedAfterAbort({ creation, signal: abortController.signal }) + adapterPromise = runBoundedMemoryLifecycle({ + operation: `${candidate.id}: benchmark execute adapter creation`, + timeoutMs: Math.min(cleanupTimeoutMs, options.dispatchTimeoutMs ?? cleanupTimeoutMs), + abortController, + run: () => creation, + }) + .then((created) => { + adapter = created + if (created.id !== expectedAdapterId) { + throw new Error( + `${candidate.id}: createAdapter returned id '${created.id}', expected '${expectedAdapterId}'`, + ) + } + assertScopedMemoryBenchmarkAdapter(created) + return created + }) + .catch((error) => { + adapterCreationError = error + throw error + }) + } + return adapterPromise + } const dispatchedExecutions: Promise[] = [] let run: RunKnowledgeBenchmarkSuiteResult | undefined let primaryError: unknown try { - const respond = createMemoryAdapterBenchmarkResponder({ - adapter, - candidateId: candidate.id, - candidateRef: candidate.ref, - storage, - attemptLogPath, - lease, - cleanupTimeoutMs, - searchLimit: candidate.searchLimit, - scope: candidate.scope, - costUsdPerCase: candidate.costUsdPerCase, - now: options.now, - }) + await lease.assertOwned() + let respond: KnowledgeBenchmarkResponder | undefined run = await runKnowledgeBenchmarkSuite({ cases: options.cases, respond(input) { - const operation = Promise.resolve().then(() => respond(input)) + const operation = getAdapter().then((activeAdapter) => { + respond ??= createMemoryAdapterBenchmarkResponder({ + adapter: activeAdapter, + candidateId: candidate.id, + candidateRef: candidate.ref, + storage, + attemptLogPath, + lease, + cleanupTimeoutMs, + searchLimit: candidate.searchLimit, + scope: candidate.scope, + adapterCreationCostUsd: candidate.adapterCreationCostUsd, + costUsdPerCase: candidate.costUsdPerCase, + recoveryCostUsdPerAttempt: candidate.recoveryCostUsdPerAttempt, + now: options.now, + }) + return respond(input) + }) dispatchedExecutions.push(operation) return operation }, @@ -1291,8 +1382,9 @@ async function runOwnedMemoryAdapterBenchmark( canonicalJson({ implementationRef: MEMORY_ADAPTER_BENCHMARK_IMPLEMENTATION_REF, candidateRef: candidate.ref, - adapterId: adapter.id, + adapterId: expectedAdapterId, searchLimit: candidate.searchLimit ?? null, + adapterCreationCostUsd: candidate.adapterCreationCostUsd ?? 0, costUsdPerCase: candidate.costUsdPerCase ?? 0, recoveryCostUsdPerAttempt: candidate.recoveryCostUsdPerAttempt ?? 0, scope: candidate.scope ?? null, @@ -1321,12 +1413,13 @@ async function runOwnedMemoryAdapterBenchmark( ? [settled.reason] : [], ) + if (!primaryError && adapterCreationError) primaryError = adapterCreationError if (run) { rows.push({ rank: 0, candidateId: candidate.id, label: candidate.label ?? candidate.id, - adapterId: adapter.id, + adapterId: expectedAdapterId, scoreMean: run.report.score.mean, passRate: run.report.dimensions.passed?.mean ?? 0, totalCases: run.report.totalCases, @@ -1346,25 +1439,30 @@ async function runOwnedMemoryAdapterBenchmark( cleanupOwned = false cleanupErrors.push(error) } - if (cleanupOwned) { + if (cleanupOwned && adapter && !adapterCreationError) { + const activeAdapter = adapter try { await runBoundedMemoryLifecycle({ operation: `${candidate.id}: benchmark adapter flush`, timeoutMs: cleanupTimeoutMs, - run: () => adapter.flush?.(), + resource: activeAdapter, + run: () => activeAdapter.flush?.(), }) } catch (error) { cleanupErrors.push(error) } } - try { - await runBoundedMemoryLifecycle({ - operation: `${candidate.id}: benchmark adapter close`, - timeoutMs: cleanupTimeoutMs, - run: () => adapter.close?.(), - }) - } catch (error) { - cleanupErrors.push(error) + if (adapter) { + try { + await runBoundedMemoryLifecycle({ + operation: `${candidate.id}: benchmark adapter close`, + timeoutMs: cleanupTimeoutMs, + resource: adapter, + run: () => adapter!.close?.(), + }) + } catch (error) { + cleanupErrors.push(error) + } } if (primaryError || dispatchCleanupErrors.length > 0 || cleanupErrors.length > 0) { const errors = [ @@ -1378,7 +1476,18 @@ async function runOwnedMemoryAdapterBenchmark( await lease.assertOwned() } + const costByCandidate = memoryAdapterBenchmarkCostByCandidate(costLedger, runDir, [ + ...options.candidates, + ...(options.recoveryCandidates ?? []), + ]) const ranked = rows + .map((row) => { + const totalCostUsd = normalizeUsd(costByCandidate.get(row.candidateId) ?? 0) + return { + ...row, + totalCostUsd, + } + }) .sort( (a, b) => Number(a.cellsFailed > 0) - Number(b.cellsFailed > 0) || @@ -1390,16 +1499,140 @@ async function runOwnedMemoryAdapterBenchmark( .map((row, index) => ({ ...row, rank: index + 1 })) const rankingJsonPath = join(runDir, 'memory-adapter-ranking.json') const rankingMarkdownPath = join(runDir, 'memory-adapter-ranking.md') - const totalCostUsd = ranked.reduce((sum, row) => sum + row.totalCostUsd, 0) - storage.write(rankingJsonPath, `${JSON.stringify({ totalCostUsd, rows: ranked }, null, 2)}\n`) - storage.write(rankingMarkdownPath, renderMemoryAdapterRankingMarkdown(ranked, totalCostUsd)) + const unrankedRecoveryCostUsd = normalizeUsd( + (options.recoveryCandidates ?? []).reduce( + (sum, candidate) => sum + (costByCandidate.get(candidate.id) ?? 0), + 0, + ), + ) + const totalCostUsd = normalizeUsd( + [...costByCandidate.values()].reduce((sum, cost) => sum + cost, 0), + ) + storage.write( + rankingJsonPath, + `${JSON.stringify({ totalCostUsd, unrankedRecoveryCostUsd, rows: ranked }, null, 2)}\n`, + ) + storage.write( + rankingMarkdownPath, + renderMemoryAdapterRankingMarkdown(ranked, totalCostUsd, unrankedRecoveryCostUsd), + ) return { rows: ranked, totalCostUsd, + unrankedRecoveryCostUsd, rankingJsonPath, rankingMarkdownPath, attemptLogPath, + recoveryLogPath, + } +} + +async function createMemoryAdapterBenchmarkAdapter(input: { + candidate: MemoryAdapterBenchmarkCandidate + purpose: 'execute' | 'recovery' + signal: AbortSignal + costLedger: CostLedgerHandle + runDir: string + costPhase: string +}): Promise { + const { candidate, purpose, signal, costLedger, runDir, costPhase } = input + const costUsd = candidate.adapterCreationCostUsd ?? 0 + let externalCallAttempted = false + const create = async (): Promise => { + const adapter = await candidate.createAdapter({ + purpose, + signal, + markExternalCall: () => { + externalCallAttempted = true + }, + }) + if (!adapter || typeof adapter !== 'object') { + throw new Error(`${candidate.id}: createAdapter returned no ${purpose} adapter`) + } + return adapter + } + if (costUsd === 0) return create() + + const tags = memoryAdapterCreationCostTags(runDir, candidate.id, purpose) + const generation = costLedger.list({ tags }).length + const receipt = { + model: candidate.id, + inputTokens: 0, + outputTokens: 0, + actualCostUsd: costUsd, + } as const + const paid = await costLedger.runPaidCall({ + callId: memoryAdapterCreationCostCallId(candidate, purpose, generation), + channel: 'driver', + phase: `${costPhase}.${candidate.id}.adapter-${purpose}`, + actor: `agent-knowledge:memory-adapter:${candidate.id}`, + model: candidate.id, + tags, + maximumCharge: { externallyEnforcedMaximumUsd: costUsd }, + execute: create, + receipt: () => ({ + ...receipt, + actualCostUsd: externalCallAttempted ? costUsd : 0, + }), + receiptFromError: () => ({ + ...receipt, + actualCostUsd: externalCallAttempted ? costUsd : 0, + }), + }) + if (!paid.succeeded) throw paid.error + return paid.value +} + +function memoryAdapterCreationCostTags( + runDir: string, + candidateId: string, + purpose: 'execute' | 'recovery', +): Record { + return { + runDir: join(runDir, candidateId), + candidateId, + memoryAdapterCreation: purpose, + } +} + +function memoryAdapterCreationCostCallId( + candidate: MemoryAdapterBenchmarkCandidate, + purpose: 'execute' | 'recovery', + generation: number, +): string { + return stableId( + 'memory_adapter_creation_cost_call', + canonicalJson({ + purpose, + generation, + candidateId: candidate.id, + candidateRef: candidate.ref, + adapterCreationCostUsd: candidate.adapterCreationCostUsd ?? 0, + }), + ) +} + +function memoryAdapterBenchmarkExpectedId(candidate: MemoryAdapterBenchmarkCandidate): string { + return candidate.adapterId ?? candidate.id +} + +function memoryAdapterBenchmarkCostByCandidate( + costLedger: CostLedgerHandle, + runDir: string, + candidates: readonly MemoryAdapterBenchmarkCandidate[], +): ReadonlyMap { + const candidateByRunDir = new Map( + candidates.map((candidate) => [join(runDir, candidate.id), candidate.id]), + ) + const totals = new Map() + for (const receipt of costLedger.list()) { + const candidateId = receipt.tags?.runDir + ? candidateByRunDir.get(receipt.tags.runDir) + : undefined + if (!candidateId) continue + totals.set(candidateId, (totals.get(candidateId) ?? 0) + receipt.costUsd) } + return totals } async function recoverMemoryAdapterBenchmarkAttempts(input: { @@ -1414,6 +1647,8 @@ async function recoverMemoryAdapterBenchmarkAttempts(input: { costLedger: CostLedgerHandle costPhase: string maxRecoveryAttempts: number + recoveryLogPath: string + maxRecoveryRetriesPerAttempt: number }): Promise { let attempts = readActiveMemoryBenchmarkAttempts(input.storage, input.attemptLogPath) if (attempts.length > input.maxRecoveryAttempts) { @@ -1429,11 +1664,7 @@ async function recoverMemoryAdapterBenchmarkAttempts(input: { `cannot recover memory benchmark attempts: candidate '${attempt.candidateId}' is missing; pass it in recoveryCandidates`, ) } - if (attempt.candidateRef !== candidate.ref) { - throw new Error( - `cannot recover memory benchmark attempt '${attempt.attemptId}': candidate ref changed from '${attempt.candidateRef}' to '${candidate.ref}'`, - ) - } + assertMemoryBenchmarkAttemptCandidateMatches(attempt, candidate) } reconcileInterruptedMemoryPaidCalls(input.costLedger) @@ -1455,6 +1686,14 @@ async function recoverMemoryAdapterBenchmarkAttempts(input: { } } attempts = readActiveMemoryBenchmarkAttempts(input.storage, input.attemptLogPath) + const recoveryGenerations = reserveRecoveryAttempts({ + storage: input.storage, + path: input.recoveryLogPath, + attemptIds: attempts.map((attempt) => attempt.attemptId), + maxRetriesPerAttempt: input.maxRecoveryRetriesPerAttempt, + label: 'memory benchmark recovery attempt log', + now: input.now, + }) const grouped = groupMemoryBenchmarkAttempts(attempts) const pool = createMemoryExecutionPool(input.maxConcurrency) const settled = await Promise.allSettled( @@ -1470,24 +1709,40 @@ async function recoverMemoryAdapterBenchmarkAttempts(input: { ) } for (const attempt of candidateAttempts) { - if (attempt.candidateRef !== candidate.ref) { + assertMemoryBenchmarkAttemptCandidateMatches(attempt, candidate) + if (recoveryGenerations.get(attempt.attemptId) === undefined) { throw new Error( - `cannot recover memory benchmark attempt '${attempt.attemptId}': candidate ref changed from '${attempt.candidateRef}' to '${candidate.ref}'`, + `missing recovery generation for memory benchmark attempt '${attempt.attemptId}'`, ) } } let recoveryAttemptsStarted = 0 + let adapterCreationExternalCallAttempted = false const cleared: MemoryAdapterBenchmarkAttemptEvent[] = [] - let cleanupCanBeRecorded = false const recover = async (): Promise => { let adapter: AgentMemoryAdapter | undefined let primaryError: unknown try { + const abortController = new AbortController() + const creation = Promise.resolve().then(() => + candidate.createAdapter({ + purpose: 'recovery', + signal: abortController.signal, + markExternalCall: () => { + adapterCreationExternalCallAttempted = true + }, + }), + ) + releaseMemoryAdapterCreatedAfterAbort({ + creation, + signal: abortController.signal, + }) adapter = await runBoundedMemoryLifecycle({ operation: `${candidate.id}: benchmark recovery adapter creation`, timeoutMs: input.cleanupTimeoutMs, - run: () => candidate.createAdapter({ purpose: 'recovery' }), + abortController, + run: () => creation, }) assertScopedMemoryBenchmarkAdapter(adapter) for (const attempt of candidateAttempts) { @@ -1512,6 +1767,7 @@ async function recoverMemoryAdapterBenchmarkAttempts(input: { await runBoundedMemoryLifecycle({ operation: `${candidate.id}: abandoned benchmark attempt cleanup`, timeoutMs: input.cleanupTimeoutMs, + resource: adapter, run: () => adapter!.clear!(attempt.scope), }) await input.lease.assertOwned() @@ -1540,46 +1796,68 @@ async function recoverMemoryAdapterBenchmarkAttempts(input: { await runBoundedMemoryLifecycle({ operation: `${candidate.id}: benchmark recovery adapter close`, timeoutMs: input.cleanupTimeoutMs, + resource: adapter, run: () => adapter!.close?.(), }) } catch (error) { closeError = error } } - cleanupCanBeRecorded = !closeError - if (primaryError && closeError) { + let journalError: unknown + if (!closeError) { + try { + for (const attempt of cleared) { + await input.lease.assertOwned() + appendMemoryBenchmarkAttemptEvent(input.storage, input.attemptLogPath, { + ...attempt, + status: 'cleaned', + recovery: true, + recordedAt: (input.now ?? (() => new Date()))().toISOString(), + }) + } + } catch (error) { + journalError = error + } + } + const failures = [primaryError, closeError, journalError].filter( + (error) => error !== undefined, + ) + if (failures.length > 1) { throw new AggregateError( - [primaryError, closeError], - `${candidate.id}: benchmark attempt recovery and adapter close failed`, + failures, + `${candidate.id}: benchmark attempt recovery, adapter close, or cleanup journal failed`, ) } if (primaryError) throw primaryError if (closeError) throw closeError + if (journalError) throw journalError } const recoveryCostUsd = candidate.recoveryCostUsdPerAttempt ?? 0 + const adapterCreationCostUsd = candidate.adapterCreationCostUsd ?? 0 + const maximumCostUsd = adapterCreationCostUsd + recoveryCostUsd * candidateAttempts.length + const actualCostUsd = (): number => + (adapterCreationExternalCallAttempted ? adapterCreationCostUsd : 0) + + recoveryCostUsd * recoveryAttemptsStarted let recoveryError: unknown - let costReceiptCaptured = recoveryCostUsd === 0 - if (recoveryCostUsd === 0) { + if (maximumCostUsd === 0) { try { await recover() } catch (error) { recoveryError = error } } else { - const maximumCostUsd = recoveryCostUsd * candidateAttempts.length const receipt = { model: candidate.id, inputTokens: 0, outputTokens: 0, - actualCostUsd: maximumCostUsd, } as const const tags = memoryBenchmarkRecoveryCostTags(input.runDir, candidate.id) const paid = await input.costLedger.runPaidCall({ callId: memoryBenchmarkRecoveryCostCallId( candidate, candidateAttempts, - input.costLedger.list({ tags }).length, + recoveryGenerations, ), channel: 'driver', phase: `${input.costPhase}.${candidate.id}.recovery`, @@ -1588,36 +1866,14 @@ async function recoverMemoryAdapterBenchmarkAttempts(input: { tags, maximumCharge: { externallyEnforcedMaximumUsd: maximumCostUsd }, execute: recover, - receipt: () => receipt, + receipt: () => ({ ...receipt, actualCostUsd: actualCostUsd() }), receiptFromError: () => ({ ...receipt, - actualCostUsd: recoveryCostUsd * recoveryAttemptsStarted, + actualCostUsd: actualCostUsd(), }), }) - costReceiptCaptured = paid.succeeded || Boolean(paid.receipt) if (!paid.succeeded) recoveryError = paid.error } - - if (cleanupCanBeRecorded && costReceiptCaptured) { - try { - for (const attempt of cleared) { - await input.lease.assertOwned() - appendMemoryBenchmarkAttemptEvent(input.storage, input.attemptLogPath, { - ...attempt, - status: 'cleaned', - recovery: true, - recordedAt: (input.now ?? (() => new Date()))().toISOString(), - }) - } - } catch (error) { - recoveryError = recoveryError - ? new AggregateError( - [recoveryError, error], - `${candidate.id}: recovery result and cleanup journal both failed`, - ) - : error - } - } if (recoveryError) throw recoveryError }), ), @@ -1657,6 +1913,9 @@ function memoryBenchmarkCostCallId( candidateId: attempt.candidateId, candidateRef: attempt.candidateRef, adapterId: attempt.adapterId, + adapterCreationCostUsd: attempt.adapterCreationCostUsd, + costUsdPerCase: attempt.costUsdPerCase, + recoveryCostUsdPerAttempt: attempt.recoveryCostUsdPerAttempt, caseId: attempt.caseId, cellId: attempt.cellId, }), @@ -1666,15 +1925,21 @@ function memoryBenchmarkCostCallId( function memoryBenchmarkRecoveryCostCallId( candidate: MemoryAdapterBenchmarkCandidate, attempts: readonly MemoryAdapterBenchmarkAttemptEvent[], - generation: number, + generations: ReadonlyMap, ): string { return stableId( 'memory_benchmark_recovery_cost_call', canonicalJson({ - generation, candidateId: candidate.id, candidateRef: candidate.ref, - attemptIds: attempts.map((attempt) => attempt.attemptId).sort(), + adapterCreationCostUsd: candidate.adapterCreationCostUsd ?? 0, + recoveryCostUsdPerAttempt: candidate.recoveryCostUsdPerAttempt ?? 0, + attempts: attempts + .map((attempt) => ({ + attemptId: attempt.attemptId, + generation: generations.get(attempt.attemptId), + })) + .sort((left, right) => left.attemptId.localeCompare(right.attemptId)), }), ) } @@ -1726,7 +1991,7 @@ function parseMemoryBenchmarkAttemptEvent( const valid = typeof event === 'object' && event !== null && - event.schema === 1 && + event.schema === 3 && (event.status === 'started' || event.status === 'cleaned') && isNonEmptyString(event.attemptId) && isNonEmptyString(event.candidateId) && @@ -1735,6 +2000,15 @@ function parseMemoryBenchmarkAttemptEvent( isNonEmptyString(event.caseId) && isNonEmptyString(event.cellId) && isAgentMemoryScope(event.scope) && + typeof event.adapterCreationCostUsd === 'number' && + Number.isFinite(event.adapterCreationCostUsd) && + event.adapterCreationCostUsd >= 0 && + typeof event.costUsdPerCase === 'number' && + Number.isFinite(event.costUsdPerCase) && + event.costUsdPerCase >= 0 && + typeof event.recoveryCostUsdPerAttempt === 'number' && + Number.isFinite(event.recoveryCostUsdPerAttempt) && + event.recoveryCostUsdPerAttempt >= 0 && typeof event.recordedAt === 'string' && !Number.isNaN(Date.parse(event.recordedAt)) && typeof event.recovery === 'boolean' @@ -1755,10 +2029,42 @@ function sameMemoryBenchmarkAttempt( left.adapterId === right.adapterId && left.caseId === right.caseId && left.cellId === right.cellId && - canonicalJson(left.scope) === canonicalJson(right.scope) + canonicalJson(left.scope) === canonicalJson(right.scope) && + left.adapterCreationCostUsd === right.adapterCreationCostUsd && + left.costUsdPerCase === right.costUsdPerCase && + left.recoveryCostUsdPerAttempt === right.recoveryCostUsdPerAttempt ) } +function assertMemoryBenchmarkAttemptCandidateMatches( + attempt: MemoryAdapterBenchmarkAttemptEvent, + candidate: MemoryAdapterBenchmarkCandidate, +): void { + if (attempt.candidateRef !== candidate.ref) { + throw new Error( + `cannot recover memory benchmark attempt '${attempt.attemptId}': candidate ref changed from '${attempt.candidateRef}' to '${candidate.ref}'`, + ) + } + const expectedAdapterId = memoryAdapterBenchmarkExpectedId(candidate) + if (attempt.adapterId !== expectedAdapterId) { + throw new Error( + `cannot recover memory benchmark attempt '${attempt.attemptId}': adapter id changed from '${attempt.adapterId}' to '${expectedAdapterId}'`, + ) + } + const executionCost = candidate.costUsdPerCase ?? 0 + const adapterCreationCost = candidate.adapterCreationCostUsd ?? 0 + const recoveryCost = candidate.recoveryCostUsdPerAttempt ?? 0 + if ( + adapterCreationCost !== attempt.adapterCreationCostUsd || + executionCost !== attempt.costUsdPerCase || + recoveryCost !== attempt.recoveryCostUsdPerAttempt + ) { + throw new Error( + `cannot recover memory benchmark attempt '${attempt.attemptId}': candidate cost settings changed; start a new run or restore the recorded costs`, + ) + } +} + function isAgentMemoryScope(value: unknown): value is AgentMemoryScope { if (!isRecordValue(value)) return false const allowed = new Set([ @@ -2459,11 +2765,13 @@ function memoryLifecycleCase(input: { function renderMemoryAdapterRankingMarkdown( rows: readonly MemoryAdapterBenchmarkRankingRow[], totalCostUsd: number, + unrankedRecoveryCostUsd: number, ): string { return [ '# Memory Adapter Ranking', '', `- total cost: $${formatNumber(totalCostUsd)}`, + `- retired-candidate recovery cost: $${formatNumber(unrankedRecoveryCostUsd)}`, '', '| rank | candidate | adapter | cases | cells | failed | mean score | pass rate | cost |', '| ---: | --- | --- | ---: | ---: | ---: | ---: | ---: | ---: |', @@ -2803,6 +3111,10 @@ function formatNumber(value: number): string { return value.toFixed(value === 0 || Math.abs(value) >= 10 ? 0 : 3) } +function normalizeUsd(value: number): number { + return Number(value.toFixed(12)) +} + function compactObject(value: unknown): unknown { if (Array.isArray(value)) return value.map(compactObject) if (!value || typeof value !== 'object') return value diff --git a/src/memory/attempt-log.ts b/src/memory/attempt-log.ts index 9ce9d0f..56f1f02 100644 --- a/src/memory/attempt-log.ts +++ b/src/memory/attempt-log.ts @@ -5,8 +5,17 @@ export interface AttemptJournalEvent { recordedAt: string } +interface RecoveryAttemptEvent { + schema: 1 + attemptId: string + generation: number + recordedAt: string +} + const journalByteLengths = new WeakMap>() +export const DEFAULT_MEMORY_RECOVERY_RETRIES_PER_ATTEMPT = 3 + const MEMORY_PROVIDER_ACTOR_PREFIXES = [ 'agent-knowledge:memory-adapter:', 'agent-knowledge:memory-adapter-recovery:', @@ -119,6 +128,52 @@ export function appendDurableJournalEvent(input: { throw new Error(`${label} '${path}' remained contended after 100 retries`) } +/** Reserves one durable recovery generation per attempt before provider work starts. */ +export function reserveRecoveryAttempts(input: { + storage: CampaignStorage + path: string + attemptIds: readonly string[] + maxRetriesPerAttempt: number + label: string + now?: () => Date +}): ReadonlyMap { + const { storage, path, attemptIds, maxRetriesPerAttempt, label } = input + if (!Number.isSafeInteger(maxRetriesPerAttempt) || maxRetriesPerAttempt <= 0) { + throw new Error(`${label} maxRetriesPerAttempt must be a positive safe integer`) + } + const uniqueAttemptIds = new Set(attemptIds) + if (uniqueAttemptIds.size !== attemptIds.length || attemptIds.some((id) => !id.trim())) { + throw new Error(`${label} attempt ids must be unique non-empty strings`) + } + const generations = readRecoveryAttemptGenerations(storage, path, label) + for (const attemptId of attemptIds) { + const prior = generations.get(attemptId) ?? 0 + if (prior >= maxRetriesPerAttempt) { + throw new Error( + `${label} '${attemptId}' exhausted ${maxRetriesPerAttempt} recovery attempts; increase maxRecoveryRetriesPerAttempt only after inspecting provider state`, + ) + } + } + const reserved = new Map() + for (const attemptId of attemptIds) { + const generation = (generations.get(attemptId) ?? 0) + 1 + appendDurableJournalEvent({ + storage, + path, + label, + event: { + schema: 1, + attemptId, + generation, + recordedAt: (input.now ?? (() => new Date()))().toISOString(), + } satisfies RecoveryAttemptEvent, + }) + generations.set(attemptId, generation) + reserved.set(attemptId, generation) + } + return reserved +} + export function readActiveAttemptJournal(input: { storage: CampaignStorage path: string @@ -167,6 +222,58 @@ export function readActiveAttemptJournal(inp return [...active.values()] } +function readRecoveryAttemptGenerations( + storage: CampaignStorage, + path: string, + label: string, +): Map { + const stored = storage.read(path) + if (stored === undefined) { + if (storage.exists(path)) throw new Error(`cannot read ${label} '${path}'`) + return new Map() + } + const generations = new Map() + for (const [index, line] of stored.split('\n').entries()) { + if (!line) continue + let raw: unknown + try { + raw = JSON.parse(line) + } catch (error) { + throw new Error(`invalid ${label} '${path}' line ${index + 1}`, { cause: error }) + } + const event = parseRecoveryAttemptEvent(raw, path, index + 1, label) + const expected = (generations.get(event.attemptId) ?? 0) + 1 + if (event.generation !== expected) { + throw new Error( + `${label} '${path}' has recovery generation ${event.generation} for '${event.attemptId}', expected ${expected}`, + ) + } + generations.set(event.attemptId, event.generation) + } + return generations +} + +function parseRecoveryAttemptEvent( + value: unknown, + path: string, + line: number, + label: string, +): RecoveryAttemptEvent { + const event = value as Partial | null + const valid = + typeof event === 'object' && + event !== null && + event.schema === 1 && + typeof event.attemptId === 'string' && + event.attemptId.length > 0 && + Number.isSafeInteger(event.generation) && + event.generation! > 0 && + typeof event.recordedAt === 'string' && + !Number.isNaN(Date.parse(event.recordedAt)) + if (!valid) throw new Error(`invalid ${label} '${path}' line ${line}`) + return value as RecoveryAttemptEvent +} + function isMissingPendingCostCall(error: unknown, callId: string): boolean { return error instanceof Error && error.message === `CostLedger: no pending call '${callId}'` } diff --git a/src/memory/experiment.ts b/src/memory/experiment.ts index 0dee261..a2a9db0 100644 --- a/src/memory/experiment.ts +++ b/src/memory/experiment.ts @@ -26,9 +26,11 @@ import { stableId } from '../ids' import { appendAttemptJournalEvent, assertNoInterruptedPaidCalls, + DEFAULT_MEMORY_RECOVERY_RETRIES_PER_ATTEMPT, hasSettledPaidCall, readActiveAttemptJournal, reconcileInterruptedMemoryPaidCalls, + reserveRecoveryAttempts, } from './attempt-log' import { type AgentMemoryBranch, @@ -39,6 +41,7 @@ import { import { createMemoryExecutionPool, memoryRecoveryDelayMs, + releaseMemoryAdapterCreatedAfterAbort, resolveMemoryCleanupTimeoutMs, runBoundedMemoryLifecycle, sleepForMemoryRecovery, @@ -52,7 +55,7 @@ import { } from './run-control' import type { AgentMemoryAdapter, AgentMemoryScope, AgentMemoryWriteInput } from './types' -const MEMORY_EXPERIMENT_IMPLEMENTATION_REF = 'agent-knowledge:memory-experiment:v3' +const MEMORY_EXPERIMENT_IMPLEMENTATION_REF = 'agent-knowledge:memory-experiment:v6' export interface AgentMemorySequenceProbe { id: string @@ -104,13 +107,15 @@ export interface AgentMemoryExperimentCandidate { label?: string /** Change when provider configuration changes so cached cells cannot be reused. */ ref: string - /** Construction must be side-effect free; billable provider work belongs in adapter methods. */ + /** Local construction is free; call markExternalCall before billable provisioning or reconnects. */ createAdapter(input: { branchId: string sequence: AgentMemorySequence rep: number seed: number purpose: 'execute' | 'recovery' + signal: AbortSignal + markExternalCall(): void }): AgentMemoryAdapter | null | Promise policy?: AgentMemorySharingPolicy baseScope?: AgentMemoryScope @@ -210,6 +215,8 @@ export interface RunAgentMemoryExperimentOptions { cleanupTimeoutMs?: number /** Refuse a damaged run with more unfinished attempts than this. Default 1000. */ maxRecoveryAttempts?: number + /** Bound repeated provider cleanup after process crashes. Default 3 per attempt. */ + maxRecoveryRetriesPerAttempt?: number now?: () => Date /** Required with custom storage when all controllers are confined to one process. */ controllerMode?: AgentMemoryControllerMode @@ -220,7 +227,7 @@ export interface RunAgentMemoryExperimentOptions { export type AgentMemoryExperimentRunLease = AgentMemoryRunLease export interface AgentMemoryAttemptEvent { - schema: 1 + schema: 2 status: 'started' | 'cleaned' branchId: string candidateId: string @@ -229,6 +236,8 @@ export interface AgentMemoryAttemptEvent { rep: number seed: number cleanupBranches: boolean + externalCostUsdPerSequence: number + externalRecoveryCostUsdPerAttempt: number recordedAt: string recovery: boolean } @@ -236,10 +245,14 @@ export interface AgentMemoryAttemptEvent { export interface RunAgentMemoryExperimentResult { campaign: CampaignResult rows: readonly AgentMemoryExperimentRankingRow[] + totalCostUsd: number + /** Recovery spend for retired candidates, excluded from ranking rows but included in totalCostUsd. */ + unrankedRecoveryCostUsd: number leaderCandidateId?: string rankingJsonPath: string rankingMarkdownPath: string attemptLogPath: string + recoveryLogPath: string } class AgentMemoryCleanupError extends AggregateError { @@ -387,6 +400,13 @@ export async function runAgentMemoryExperiment( if (!Number.isSafeInteger(maxRecoveryAttempts) || maxRecoveryAttempts <= 0) { throw new Error('memory experiment maxRecoveryAttempts must be a positive safe integer') } + const maxRecoveryRetriesPerAttempt = + options.maxRecoveryRetriesPerAttempt ?? DEFAULT_MEMORY_RECOVERY_RETRIES_PER_ATTEMPT + if (!Number.isSafeInteger(maxRecoveryRetriesPerAttempt) || maxRecoveryRetriesPerAttempt <= 0) { + throw new Error( + 'memory experiment maxRecoveryRetriesPerAttempt must be a positive safe integer', + ) + } storage.ensureDir(runDir) const lease = await acquireAgentMemoryRunLease({ experimentId: options.experimentId, @@ -453,6 +473,7 @@ async function runOwnedAgentMemoryExperiment( const sequenceById = new Map(options.sequences.map((sequence) => [sequence.id, sequence])) const scenarios = buildAgentMemorySequenceScenarios(options.sequences, options.candidates) const attemptLogPath = join(runDir, 'memory-attempts.jsonl') + const recoveryLogPath = join(runDir, 'memory-recovery-attempts.jsonl') const costCeiling = options.costCeiling ?? options.costLedger?.costCeilingUsd ?? 0 const costLedger = options.costLedger ?? @@ -465,7 +486,7 @@ async function runOwnedAgentMemoryExperiment( throw new Error('memory experiment costCeiling must match the shared cost ledger ceiling') } storage.ensureDir(runDir) - const recoveryCostByCandidate = await recoverAbandonedMemoryAttempts({ + await recoverAbandonedMemoryAttempts({ options, storage, runDir, @@ -476,6 +497,9 @@ async function runOwnedAgentMemoryExperiment( maxConcurrency, costLedger, maxRecoveryAttempts: options.maxRecoveryAttempts ?? 1_000, + recoveryLogPath, + maxRecoveryRetriesPerAttempt: + options.maxRecoveryRetriesPerAttempt ?? DEFAULT_MEMORY_RECOVERY_RETRIES_PER_ATTEMPT, }) await lease.assertOwned() let campaign: CampaignResult | undefined @@ -539,26 +563,46 @@ async function runOwnedAgentMemoryExperiment( } if (!campaign) throw new Error('memory experiment produced no campaign result') await lease.assertOwned() - const rows = rankAgentMemoryExperiment( - options.candidates, + const costByCandidate = memoryExperimentCostByCandidate( + costLedger, + campaign.runDir, scenarios, - campaign, - recoveryCostByCandidate, + [...options.candidates, ...(options.recoveryCandidates ?? [])].map((candidate) => candidate.id), ) + const unrankedRecoveryCostUsd = normalizeUsd( + (options.recoveryCandidates ?? []).reduce( + (sum, candidate) => sum + (costByCandidate.get(candidate.id) ?? 0), + 0, + ), + ) + const totalCostUsd = normalizeUsd( + [...costByCandidate.values()].reduce((sum, cost) => sum + cost, 0), + ) + const rows = rankAgentMemoryExperiment(options.candidates, scenarios, campaign, costByCandidate) const rankingJsonPath = join(campaign.runDir, 'memory-experiment-ranking.json') const rankingMarkdownPath = join(campaign.runDir, 'memory-experiment-ranking.md') storage.write( rankingJsonPath, - `${JSON.stringify({ experimentId: options.experimentId, rows }, null, 2)}\n`, + `${JSON.stringify( + { experimentId: options.experimentId, totalCostUsd, unrankedRecoveryCostUsd, rows }, + null, + 2, + )}\n`, + ) + storage.write( + rankingMarkdownPath, + renderAgentMemoryExperimentRanking(rows, totalCostUsd, unrankedRecoveryCostUsd), ) - storage.write(rankingMarkdownPath, renderAgentMemoryExperimentRanking(rows)) return { campaign, rows, + totalCostUsd, + unrankedRecoveryCostUsd, leaderCandidateId: rows.find((row) => row.cellsFailed === 0)?.candidateId, rankingJsonPath, rankingMarkdownPath, attemptLogPath, + recoveryLogPath, } } @@ -658,10 +702,21 @@ async function runSequenceCell(input: { appendMemoryAttemptEvent(storage, attemptLogPath, attempt) let externalCallAttempted = false - let executeStarted = false - let cleanupConfirmed = false + const appendCleanedAttempt = (priorError?: unknown): void => { + try { + appendMemoryAttemptEvent(storage, attemptLogPath, { + ...attempt, + status: 'cleaned', + recordedAt: (options.now ?? (() => new Date()))().toISOString(), + }) + } catch (error) { + throw new AgentMemoryCleanupError( + [...(priorError ? [priorError] : []), error], + `${candidate.id}: memory branch cleanup could not be recorded`, + ) + } + } const execute = async (): Promise => { - executeStarted = true context.signal.throwIfAborted() await lease.assertOwned() let rawAdapter: AgentMemoryAdapter | undefined @@ -678,6 +733,10 @@ async function runSequenceCell(input: { rep: context.rep, seed: context.seed, purpose: 'execute', + signal: context.signal, + markExternalCall: () => { + externalCallAttempted = true + }, }) if (!created) throw new Error(`${candidate.id}: createAdapter returned no execution adapter`) rawAdapter = created @@ -699,14 +758,6 @@ async function runSequenceCell(input: { allowedWriteScopes: sequenceCleanupScopes(scenario.sequence), baseScope: memoryExperimentBaseScope(options, candidate, scenario.sequenceId), }) - if (cleanupBranches) { - await runBoundedMemoryLifecycle({ - operation: `${candidate.id}: initial branch cleanup`, - timeoutMs: cleanupTimeoutMs, - run: () => clearSequenceScopes(memory!, scenario.sequence), - }) - await lease.assertOwned() - } const probes: AgentMemorySequenceProbeResult[] = [] for (const step of scenario.sequence.steps) { context.signal.throwIfAborted() @@ -756,6 +807,7 @@ async function runSequenceCell(input: { await runBoundedMemoryLifecycle({ operation: `${candidate.id}: final branch cleanup`, timeoutMs: cleanupTimeoutMs, + resource: adapter, run: () => clearSequenceScopes(memory!, scenario.sequence), }) finalClearCompleted = true @@ -789,6 +841,7 @@ async function runSequenceCell(input: { await runBoundedMemoryLifecycle({ operation: `${candidate.id}: failed branch cleanup`, timeoutMs: cleanupTimeoutMs, + resource: adapter, run: () => clearSequenceScopes(memory!, scenario.sequence), }) } catch (error) { @@ -800,6 +853,7 @@ async function runSequenceCell(input: { await runBoundedMemoryLifecycle({ operation: `${candidate.id}: adapter close`, timeoutMs: cleanupTimeoutMs, + resource: adapter, run: async () => { if (memory && cleanupOwned) await memory.close?.() else { @@ -816,6 +870,7 @@ async function runSequenceCell(input: { await runBoundedMemoryLifecycle({ operation: `${candidate.id}: adapter disposal`, timeoutMs: cleanupTimeoutMs, + resource: adapter, run: () => { if (candidate.disposeAdapter) externalCallAttempted = true return candidate.disposeAdapter?.(rawAdapter!) @@ -830,7 +885,7 @@ async function runSequenceCell(input: { new Error(`${candidate.id}: adapter creation failed before cleanup could be confirmed`), ) } - if (cleanupOwned && cleanupErrors.length === 0) cleanupConfirmed = true + if (cleanupOwned && cleanupErrors.length === 0) appendCleanedAttempt(primaryError) if (!cleanupOwned && cleanupErrors.length === 0) { if (primaryError) throw primaryError throw ownershipError @@ -852,21 +907,6 @@ async function runSequenceCell(input: { return completedArtifact } - const appendCleanedAttempt = (priorError?: unknown): void => { - try { - appendMemoryAttemptEvent(storage, attemptLogPath, { - ...attempt, - status: 'cleaned', - recordedAt: (options.now ?? (() => new Date()))().toISOString(), - }) - } catch (error) { - throw new AgentMemoryCleanupError( - [...(priorError ? [priorError] : []), error], - `${candidate.id}: memory branch cleanup could not be recorded`, - ) - } - } - if (costUsd === 0) { let artifact: AgentMemorySequenceArtifact | undefined let error: unknown @@ -875,7 +915,6 @@ async function runSequenceCell(input: { } catch (caught) { error = caught } - if (cleanupConfirmed) appendCleanedAttempt(error) if (error) throw error if (!artifact) throw new Error(`${candidate.id}: memory sequence produced no result`) return artifact @@ -898,9 +937,6 @@ async function runSequenceCell(input: { actualCostUsd: externalCallAttempted ? costUsd : 0, }), }) - if (!executeStarted || (cleanupConfirmed && (paid.succeeded || paid.receipt))) { - appendCleanedAttempt(paid.succeeded ? undefined : paid.error) - } if (!paid.succeeded) throw paid.error return paid.value } @@ -962,7 +998,9 @@ async function recoverAbandonedMemoryAttempts(input: { maxConcurrency: number costLedger: CostLedgerHandle maxRecoveryAttempts: number -}): Promise> { + recoveryLogPath: string + maxRecoveryRetriesPerAttempt: number +}): Promise { let attempts = readActiveMemoryAttempts(input.storage, input.attemptLogPath) if (attempts.length > input.maxRecoveryAttempts) { throw new Error( @@ -976,11 +1014,7 @@ async function recoverAbandonedMemoryAttempts(input: { `cannot recover memory branch '${attempt.branchId}': candidate '${attempt.candidateId}' is missing; pass it in recoveryCandidates`, ) } - if (candidate.ref !== attempt.candidateRef) { - throw new Error( - `cannot recover memory branch '${attempt.branchId}': candidate ref changed from '${attempt.candidateRef}' to '${candidate.ref}'`, - ) - } + assertMemoryAttemptCandidateMatches(attempt, candidate) if (!input.sequenceById.has(attempt.sequenceId)) { throw new Error( `cannot recover memory branch '${attempt.branchId}': sequence '${attempt.sequenceId}' is missing`, @@ -1010,6 +1044,14 @@ async function recoverAbandonedMemoryAttempts(input: { } } attempts = readActiveMemoryAttempts(input.storage, input.attemptLogPath) + const recoveryGenerations = reserveRecoveryAttempts({ + storage: input.storage, + path: input.recoveryLogPath, + attemptIds: attempts.map((attempt) => attempt.branchId), + maxRetriesPerAttempt: input.maxRecoveryRetriesPerAttempt, + label: 'memory recovery attempt log', + now: input.options.now, + }) const pool = createMemoryExecutionPool(input.maxConcurrency) const settled = await Promise.allSettled( attempts @@ -1023,11 +1065,7 @@ async function recoverAbandonedMemoryAttempts(input: { `cannot recover memory branch '${attempt.branchId}': candidate '${attempt.candidateId}' is missing; pass it in recoveryCandidates`, ) } - if (candidate.ref !== attempt.candidateRef) { - throw new Error( - `cannot recover memory branch '${attempt.branchId}': candidate ref changed from '${attempt.candidateRef}' to '${candidate.ref}'`, - ) - } + assertMemoryAttemptCandidateMatches(attempt, candidate) const sequence = input.sequenceById.get(attempt.sequenceId) if (!sequence) { throw new Error( @@ -1040,6 +1078,10 @@ async function recoverAbandonedMemoryAttempts(input: { ) } const recoveryCostUsd = candidate.externalRecoveryCostUsdPerAttempt ?? 0 + const recoveryGeneration = recoveryGenerations.get(attempt.branchId) + if (recoveryGeneration === undefined) { + throw new Error(`missing recovery generation for memory branch '${attempt.branchId}'`) + } let externalRecoveryAttempted = false const recover = async (): Promise => { await recoverMemoryAttempt({ @@ -1052,15 +1094,15 @@ async function recoverAbandonedMemoryAttempts(input: { externalRecoveryAttempted = true }, }) - } - if (recoveryCostUsd === 0) { - await recover() appendMemoryAttemptEvent(input.storage, input.attemptLogPath, { ...attempt, status: 'cleaned', recovery: true, recordedAt: (input.options.now ?? (() => new Date()))().toISOString(), }) + } + if (recoveryCostUsd === 0) { + await recover() } else { const tags = memoryRecoveryCostTags(input.runDir, candidate.id, attempt.branchId) const receipt = { @@ -1070,11 +1112,7 @@ async function recoverAbandonedMemoryAttempts(input: { actualCostUsd: recoveryCostUsd, } as const const paid = await input.costLedger.runPaidCall({ - callId: memoryAttemptCostCallId( - attempt, - 'recovery', - input.costLedger.list({ tags }).length, - ), + callId: memoryAttemptCostCallId(attempt, 'recovery', recoveryGeneration), channel: 'driver', phase: `${input.options.costPhase ?? 'memory.experiment'}.recovery`, actor: `agent-knowledge:memory-recovery:${candidate.id}`, @@ -1092,12 +1130,6 @@ async function recoverAbandonedMemoryAttempts(input: { }), }) if (!paid.succeeded) throw paid.error - appendMemoryAttemptEvent(input.storage, input.attemptLogPath, { - ...attempt, - status: 'cleaned', - recovery: true, - recordedAt: (input.options.now ?? (() => new Date()))().toISOString(), - }) } }), ), @@ -1109,12 +1141,6 @@ async function recoverAbandonedMemoryAttempts(input: { if (failures.length > 1) { throw new AggregateError(failures, 'multiple abandoned memory branches failed recovery') } - return new Map( - [...input.candidateById.keys()].map((candidateId) => [ - candidateId, - input.costLedger.summary({ tags: { runDir: input.runDir, candidateId } }).totalCostUsd, - ]), - ) } async function recoverMemoryAttempt(input: { @@ -1136,17 +1162,33 @@ async function recoverMemoryAttempt(input: { let memory: AgentMemoryBranch | undefined let primaryError: unknown try { + const abortController = new AbortController() + const creation = Promise.resolve().then(() => + candidate.createAdapter({ + branchId: attempt.branchId, + sequence, + rep: attempt.rep, + seed: attempt.seed, + purpose: 'recovery', + signal: abortController.signal, + markExternalCall: onExternalCall, + }), + ) + releaseMemoryAdapterCreatedAfterAbort({ + creation, + signal: abortController.signal, + dispose: candidate.disposeAdapter + ? async (created) => { + onExternalCall() + await candidate.disposeAdapter?.(created) + } + : undefined, + }) const recovered = await runBoundedMemoryLifecycle({ operation: `${candidate.id}: recovery adapter creation`, timeoutMs: cleanupTimeoutMs, - run: () => - candidate.createAdapter({ - branchId: attempt.branchId, - sequence, - rep: attempt.rep, - seed: attempt.seed, - purpose: 'recovery', - }), + abortController, + run: () => creation, }) await lease.assertOwned() if (recovered === null) return @@ -1176,6 +1218,7 @@ async function recoverMemoryAttempt(input: { await runBoundedMemoryLifecycle({ operation: `${candidate.id}: abandoned branch cleanup`, timeoutMs: cleanupTimeoutMs, + resource: adapter, run: () => clearSequenceScopes(memory!, sequence), }) await lease.assertOwned() @@ -1197,6 +1240,7 @@ async function recoverMemoryAttempt(input: { await runBoundedMemoryLifecycle({ operation: `${candidate.id}: recovery adapter close`, timeoutMs: cleanupTimeoutMs, + resource: adapter, run: async () => { if (memory && cleanupOwned) { await memory.close?.() @@ -1217,6 +1261,7 @@ async function recoverMemoryAttempt(input: { await runBoundedMemoryLifecycle({ operation: `${candidate.id}: recovery adapter disposal`, timeoutMs: cleanupTimeoutMs, + resource: adapter, run: () => candidate.disposeAdapter?.(rawAdapter!), }) } catch (error) { @@ -1251,6 +1296,8 @@ function memoryAttemptCostCallId( branchId: attempt.branchId, candidateId: attempt.candidateId, candidateRef: attempt.candidateRef, + externalCostUsdPerSequence: attempt.externalCostUsdPerSequence, + externalRecoveryCostUsdPerAttempt: attempt.externalRecoveryCostUsdPerAttempt, sequenceId: attempt.sequenceId, rep: attempt.rep, seed: attempt.seed, @@ -1283,7 +1330,7 @@ function memoryAttemptEvent(input: { now?: () => Date }): AgentMemoryAttemptEvent { return { - schema: 1, + schema: 2, status: input.status, branchId: input.branchId, candidateId: input.candidate.id, @@ -1292,6 +1339,8 @@ function memoryAttemptEvent(input: { rep: input.rep, seed: input.seed, cleanupBranches: input.cleanupBranches ?? true, + externalCostUsdPerSequence: input.candidate.externalCostUsdPerSequence ?? 0, + externalRecoveryCostUsdPerAttempt: input.candidate.externalRecoveryCostUsdPerAttempt ?? 0, recordedAt: (input.now ?? (() => new Date()))().toISOString(), recovery: input.recovery, } @@ -1333,7 +1382,7 @@ function parseMemoryAttemptEvent( const valid = typeof event === 'object' && event !== null && - event.schema === 1 && + event.schema === 2 && (event.status === 'started' || event.status === 'cleaned') && typeof event.branchId === 'string' && event.branchId.length > 0 && @@ -1346,6 +1395,12 @@ function parseMemoryAttemptEvent( Number.isSafeInteger(event.rep) && Number.isSafeInteger(event.seed) && typeof event.cleanupBranches === 'boolean' && + typeof event.externalCostUsdPerSequence === 'number' && + Number.isFinite(event.externalCostUsdPerSequence) && + event.externalCostUsdPerSequence >= 0 && + typeof event.externalRecoveryCostUsdPerAttempt === 'number' && + Number.isFinite(event.externalRecoveryCostUsdPerAttempt) && + event.externalRecoveryCostUsdPerAttempt >= 0 && typeof event.recordedAt === 'string' && !Number.isNaN(Date.parse(event.recordedAt)) && typeof event.recovery === 'boolean' @@ -1361,10 +1416,33 @@ function sameMemoryAttempt(left: AgentMemoryAttemptEvent, right: AgentMemoryAtte left.sequenceId === right.sequenceId && left.rep === right.rep && left.seed === right.seed && - left.cleanupBranches === right.cleanupBranches + left.cleanupBranches === right.cleanupBranches && + left.externalCostUsdPerSequence === right.externalCostUsdPerSequence && + left.externalRecoveryCostUsdPerAttempt === right.externalRecoveryCostUsdPerAttempt ) } +function assertMemoryAttemptCandidateMatches( + attempt: AgentMemoryAttemptEvent, + candidate: AgentMemoryExperimentCandidate, +): void { + if (candidate.ref !== attempt.candidateRef) { + throw new Error( + `cannot recover memory branch '${attempt.branchId}': candidate ref changed from '${attempt.candidateRef}' to '${candidate.ref}'`, + ) + } + const executionCost = candidate.externalCostUsdPerSequence ?? 0 + const recoveryCost = candidate.externalRecoveryCostUsdPerAttempt ?? 0 + if ( + executionCost !== attempt.externalCostUsdPerSequence || + recoveryCost !== attempt.externalRecoveryCostUsdPerAttempt + ) { + throw new Error( + `cannot recover memory branch '${attempt.branchId}': candidate cost settings changed; start a new run or restore the recorded costs`, + ) + } +} + async function writeStep(memory: AgentMemoryBranch, step: AgentMemorySequenceStep): Promise { const writes = (step.writes ?? []).map((write) => ({ ...write, @@ -1443,7 +1521,7 @@ function rankAgentMemoryExperiment( candidates: readonly AgentMemoryExperimentCandidate[], scenarios: readonly AgentMemorySequenceScenario[], campaign: CampaignResult, - recoveryCostByCandidate: ReadonlyMap, + costByCandidate: ReadonlyMap, ): AgentMemoryExperimentRankingRow[] { const scenarioById = new Map(scenarios.map((scenario) => [scenario.id, scenario])) const rows = candidates.map((candidate): AgentMemoryExperimentRankingRow => { @@ -1465,9 +1543,7 @@ function rankAgentMemoryExperiment( totalCells: cells.length, totalProbes: successful.reduce((sum, cell) => sum + cell.artifact.probes.length, 0), cellsFailed: cells.filter((cell) => Boolean(cell.error)).length, - totalCostUsd: - cells.reduce((sum, cell) => sum + cell.costUsd, 0) + - (recoveryCostByCandidate.get(candidate.id) ?? 0), + totalCostUsd: normalizeUsd(costByCandidate.get(candidate.id) ?? 0), durationMs: cells.reduce((sum, cell) => sum + cell.durationMs, 0), dimensions: meanDimensions(dimensionRows), } @@ -1484,12 +1560,44 @@ function rankAgentMemoryExperiment( .map((row, index) => ({ ...row, rank: index + 1 })) } +function memoryExperimentCostByCandidate( + costLedger: CostLedgerHandle, + runDir: string, + scenarios: readonly AgentMemorySequenceScenario[], + candidateIdsInput: readonly string[], +): ReadonlyMap { + const candidateByScenario = new Map( + scenarios.map((scenario) => [scenario.id, scenario.candidateId]), + ) + const candidateIds = new Set(candidateIdsInput) + const totals = new Map() + for (const receipt of costLedger.list()) { + if (receipt.tags?.runDir !== runDir) continue + const scenarioCandidate = receipt.tags.scenarioId + ? candidateByScenario.get(receipt.tags.scenarioId) + : undefined + const recoveryCandidate = + receipt.tags.memoryRecovery === 'attempt' && receipt.tags.candidateId + ? receipt.tags.candidateId + : undefined + const candidateId = scenarioCandidate ?? recoveryCandidate + if (!candidateId || !candidateIds.has(candidateId)) continue + totals.set(candidateId, (totals.get(candidateId) ?? 0) + receipt.costUsd) + } + return totals +} + function renderAgentMemoryExperimentRanking( rows: readonly AgentMemoryExperimentRankingRow[], + totalCostUsd: number, + unrankedRecoveryCostUsd: number, ): string { return [ '# Agent Memory Experiment', '', + `- total cost: $${format(totalCostUsd)}`, + `- retired-candidate recovery cost: $${format(unrankedRecoveryCostUsd)}`, + '', '| rank | candidate | sequences | cells | probes | failed | score | pass rate | cost | duration ms |', '| ---: | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |', ...rows.map( @@ -1738,3 +1846,7 @@ function assertNonEmptyString(value: unknown, label: string): asserts value is s function format(value: number): string { return Number.isFinite(value) ? value.toFixed(4) : '0.0000' } + +function normalizeUsd(value: number): number { + return Number(value.toFixed(12)) +} diff --git a/src/memory/improvement.ts b/src/memory/improvement.ts index 346edb1..1e6029c 100644 --- a/src/memory/improvement.ts +++ b/src/memory/improvement.ts @@ -171,6 +171,7 @@ export interface RunAgentMemoryImprovementOptions { dispatchTimeoutMs?: number cleanupTimeoutMs?: number maxRecoveryAttempts?: number + maxRecoveryRetriesPerAttempt?: number maxTotalCostUsd?: number executeStep?: RunAgentMemoryExperimentOptions['executeStep'] executeStepRef?: string @@ -212,6 +213,7 @@ const DEFAULT_CRITICAL_DIMENSIONS = [ 'memory_actor_recall', 'memory_event_recall', ] as const +const MEMORY_IMPROVEMENT_IMPLEMENTATION_REF = 'agent-knowledge:memory-improvement:v2' /** Searches branchable memory configurations and activates only a fresh holdout win. */ export async function runAgentMemoryImprovement( @@ -682,6 +684,7 @@ function experimentOptions( | 'dispatchTimeoutMs' | 'cleanupTimeoutMs' | 'maxRecoveryAttempts' + | 'maxRecoveryRetriesPerAttempt' | 'executeStep' | 'executeStepRef' | 'onBranchSnapshot' @@ -699,6 +702,7 @@ function experimentOptions( dispatchTimeoutMs: options.dispatchTimeoutMs, cleanupTimeoutMs: options.cleanupTimeoutMs, maxRecoveryAttempts: options.maxRecoveryAttempts, + maxRecoveryRetriesPerAttempt: options.maxRecoveryRetriesPerAttempt, executeStep: options.executeStep, executeStepRef: options.executeStepRef, onBranchSnapshot: options.onBranchSnapshot, @@ -722,7 +726,8 @@ function assertMemoryImprovementIdentity( ): void { const path = join(runDir, 'memory-improvement-manifest.json') const identity = { - schema: 5, + schema: 6, + implementationRef: MEMORY_IMPROVEMENT_IMPLEMENTATION_REF, experimentId: options.experimentId, improvementRef: options.improvementRef, activationRef: options.activation?.ref ?? null, @@ -1172,6 +1177,7 @@ function assertMemoryImprovementOptions( ['sequenceConcurrency', options.sequenceConcurrency], ['reps', options.reps], ['maxRecoveryAttempts', options.maxRecoveryAttempts], + ['maxRecoveryRetriesPerAttempt', options.maxRecoveryRetriesPerAttempt], ] as const) { if (value !== undefined && (!Number.isSafeInteger(value) || value <= 0)) { throw new Error(`memory improvement ${name} must be a positive safe integer`) diff --git a/src/memory/index.ts b/src/memory/index.ts index 7bef15c..6820790 100644 --- a/src/memory/index.ts +++ b/src/memory/index.ts @@ -4,7 +4,16 @@ export * from './experiment' export * from './graphiti' export * from './holdout' export * from './improvement' -export * from './lifecycle' +export { + AgentMemoryLifecycleTimeoutError, + AgentMemoryLifecycleUnsafeError, + createMemoryExecutionPool, + DEFAULT_MEMORY_CLEANUP_TIMEOUT_MS, + memoryRecoveryDelayMs, + resolveMemoryCleanupTimeoutMs, + runBoundedMemoryLifecycle, + sleepForMemoryRecovery, +} from './lifecycle' export * from './mem0' export * from './neo4j' export * from './run-control' diff --git a/src/memory/lifecycle.ts b/src/memory/lifecycle.ts index 4f907a1..1a8ab5e 100644 --- a/src/memory/lifecycle.ts +++ b/src/memory/lifecycle.ts @@ -12,6 +12,40 @@ export class AgentMemoryLifecycleTimeoutError extends Error { } } +export class AgentMemoryLifecycleUnsafeError extends Error { + constructor( + readonly operation: string, + readonly priorTimeout: AgentMemoryLifecycleTimeoutError, + ) { + super( + `${operation} cannot start because '${priorTimeout.operation}' is still running after its ${priorTimeout.timeoutMs}ms timeout`, + { cause: priorTimeout }, + ) + this.name = 'AgentMemoryLifecycleUnsafeError' + } +} + +const timedOutResources = new WeakMap() + +export function releaseMemoryAdapterCreatedAfterAbort(input: { + creation: Promise + signal: AbortSignal + dispose?: (adapter: AgentMemoryAdapter) => Promise +}): void { + void input.creation.then( + async (adapter) => { + if (!adapter || !input.signal.aborted) return + try { + await adapter.close?.() + } catch {} + try { + await input.dispose?.(adapter) + } catch {} + }, + () => undefined, + ) +} + export function resolveMemoryCleanupTimeoutMs(value: number | undefined, label: string): number { const timeoutMs = value ?? DEFAULT_MEMORY_CLEANUP_TIMEOUT_MS if (!Number.isSafeInteger(timeoutMs) || timeoutMs <= 0) { @@ -23,15 +57,39 @@ export function resolveMemoryCleanupTimeoutMs(value: number | undefined, label: export async function runBoundedMemoryLifecycle(input: { operation: string timeoutMs: number + /** Prevent later operations from using the same client while timed-out work may still be running. */ + resource?: object + /** Cooperatively cancel provider work before reporting a timeout. */ + abortController?: AbortController run(): Promise | T }): Promise { + const priorTimeout = input.resource ? timedOutResources.get(input.resource) : undefined + if (priorTimeout) throw new AgentMemoryLifecycleUnsafeError(input.operation, priorTimeout) let timeout: ReturnType | undefined + let timeoutError: AgentMemoryLifecycleTimeoutError | undefined const work = Promise.resolve().then(input.run) - const deadline = new Promise((_, reject) => { - timeout = setTimeout( - () => reject(new AgentMemoryLifecycleTimeoutError(input.operation, input.timeoutMs)), - input.timeoutMs, + if (input.resource) { + const resource = input.resource + void work.then( + () => { + if (timeoutError && timedOutResources.get(resource) === timeoutError) { + timedOutResources.delete(resource) + } + }, + () => { + if (timeoutError && timedOutResources.get(resource) === timeoutError) { + timedOutResources.delete(resource) + } + }, ) + } + const deadline = new Promise((_, reject) => { + timeout = setTimeout(() => { + timeoutError = new AgentMemoryLifecycleTimeoutError(input.operation, input.timeoutMs) + if (input.resource) timedOutResources.set(input.resource, timeoutError) + input.abortController?.abort(timeoutError) + reject(timeoutError) + }, input.timeoutMs) }) try { return await Promise.race([work, deadline]) diff --git a/src/memory/mem0.ts b/src/memory/mem0.ts index f6e2124..d390fa6 100644 --- a/src/memory/mem0.ts +++ b/src/memory/mem0.ts @@ -13,7 +13,6 @@ import type { } from './types' export type Mem0ClientMode = 'hosted' | 'oss' -export type Mem0Consistency = 'queued' | 'visible' export interface Mem0ClientLike { add( @@ -33,9 +32,7 @@ export interface Mem0MemoryAdapterOptions { infer?: boolean rerank?: boolean latestOnly?: boolean - consistency?: Mem0Consistency - /** Reads `GET /v1/event/{eventId}/`; required when a hosted add is asynchronous. */ - getEvent?: (eventId: string) => Promise + /** Bounds delayed-delete visibility checks and abandoned hosted-write recovery waits. */ ingestionTimeoutMs?: number pollIntervalMs?: number /** Stable deployment/account identity for cache-key helpers. Never put credentials here. */ @@ -43,33 +40,34 @@ export interface Mem0MemoryAdapterOptions { defaultScope?: AgentMemoryScope } +interface Mem0PendingWriteProbe { + text: string + providerIds: Set + filters: Record + expiresAt: number +} + /** Connects both the hosted Mem0 client and the open-source `Memory` class. */ export function createMem0MemoryAdapter(options: Mem0MemoryAdapterOptions): AgentMemoryAdapter { assertMem0Options(options) const id = options.id ?? `mem0-${options.mode}` - const consistency = options.mode === 'oss' ? 'visible' : (options.consistency ?? 'visible') - const pendingWrites = new Map() + const pendingWrites = new Set() const adapter: AgentMemoryAdapter = { id, branchIsolation: options.mode === 'oss' ? { mode: 'scoped' } - : consistency === 'visible' - ? { - mode: 'scoped', - processExitSafe: false, - recoveryDelayMs: options.ingestionTimeoutMs ?? 30_000, - } - : { - mode: 'unsupported', - reason: - 'queued hosted Mem0 writes can become visible after branch cleanup; use consistency="visible" with an attempt branch', - }, + : { + mode: 'scoped', + processExitSafe: false, + recoveryDelayMs: options.ingestionTimeoutMs ?? 30_000, + }, async search(query, searchOptions = {}) { assertMem0SearchOptions(searchOptions, id) if (searchOptions.limit === 0) return [] const scope = mergeScopes(options.defaultScope, searchOptions.scope) + assertMem0ProviderScope(scope, options.mode, id, 'search') const requestedKinds = [...new Set(searchOptions.kinds ?? [])] const kindQueries = requestedKinds.length > 0 ? requestedKinds : [undefined] const payloads = await Promise.all( @@ -95,7 +93,17 @@ export function createMem0MemoryAdapter(options: Mem0MemoryAdapterOptions): Agen }, async write(input) { const scope = mergeScopes(options.defaultScope, input.scope) + assertMem0ProviderScope(scope, options.mode, id, 'write') + pruneExpiredMem0PendingWrites(pendingWrites) const writeId = input.id ?? randomUUID() + const writeFilters = mem0Filters(scope, options.appId) + const pendingProbe: Mem0PendingWriteProbe = { + text: input.text, + providerIds: new Set(), + filters: writeFilters, + expiresAt: Number.POSITIVE_INFINITY, + } + pendingWrites.add(pendingProbe) const scopeMetadata = mem0ScopeMetadata(scope, options.appId) const metadata = compactRecord({ ...input.metadata, @@ -113,42 +121,44 @@ export function createMem0MemoryAdapter(options: Mem0MemoryAdapterOptions): Agen ...scopeMetadata, }) const entity = mem0Entity(scope, options.appId) - const raw = await options.client.add([{ role: mem0Role(input.role), content: input.text }], { - ...entity, - metadata, - infer: options.infer ?? true, - ...(options.mode === 'oss' ? { filters: mem0Filters(scope, options.appId) } : {}), - }) - let items = extractMem0Items(raw) - const pending = options.mode === 'hosted' && mem0ResponsePending(raw) - const eventId = stringField(isRecord(raw) ? raw : undefined, ['eventId', 'event_id']) - if (pending) { - if (!eventId) throw new Error(`${id}: asynchronous Mem0 add returned no event_id`) - pendingWrites.set(eventId, cloneScope(scope)) - if (consistency === 'visible') { - const terminal = await waitForMem0Event(options, eventId) - pendingWrites.delete(eventId) - assertMem0EventSucceeded(terminal, id, eventId) - items = extractMem0Items(terminal) - } - } else if (options.mode === 'hosted' && isRecord(raw) && mem0Status(raw) === 'FAILED') { - assertMem0EventSucceeded(raw, id, eventId ?? 'unknown') + let raw: unknown + try { + raw = await options.client.add([{ role: mem0Role(input.role), content: input.text }], { + ...entity, + metadata, + infer: options.infer ?? true, + ...(options.mode === 'oss' ? { filters: writeFilters } : {}), + }) + } finally { + pendingProbe.expiresAt = Date.now() + (options.ingestionTimeoutMs ?? 30_000) } + if (options.mode === 'hosted' && !Array.isArray(raw)) { + throw new Error( + `${id}: hosted Mem0 add returned an unsupported response; mem0ai 3.x must return a memory array`, + ) + } + const items = extractMem0Items(raw) const mutations = items.filter((candidate) => mem0Event(candidate) !== 'NOOP') + const durableMutations = mutations.filter((candidate) => mem0Event(candidate) !== 'DELETE') + for (const candidate of durableMutations) { + const providerId = stringField(candidate, ['id']) + if (providerId) pendingProbe.providerIds.add(providerId) + } + if (durableMutations.length === 0) { + pendingWrites.delete(pendingProbe) + } const item = mutations[0] ?? items[0] - const itemId = stringField(item, ['id']) ?? eventId ?? writeId + const itemId = stringField(item, ['id']) ?? writeId const event = mem0Event(item) const events = [...new Set(mutations.map(mem0Event).filter(Boolean))] const result = { - accepted: pending && consistency === 'queued' ? true : mutations.length > 0, + accepted: mutations.length > 0, id: itemId, uri: `memory://${id}/${encodeURIComponent(itemId)}`, kind: input.kind, metadata: { provider: 'mem0', mode: options.mode, - consistency, - queued: pending && consistency === 'queued', ...(event ? { event } : {}), ...(events.length > 1 ? { events } : {}), }, @@ -160,14 +170,32 @@ export function createMem0MemoryAdapter(options: Mem0MemoryAdapterOptions): Agen }, async clear(scope) { const mergedScope = mergeScopes(options.defaultScope, scope) + assertMem0ProviderScope(mergedScope, options.mode, id, 'clear') + pruneExpiredMem0PendingWrites(pendingWrites) const clearFilters = mem0Filters(mergedScope, options.appId) if (Object.keys(clearFilters).length === 0) { throw new Error(`${id}: refusing an unscoped Mem0 clear`) } - await settlePendingMem0Writes(options, pendingWrites, mergedScope, false) if (options.client.getAll && options.client.delete) { + const pendingForClear = [...pendingWrites].filter((pending) => + mem0FiltersInclude(pending.filters, clearFilters), + ) const deletedIds = new Set() - const deletedSearchProbes = new Map>() + const searchProbes = new Map>() + const observedPending = new Set() + for (const pending of pendingForClear) { + const ids = searchProbes.get(pending.text) ?? new Set() + for (const providerId of pending.providerIds) ids.add(providerId) + searchProbes.set(pending.text, ids) + if (pending.providerIds.size > 0) observedPending.add(pending) + } + for (const pending of pendingForClear) { + for (const providerId of pending.providerIds) { + if (deletedIds.has(providerId)) continue + await options.client.delete(providerId) + deletedIds.add(providerId) + } + } const visibilityTimeoutMs = options.ingestionTimeoutMs ?? 30_000 const pollIntervalMs = options.pollIntervalMs ?? 250 let visibilityDeadline = Date.now() + visibilityTimeoutMs @@ -189,22 +217,61 @@ export function createMem0MemoryAdapter(options: Mem0MemoryAdapterOptions): Agen ) const items = extractMem0Items(raw) if (items.length === 0) { - if ( - !(await deletedMem0IdsRemainSearchable( - options, - clearFilters, - deletedIds, - deletedSearchProbes, - )) - ) { - return + const searchableByQuery = await searchableMem0IdsForCleanup( + options, + clearFilters, + searchProbes, + ) + const searchableIds = new Set( + [...searchableByQuery.values()].flatMap((ids) => [...ids]), + ) + const unseenSearchIds = [...searchableIds].filter( + (memoryId) => !deletedIds.has(memoryId), + ) + for (const [query, ids] of searchableByQuery) { + const known = searchProbes.get(query) ?? new Set() + for (const memoryId of ids) known.add(memoryId) + searchProbes.set(query, known) + if (ids.size > 0) { + for (const pending of pendingForClear) { + if (pending.text === query) observedPending.add(pending) + } + } } + if (unseenSearchIds.length > 0) { + for (const memoryId of unseenSearchIds) { + await options.client.delete(memoryId) + deletedIds.add(memoryId) + } + batch += 1 + visibilityDeadline = Date.now() + visibilityTimeoutMs + continue + } + const unresolvedPending = pendingForClear.some( + (pending) => !observedPending.has(pending), + ) + const needsQuietWindow = pendingForClear.some( + (pending) => pending.providerIds.size === 0, + ) const remainingMs = visibilityDeadline - Date.now() if (remainingMs <= 0) { + if (!unresolvedPending && searchableIds.size === 0) { + removeMem0PendingWrites(pendingWrites, pendingForClear) + return + } + if (searchableIds.size > 0) { + throw new Error( + `${id}: deleted Mem0 memories remained searchable after ${visibilityTimeoutMs}ms`, + ) + } throw new Error( - `${id}: deleted Mem0 memories remained searchable after ${visibilityTimeoutMs}ms`, + `${id}: a Mem0 write never became visible for exact cleanup within ${visibilityTimeoutMs}ms`, ) } + if (!unresolvedPending && searchableIds.size === 0 && !needsQuietWindow) { + removeMem0PendingWrites(pendingWrites, pendingForClear) + return + } await sleep(Math.min(pollIntervalMs, remainingMs)) continue } @@ -219,9 +286,14 @@ export function createMem0MemoryAdapter(options: Mem0MemoryAdapterOptions): Agen stringField(item, ['memory', 'text', 'content']) ?? stringField(recordField(item, 'data'), ['memory', 'text', 'content']) if (!text) continue - const idsForText = deletedSearchProbes.get(text) ?? new Set() + const idsForText = searchProbes.get(text) ?? new Set() idsForText.add(memoryId) - deletedSearchProbes.set(text, idsForText) + searchProbes.set(text, idsForText) + for (const pending of pendingForClear) { + if (pending.text === text || pending.providerIds.has(memoryId)) { + observedPending.add(pending) + } + } } const unseenIds = ids.filter((memoryId) => !deletedIds.has(memoryId)) if (unseenIds.length === 0) { @@ -247,9 +319,6 @@ export function createMem0MemoryAdapter(options: Mem0MemoryAdapterOptions): Agen } throw new Error(`${id}: exact scoped clear requires Mem0 getAll plus per-memory delete`) }, - async flush() { - await settlePendingMem0Writes(options, pendingWrites, undefined, true) - }, } return adapter } @@ -267,9 +336,6 @@ function assertMem0Options(options: Mem0MemoryAdapterOptions): void { ) { throw new Error('Mem0 backendRef must be a non-empty string') } - if (options.mode === 'oss' && options.consistency === 'queued') { - throw new Error('Mem0 OSS writes are synchronous and do not support consistency="queued"') - } for (const [name, value] of [ ['ingestionTimeoutMs', options.ingestionTimeoutMs], ['pollIntervalMs', options.pollIntervalMs], @@ -289,6 +355,23 @@ function assertMem0SearchOptions(options: AgentMemorySearchOptions, adapterId: s } } +function assertMem0ProviderScope( + scope: AgentMemoryScope, + mode: Mem0ClientMode, + adapterId: string, + operation: 'search' | 'write' | 'clear', +): void { + if (scope.userId || scope.agentId || scope.runId || scope.namespace) return + if (operation === 'clear') { + throw new Error( + `${adapterId}: refusing an unscoped Mem0 clear; provide scope.userId, scope.agentId, scope.runId, or scope.namespace`, + ) + } + throw new Error( + `${adapterId}: Mem0 ${mode} ${operation} requires scope.userId, scope.agentId, scope.runId, or scope.namespace`, + ) +} + function normalizeMem0Hits( raw: unknown, adapterId: string, @@ -345,7 +428,7 @@ function normalizeMem0Hits( function mergeMem0Hits(hits: readonly AgentMemoryHit[], limit?: number): AgentMemoryHit[] { const unique = new Map() for (const [index, hit] of hits.entries()) { - const key = `${hit.uri}\n${hit.text}` + const key = JSON.stringify([hit.uri, hit.text]) const score = hit.normalizedScore ?? hit.score const finiteScore = score !== undefined && Number.isFinite(score) ? score : undefined const prior = unique.get(key) @@ -379,88 +462,9 @@ function extractMem0Items(raw: unknown): Array> { } function mem0Event(item: Record | undefined): string | undefined { - return stringField(item, ['event'])?.toUpperCase() -} - -function mem0ResponsePending(raw: unknown): boolean { - if (!isRecord(raw)) return false - const status = stringField(raw, ['status'])?.toUpperCase() - return status === 'PENDING' || status === 'RUNNING' || status === 'QUEUED' -} - -async function waitForMem0Event( - options: Mem0MemoryAdapterOptions, - eventId: string, -): Promise> { - if (!options.getEvent) { - throw new Error( - `${options.id ?? 'mem0-hosted'}: hosted asynchronous writes require getEvent(eventId) for GET /v1/event/{eventId}/`, - ) - } - const timeoutMs = options.ingestionTimeoutMs ?? 30_000 - const pollIntervalMs = options.pollIntervalMs ?? 250 - const deadline = Date.now() + timeoutMs - for (;;) { - const raw = await options.getEvent(eventId) - if (!isRecord(raw)) throw new Error(`${options.id ?? 'mem0-hosted'}: invalid event response`) - const status = mem0Status(raw) - if (status === 'SUCCEEDED' || status === 'FAILED') return raw - if (status !== 'PENDING' && status !== 'RUNNING' && status !== 'QUEUED') { - throw new Error( - `${options.id ?? 'mem0-hosted'}: event ${eventId} returned unsupported status ${status ?? 'missing'}`, - ) - } - const remaining = deadline - Date.now() - if (remaining <= 0) { - throw new Error( - `${options.id ?? 'mem0-hosted'}: event ${eventId} did not finish within ${timeoutMs}ms`, - ) - } - await sleep(Math.min(pollIntervalMs, remaining)) - } -} - -function assertMem0EventSucceeded( - event: Record, - adapterId: string, - eventId: string, -): void { - if (mem0Status(event) === 'SUCCEEDED') return - const detail = - stringField(event, ['message', 'error', 'errorMessage', 'error_message']) ?? - stringField(recordField(event, 'metadata'), ['message', 'error']) - throw new Error(`${adapterId}: Mem0 event ${eventId} failed${detail ? `: ${detail}` : ''}`) -} - -function mem0Status(raw: Record): string | undefined { - return stringField(raw, ['status'])?.toUpperCase() -} - -async function settlePendingMem0Writes( - options: Mem0MemoryAdapterOptions, - pendingWrites: Map, - requestedScope?: AgentMemoryScope, - failOnProviderFailure = true, -): Promise { - const pending = [...pendingWrites].filter( - ([, scope]) => !requestedScope || scopeMatches(scope, requestedScope), - ) - const settled = await Promise.allSettled( - pending.map(async ([eventId]) => { - const terminal = await waitForMem0Event(options, eventId) - pendingWrites.delete(eventId) - if (failOnProviderFailure) { - assertMem0EventSucceeded(terminal, options.id ?? 'mem0-hosted', eventId) - } - }), - ) - const failures = settled.flatMap((result) => - result.status === 'rejected' ? [result.reason] : [], - ) - if (failures.length === 1) throw failures[0] - if (failures.length > 1) { - throw new AggregateError(failures, `${options.id ?? 'mem0-hosted'}: pending writes failed`) - } + return ( + stringField(item, ['event']) ?? stringField(recordField(item, 'metadata'), ['event']) + )?.toUpperCase() } function mem0Entity(scope: AgentMemoryScope, appId?: string): Record { @@ -511,12 +515,12 @@ function mem0CustomScopeMetadata(scope: AgentMemoryScope): Record, - deletedIds: ReadonlySet, probes: ReadonlyMap>, -): Promise { +): Promise>> { + const searchable = new Map>() const entries = [...probes] for (let offset = 0; offset < entries.length; offset += 8) { const results = await Promise.all( @@ -537,9 +541,33 @@ async function deletedMem0IdsRemainSearchable( }) }), ) - if (results.some((ids) => ids.some((memoryId) => deletedIds.has(memoryId)))) return true + for (const [index, ids] of results.entries()) { + const query = entries[offset + index]![0] + searchable.set(query, new Set(ids)) + } + } + return searchable +} + +function mem0FiltersInclude( + candidate: Readonly>, + requested: Readonly>, +): boolean { + return Object.entries(requested).every(([key, value]) => Object.is(candidate[key], value)) +} + +function removeMem0PendingWrites( + pendingWrites: Set, + removed: readonly Mem0PendingWriteProbe[], +): void { + for (const pending of removed) pendingWrites.delete(pending) +} + +function pruneExpiredMem0PendingWrites(pendingWrites: Set): void { + const now = Date.now() + for (const pending of pendingWrites) { + if (pending.expiresAt <= now) pendingWrites.delete(pending) } - return false } function mem0Role(role: AgentMemoryWriteInput['role']): 'user' | 'assistant' { @@ -568,28 +596,6 @@ function mergeScopes(base?: AgentMemoryScope, extra?: AgentMemoryScope): AgentMe } } -function cloneScope(scope: AgentMemoryScope): AgentMemoryScope { - return { ...scope, tags: { ...(scope.tags ?? {}) } } -} - -function scopeMatches(actual: AgentMemoryScope, requested: AgentMemoryScope): boolean { - for (const key of [ - 'tenantId', - 'userId', - 'agentId', - 'teamId', - 'runId', - 'sessionId', - 'namespace', - ] as const) { - if (requested[key] !== undefined && actual[key] !== requested[key]) return false - } - for (const [key, value] of Object.entries(requested.tags ?? {})) { - if (actual.tags?.[key] !== value) return false - } - return true -} - function sleep(ms: number): Promise { return new Promise((resolve) => setTimeout(resolve, ms)) } @@ -660,7 +666,6 @@ export function mem0MemoryAdapterIdentity( | 'infer' | 'rerank' | 'latestOnly' - | 'consistency' | 'ingestionTimeoutMs' | 'pollIntervalMs' | 'defaultScope' diff --git a/src/memory/neo4j.ts b/src/memory/neo4j.ts index a8c91b7..cb3b40a 100644 --- a/src/memory/neo4j.ts +++ b/src/memory/neo4j.ts @@ -30,6 +30,7 @@ export function createNeo4jAgentMemoryAdapter( assertNeo4jOptions(options) const client = options.client as Record const id = options.id ?? 'neo4j-agent-memory' + const isolatedInstance = options.branchId !== undefined const adapter: AgentMemoryAdapter = { id, branchIsolation: options.branchId @@ -40,7 +41,14 @@ export function createNeo4jAgentMemoryAdapter( }, async search(query, searchOptions = {}) { assertNeo4jSearchOptions(searchOptions, id) - return searchNeo4jMemory(client, query, searchOptions, id, options.transport) + return searchNeo4jMemory( + client, + query, + searchOptions, + id, + options.transport, + isolatedInstance, + ) }, async getContext(query, searchOptions = {}) { assertNeo4jSearchOptions(searchOptions, id) @@ -62,6 +70,13 @@ export function createNeo4jAgentMemoryAdapter( ) } if (options.contextMode === 'native' && options.transport === 'rest' && sessionId) { + assertNeo4jScopeSupported( + searchOptions.scope, + ['sessionId'], + id, + 'native context read', + isolatedInstance, + ) const conversationContext = await callRequired(shortTerm, ['getContext'], [sessionId]) const hits = normalizeConversationContextHits(conversationContext, searchOptions, id) const text = renderHits(hits) @@ -87,7 +102,7 @@ export function createNeo4jAgentMemoryAdapter( return defaultGetMemoryContext(adapter, query, searchOptions) }, async write(input) { - const result = await writeNeo4jMemory(client, input, id, options.transport) + const result = await writeNeo4jMemory(client, input, id, options.transport, isolatedInstance) return { ...result, sourceRecord: memoryWriteResultToSourceRecord(result, input.text, { scope: input.scope }), @@ -162,9 +177,31 @@ async function writeNeo4jMemory( input: AgentMemoryWriteInput, adapterId: string, transport: 'rest' | 'bridge', + isolatedInstance: boolean, ): Promise { const scope = input.scope ?? {} const sessionId = scope.sessionId ?? input.metadata?.sessionId + assertNeo4jScopeSupported( + input.scope, + input.kind === 'message' || input.kind === 'observation' + ? ['sessionId'] + : input.kind === 'reasoning-trace' + ? ['sessionId', 'runId'] + : [], + adapterId, + `${input.kind} write`, + isolatedInstance, + ) + if ( + !isolatedInstance && + input.kind === 'reasoning-trace' && + scope.sessionId !== undefined && + scope.runId !== undefined + ) { + throw new Error( + `${adapterId}: Neo4j reasoning write cannot enforce both sessionId and runId; provide one`, + ) + } let result: unknown if (input.kind === 'message' || input.kind === 'observation') { if (transport === 'rest' && typeof sessionId !== 'string') { @@ -337,10 +374,22 @@ async function searchNeo4jMemory( options: AgentMemorySearchOptions, adapterId: string, transport: 'rest' | 'bridge', + isolatedInstance: boolean, ): Promise { const limit = options.limit ?? 10 if (limit === 0) return [] assertNeo4jSearchKinds(options.kinds, transport, adapterId) + const onlyConversationKinds = + options.kinds !== undefined && + options.kinds.length > 0 && + options.kinds.every((kind) => kind === 'message' || kind === 'observation') + assertNeo4jScopeSupported( + options.scope, + onlyConversationKinds ? ['sessionId'] : [], + adapterId, + 'search', + isolatedInstance, + ) const searches: Promise[] = [] const kinds = options.kinds const includeKind = (kind: AgentMemoryHit['kind']) => !kinds?.length || kinds.includes(kind) @@ -443,6 +492,33 @@ function similarTracesOptions(options: AgentMemorySearchOptions): Record(allowedKeys) + const scopeKeys: readonly (keyof AgentMemoryScope)[] = [ + 'tenantId', + 'userId', + 'agentId', + 'teamId', + 'runId', + 'sessionId', + 'namespace', + ] + const supplied = scopeKeys.filter((key) => scope[key] !== undefined) + if (scope.tags && Object.keys(scope.tags).length > 0) supplied.push('tags') + const unsupported = supplied.filter((key) => !allowed.has(key)) + if (unsupported.length === 0) return + throw new Error( + `${adapterId}: Neo4j ${operation} cannot enforce scope fields: ${unsupported.join(', ')}; create a dedicated client and pass branchId`, + ) +} + function neo4jEntityOptions(input: AgentMemoryWriteInput): Record { return { description: input.metadata?.description ?? input.title, diff --git a/src/memory/rank.ts b/src/memory/rank.ts index d1c6859..ef3ab9e 100644 --- a/src/memory/rank.ts +++ b/src/memory/rank.ts @@ -19,7 +19,7 @@ export function mergeRankedMemoryHits( let encounter = 0 for (const [groupIndex, group] of groups.entries()) { for (const [rank, hit] of group.entries()) { - const key = `${hit.uri}\n${hit.text}` + const key = JSON.stringify([hit.uri, hit.text]) const score = memoryHitScore(hit) const prior = byIdentity.get(key) if (!prior) { diff --git a/tests/benchmarks.test.ts b/tests/benchmarks.test.ts index dd4ca63..e0a7944 100644 --- a/tests/benchmarks.test.ts +++ b/tests/benchmarks.test.ts @@ -383,6 +383,7 @@ describe('knowledge benchmark adapters', () => { const candidate = { id: 'recoverable', ref: 'recoverable:v1', + adapterId: 'recoverable-provider', recoveryCostUsdPerAttempt: 0.1, createAdapter({ purpose }: { purpose: 'execute' | 'recovery' }) { purposes.push(purpose) @@ -453,6 +454,229 @@ describe('knowledge benchmark adapters', () => { ]) }) + it('includes paid cleanup for a retired candidate in the benchmark total', async () => { + const storage = inMemoryCampaignStorage() + const runDir = '/runs/retired-candidate-recovery-cost' + storage.write( + `${runDir}/memory-adapter-attempts.jsonl`, + `${JSON.stringify({ + schema: 3, + status: 'started', + attemptId: 'retired-attempt', + candidateId: 'retired', + candidateRef: 'retired:v1', + adapterId: 'retired-provider', + caseId: 'old-case', + cellId: 'old-cell', + scope: { namespace: 'retired-scope' }, + adapterCreationCostUsd: 0.05, + costUsdPerCase: 0, + recoveryCostUsdPerAttempt: 0.1, + recordedAt: '2026-01-01T00:00:00.000Z', + recovery: false, + })}\n`, + ) + let retiredClears = 0 + const result = await runMemoryAdapterBenchmark({ + cases: buildFirstPartyMemoryLifecycleBenchmarkCases().slice(0, 1), + runDir, + storage, + costCeiling: 1, + candidates: [ + { + id: 'active', + ref: 'active:v1', + adapterId: 'active-provider', + createAdapter: () => createInMemoryBenchmarkAdapter({ id: 'active-provider' }), + }, + ], + recoveryCandidates: [ + { + id: 'retired', + ref: 'retired:v1', + adapterId: 'retired-provider', + adapterCreationCostUsd: 0.05, + recoveryCostUsdPerAttempt: 0.1, + createAdapter({ markExternalCall }) { + markExternalCall() + const adapter = createInMemoryBenchmarkAdapter({ id: 'retired-provider' }) + adapter.clear = async () => { + retiredClears += 1 + } + return adapter + }, + }, + ], + }) + + expect(retiredClears).toBe(1) + expect(result.rows[0]).toMatchObject({ candidateId: 'active', totalCostUsd: 0 }) + expect(result).toMatchObject({ totalCostUsd: 0.15, unrankedRecoveryCostUsd: 0.15 }) + expect(storage.read(result.rankingJsonPath)).toContain('"unrankedRecoveryCostUsd": 0.15') + }) + + it('refuses benchmark recovery when candidate cost settings changed', async () => { + const storage = inMemoryCampaignStorage() + const runDir = '/runs/benchmark-changed-recovery-costs' + storage.write( + `${runDir}/memory-adapter-attempts.jsonl`, + `${JSON.stringify({ + schema: 3, + status: 'started', + attemptId: 'unfinished-attempt', + candidateId: 'memory', + candidateRef: 'memory:v1', + adapterId: 'memory-provider', + caseId: 'old-case', + cellId: 'old-cell', + scope: { namespace: 'unfinished-scope' }, + adapterCreationCostUsd: 0, + costUsdPerCase: 0, + recoveryCostUsdPerAttempt: 0, + recordedAt: '2026-01-01T00:00:00.000Z', + recovery: false, + })}\n`, + ) + let adapterCreates = 0 + + await expect( + runMemoryAdapterBenchmark({ + cases: buildFirstPartyMemoryLifecycleBenchmarkCases().slice(0, 1), + runDir, + storage, + costCeiling: 1, + candidates: [ + { + id: 'memory', + ref: 'memory:v1', + adapterId: 'memory-provider', + costUsdPerCase: 0.1, + createAdapter() { + adapterCreates += 1 + return createInMemoryBenchmarkAdapter({ id: 'memory-provider' }) + }, + }, + ], + }), + ).rejects.toThrow('candidate cost settings changed') + + expect(adapterCreates).toBe(0) + expect( + storage.read(`${runDir}/memory-adapter-attempts.jsonl`)?.trim().split('\n'), + ).toHaveLength(1) + }) + + it('bounds repeated direct benchmark recovery across process restarts', async () => { + const storage = inMemoryCampaignStorage() + const runDir = '/runs/benchmark-recovery-retry-limit' + storage.write( + `${runDir}/memory-adapter-attempts.jsonl`, + `${JSON.stringify({ + schema: 3, + status: 'started', + attemptId: 'unfinished-attempt', + candidateId: 'memory', + candidateRef: 'memory:v1', + adapterId: 'memory-provider', + caseId: 'old-case', + cellId: 'old-cell', + scope: { namespace: 'unfinished-scope' }, + adapterCreationCostUsd: 0, + costUsdPerCase: 0, + recoveryCostUsdPerAttempt: 0, + recordedAt: '2026-01-01T00:00:00.000Z', + recovery: false, + })}\n`, + ) + let recoveryCreates = 0 + const run = () => + runMemoryAdapterBenchmark({ + cases: buildFirstPartyMemoryLifecycleBenchmarkCases().slice(0, 1), + runDir, + storage, + maxRecoveryRetriesPerAttempt: 2, + candidates: [ + { + id: 'memory', + ref: 'memory:v1', + adapterId: 'memory-provider', + createAdapter({ purpose }) { + if (purpose === 'recovery') recoveryCreates += 1 + throw new Error('provider recovery unavailable') + }, + }, + ], + }) + + await expect(run()).rejects.toThrow('provider recovery unavailable') + await expect(run()).rejects.toThrow('provider recovery unavailable') + await expect(run()).rejects.toThrow('exhausted 2 recovery attempts') + expect(recoveryCreates).toBe(2) + expect( + storage.read(`${runDir}/memory-adapter-recovery-attempts.jsonl`)?.trim().split('\n'), + ).toHaveLength(2) + }) + + it('closes a recovery adapter that arrives after its factory timeout', async () => { + const storage = inMemoryCampaignStorage() + const runDir = '/runs/late-benchmark-recovery-adapter' + storage.write( + `${runDir}/memory-adapter-attempts.jsonl`, + `${JSON.stringify({ + schema: 3, + status: 'started', + attemptId: 'unfinished-attempt', + candidateId: 'memory', + candidateRef: 'memory:v1', + adapterId: 'memory-provider', + caseId: 'old-case', + cellId: 'old-cell', + scope: { namespace: 'unfinished-scope' }, + adapterCreationCostUsd: 0, + costUsdPerCase: 0, + recoveryCostUsdPerAttempt: 0, + recordedAt: '2026-01-01T00:00:00.000Z', + recovery: false, + })}\n`, + ) + let resolveCreation!: (adapter: AgentMemoryAdapter) => void + const creation = new Promise((resolve) => { + resolveCreation = resolve + }) + let reportClosed!: () => void + const closed = new Promise((resolve) => { + reportClosed = resolve + }) + let closeCalls = 0 + const lateAdapter = createInMemoryBenchmarkAdapter({ id: 'memory-provider' }) + lateAdapter.close = async () => { + closeCalls += 1 + reportClosed() + } + + await expect( + runMemoryAdapterBenchmark({ + cases: buildFirstPartyMemoryLifecycleBenchmarkCases().slice(0, 1), + runDir, + storage, + cleanupTimeoutMs: 10, + candidates: [ + { + id: 'memory', + ref: 'memory:v1', + adapterId: 'memory-provider', + createAdapter: ({ purpose }) => + purpose === 'recovery' ? creation : createInMemoryBenchmarkAdapter(), + }, + ], + }), + ).rejects.toThrow('benchmark recovery adapter creation did not finish within 10ms') + + resolveCreation(lateAdapter) + await closed + expect(closeCalls).toBe(1) + }) + it('reconciles a crash after direct recovery but before its cost receipt', async () => { const storage = inMemoryCampaignStorage() const append = storage.append!.bind(storage) @@ -475,6 +699,7 @@ describe('knowledge benchmark adapters', () => { const candidate = { id: 'receipt-crash', ref: 'receipt-crash:v1', + adapterId: 'receipt-crash-provider', costUsdPerCase: 0.1, recoveryCostUsdPerAttempt: 0.1, createAdapter({ purpose }: { purpose: 'execute' | 'recovery' }) { @@ -507,7 +732,7 @@ describe('knowledge benchmark adapters', () => { .read('/runs/direct-recovery-receipt-crash/memory-adapter-attempts.jsonl') ?.trim() .split('\n'), - ).toHaveLength(1) + ).toHaveLength(2) const result = await run() const costLedger = createRunCostLedger({ @@ -516,19 +741,25 @@ describe('knowledge benchmark adapters', () => { costCeilingUsd: 1, }) - expect(result.rows[0]).toMatchObject({ candidateId: 'receipt-crash', cellsFailed: 0 }) - expect(recoveryClears).toBe(2) - expect(costLedger.summary()).toMatchObject({ - unresolvedCalls: 0, - totalCostUsd: 0.4, - accountingComplete: true, + expect(result.rows[0]).toMatchObject({ + candidateId: 'receipt-crash', + cellsFailed: 0, }) + expect(result.rows[0]?.totalCostUsd).toBeCloseTo(0.3) + expect(result.totalCostUsd).toBeCloseTo(0.3) + expect(recoveryClears).toBe(1) + expect(costLedger.summary()).toMatchObject({ unresolvedCalls: 0, accountingComplete: true }) + expect(costLedger.summary().totalCostUsd).toBeCloseTo(0.3) }) it('bounds direct benchmark cleanup and preserves its recovery record', async () => { const storage = inMemoryCampaignStorage() const adapter = createInMemoryBenchmarkAdapter({ id: 'hung-cleanup' }) + let closeCalls = 0 adapter.clear = () => new Promise(() => {}) + adapter.close = async () => { + closeCalls += 1 + } const startedAt = Date.now() await expect( @@ -545,7 +776,7 @@ describe('knowledge benchmark adapters', () => { }, ], }), - ).rejects.toThrow('memory benchmark attempt cleanup failed') + ).rejects.toThrow('memory adapter benchmark cleanup failed') expect(Date.now() - startedAt).toBeLessThan(500) expect( @@ -554,6 +785,186 @@ describe('knowledge benchmark adapters', () => { ?.trim() .split('\n'), ).toHaveLength(1) + expect(closeCalls).toBe(0) + }) + + it('counts billable adapter creation in the shared benchmark budget', async () => { + const storage = inMemoryCampaignStorage() + let creates = 0 + const result = await runMemoryAdapterBenchmark({ + cases: buildFirstPartyMemoryLifecycleBenchmarkCases().slice(0, 1), + runDir: '/runs/paid-adapter-creation', + storage, + costCeiling: 0.5, + candidates: [ + { + id: 'paid-creation', + ref: 'paid-creation:v1', + adapterId: 'paid-creation-provider', + adapterCreationCostUsd: 0.2, + createAdapter({ markExternalCall }) { + markExternalCall() + creates += 1 + return createInMemoryBenchmarkAdapter({ id: 'paid-creation-provider' }) + }, + }, + ], + }) + + const ledger = createRunCostLedger({ + storage, + runDir: '/runs/paid-adapter-creation', + costCeilingUsd: 0.5, + }) + expect(creates).toBe(1) + expect(result.rows[0]).toMatchObject({ candidateId: 'paid-creation', totalCostUsd: 0.2 }) + expect(result.totalCostUsd).toBe(0.2) + expect(ledger.list()).toMatchObject([ + { + actor: 'agent-knowledge:memory-adapter:paid-creation', + costUsd: 0.2, + tags: { + candidateId: 'paid-creation', + memoryAdapterCreation: 'execute', + runDir: '/runs/paid-adapter-creation/paid-creation', + }, + }, + ]) + }) + + it('does not recreate or recharge a billable adapter when every cell resumes', async () => { + const storage = inMemoryCampaignStorage() + let creates = 0 + const run = () => + runMemoryAdapterBenchmark({ + cases: buildFirstPartyMemoryLifecycleBenchmarkCases().slice(0, 1), + runDir: '/runs/resumed-paid-adapter-creation', + storage, + costCeiling: 0.5, + candidates: [ + { + id: 'paid-resume', + ref: 'paid-resume:v1', + adapterId: 'paid-resume-provider', + adapterCreationCostUsd: 0.2, + createAdapter({ markExternalCall }) { + markExternalCall() + creates += 1 + return createInMemoryBenchmarkAdapter({ id: 'paid-resume-provider' }) + }, + }, + ], + }) + + const initial = await run() + const resumed = await run() + + expect(creates).toBe(1) + expect(initial).toMatchObject({ totalCostUsd: 0.2 }) + expect(resumed).toMatchObject({ totalCostUsd: 0.2 }) + expect(resumed.rows[0]).toMatchObject({ adapterId: 'paid-resume-provider' }) + expect(resumed.rows[0]?.report.cellsCached).toBe(1) + }) + + it('closes a mismatched lazy adapter before any benchmark case writes', async () => { + const storage = inMemoryCampaignStorage() + let closes = 0 + let writes = 0 + const adapter = createInMemoryBenchmarkAdapter({ id: 'actual-provider' }) + const write = adapter.write.bind(adapter) + adapter.write = async (input) => { + writes += 1 + return write(input) + } + adapter.close = async () => { + closes += 1 + } + + await expect( + runMemoryAdapterBenchmark({ + cases: buildFirstPartyMemoryLifecycleBenchmarkCases().slice(0, 1), + runDir: '/runs/mismatched-lazy-adapter', + storage, + candidates: [ + { + id: 'candidate', + ref: 'candidate:v1', + adapterId: 'expected-provider', + createAdapter: () => adapter, + }, + ], + }), + ).rejects.toThrow("returned id 'actual-provider', expected 'expected-provider'") + expect({ closes, writes }).toEqual({ closes: 1, writes: 0 }) + expect( + storage.read('/runs/mismatched-lazy-adapter/memory-adapter-attempts.jsonl'), + ).toBeUndefined() + }) + + it('aborts execute adapter creation at the configured timeout', async () => { + const storage = inMemoryCampaignStorage() + let aborted = false + const startedAt = Date.now() + + await expect( + runMemoryAdapterBenchmark({ + cases: buildFirstPartyMemoryLifecycleBenchmarkCases().slice(0, 1), + runDir: '/runs/timed-out-adapter-creation', + storage, + cleanupTimeoutMs: 10, + candidates: [ + { + id: 'hung-factory', + ref: 'hung-factory:v1', + createAdapter: ({ signal }) => + new Promise((_resolve, reject) => { + signal.addEventListener( + 'abort', + () => { + aborted = true + reject(signal.reason) + }, + { once: true }, + ) + }), + }, + ], + }), + ).rejects.toThrow('benchmark execute adapter creation did not finish within 10ms') + expect(aborted).toBe(true) + expect(Date.now() - startedAt).toBeLessThan(500) + expect( + storage.read('/runs/timed-out-adapter-creation/memory-adapter-attempts.jsonl'), + ).toBeUndefined() + }) + + it('does not charge a local adapter factory failure', async () => { + const storage = inMemoryCampaignStorage() + await expect( + runMemoryAdapterBenchmark({ + cases: buildFirstPartyMemoryLifecycleBenchmarkCases().slice(0, 1), + runDir: '/runs/local-adapter-creation-failure', + storage, + costCeiling: 0.5, + candidates: [ + { + id: 'local-failure', + ref: 'local-failure:v1', + adapterCreationCostUsd: 0.2, + createAdapter() { + throw new Error('invalid local configuration') + }, + }, + ], + }), + ).rejects.toThrow('invalid local configuration') + + const ledger = createRunCostLedger({ + storage, + runDir: '/runs/local-adapter-creation-failure', + costCeilingUsd: 0.5, + }) + expect(ledger.summary()).toMatchObject({ totalCostUsd: 0, unresolvedCalls: 0 }) }) it('enforces one cost ceiling across every compared adapter', async () => { diff --git a/tests/memory-adapter.test.ts b/tests/memory-adapter.test.ts index 2ab308c..5902bac 100644 --- a/tests/memory-adapter.test.ts +++ b/tests/memory-adapter.test.ts @@ -56,7 +56,11 @@ describe('memory adapters', () => { }, }, } - const adapter = createNeo4jAgentMemoryAdapter({ client, transport: 'bridge' }) + const adapter = createNeo4jAgentMemoryAdapter({ + client, + transport: 'bridge', + branchId: 'branch-1', + }) const hits = await adapter.search('writing style', { kinds: ['preference'], @@ -105,7 +109,11 @@ describe('memory adapters', () => { }, }, } - const adapter = createNeo4jAgentMemoryAdapter({ client, transport: 'bridge' }) + const adapter = createNeo4jAgentMemoryAdapter({ + client, + transport: 'bridge', + branchId: 'branch-1', + }) const hits = await adapter.search('project preferences', { scope: { sessionId: 'session-1' }, @@ -187,7 +195,7 @@ describe('memory adapters', () => { kind: 'message', role: 'assistant', text: 'I will keep updates short.', - scope: { sessionId: 'session-1', userId: 'user-1' }, + scope: { sessionId: 'session-1' }, metadata: { source: 'test' }, }) @@ -246,14 +254,12 @@ describe('memory adapters', () => { entityType: 'PERSON', text: 'Alice Johnson is a software engineer.', metadata: { description: 'Software engineer at Acme Corp' }, - scope: { userId: 'user-1' }, }) await adapter.write({ kind: 'preference', category: 'writing', text: 'Prefers direct answers', metadata: { context: 'profile' }, - scope: { userId: 'user-1' }, }) expect(calls[0]).toEqual([ @@ -286,7 +292,6 @@ describe('memory adapters', () => { kind: 'preference', category: 'writing', text: 'prefers direct answers', - scope: { userId: 'user-1' }, }) expect(calls[0]?.[0]).toBe('writing') @@ -300,6 +305,55 @@ describe('memory adapters', () => { expect(result.sourceRecord?.metadata?.memoryKind).toBe('preference') }) + it('rejects direct scopes the Neo4j SDK cannot enforce before provider calls', async () => { + let providerCalls = 0 + const adapter = createNeo4jAgentMemoryAdapter({ + transport: 'bridge', + client: { + shortTerm: { + async searchMessages() { + providerCalls += 1 + return [] + }, + }, + longTerm: { + async searchEntities() { + providerCalls += 1 + return [] + }, + async addEntity() { + providerCalls += 1 + return { id: 'entity-1' } + }, + }, + reasoning: { + async recordStep() { + providerCalls += 1 + return { id: 'reasoning-1' } + }, + }, + }, + }) + + await expect( + adapter.search('company', { kinds: ['entity'], scope: { tenantId: 'tenant-1' } }), + ).rejects.toThrow('cannot enforce scope fields: tenantId') + await expect(adapter.search('anything', { scope: { sessionId: 'session-1' } })).rejects.toThrow( + 'cannot enforce scope fields: sessionId', + ) + await expect( + adapter.write({ kind: 'entity', text: 'Acme', scope: { sessionId: 'session-1' } }), + ).rejects.toThrow('cannot enforce scope fields: sessionId') + await expect( + adapter.write({ + kind: 'reasoning-trace', + text: 'Investigated the issue.', + scope: { sessionId: 'session-1', runId: 'run-1' }, + }), + ).rejects.toThrow('cannot enforce both sessionId and runId') + expect(providerCalls).toBe(0) + }) + it('preserves Neo4j SDK unsupported errors instead of masking them with fallbacks', async () => { const calls: string[] = [] const client = { diff --git a/tests/memory-holdout.test.ts b/tests/memory-holdout.test.ts index b0a0f7a..6eecc6a 100644 --- a/tests/memory-holdout.test.ts +++ b/tests/memory-holdout.test.ts @@ -283,6 +283,7 @@ describe('retrieval holdout: adapter bypass paths still log the call', () => { client, transport: 'rest', contextMode: 'native', + branchId: 'holdout-branch', }) const context = await adapter.getContext('style', { diff --git a/tests/memory-systems.test.ts b/tests/memory-systems.test.ts index 7a2e542..cabfb70 100644 --- a/tests/memory-systems.test.ts +++ b/tests/memory-systems.test.ts @@ -12,6 +12,7 @@ import { type AgentMemoryHit, type AgentMemoryScope, agentMemorySequenceJudge, + buildAgentMemorySequenceScenarios, buildAgentMemorySequencesFromBenchmarkCases, createAgentMemoryBranch, createGraphitiMemoryAdapter, @@ -26,7 +27,9 @@ import { type RunAgentMemoryImprovementOptions, runAgentMemoryExperiment as runAgentMemoryExperimentRaw, runAgentMemoryImprovement as runAgentMemoryImprovementRaw, + runBoundedMemoryLifecycle, } from '../src/memory/index' +import { mergeRankedMemoryHits } from '../src/memory/rank' function withProcessLocalController< T extends { @@ -47,6 +50,73 @@ function runAgentMemoryImprovement(options: RunAgentMemoryImprovementOp return runAgentMemoryImprovementRaw(withProcessLocalController(options)) } +describe('memory lifecycle', () => { + it('blocks later operations until timed-out work on the same adapter settles', async () => { + const resource = {} + const abortController = new AbortController() + let release: (() => void) | undefined + const pending = new Promise((resolve) => { + release = resolve + }) + let laterRuns = 0 + + await expect( + runBoundedMemoryLifecycle({ + operation: 'slow clear', + timeoutMs: 5, + resource, + abortController, + run: () => pending, + }), + ).rejects.toThrow('slow clear did not finish within 5ms') + expect(abortController.signal.aborted).toBe(true) + await expect( + runBoundedMemoryLifecycle({ + operation: 'unsafe close', + timeoutMs: 5, + resource, + run: () => { + laterRuns += 1 + }, + }), + ).rejects.toThrow("unsafe close cannot start because 'slow clear' is still running") + expect(laterRuns).toBe(0) + + release?.() + await pending + await new Promise((resolve) => setTimeout(resolve, 0)) + await runBoundedMemoryLifecycle({ + operation: 'settled close', + timeoutMs: 5, + resource, + run: () => { + laterRuns += 1 + }, + }) + expect(laterRuns).toBe(1) + }) + + it('keeps URI and text pairs distinct when newline-delimited identities collide', () => { + const first: AgentMemoryHit = { + id: 'first', + uri: 'memory://provider/a\nb', + kind: 'fact', + text: 'c', + } + const second: AgentMemoryHit = { + id: 'second', + uri: 'memory://provider/a', + kind: 'fact', + text: 'b\nc', + } + + expect(mergeRankedMemoryHits([[first], [second]]).map((hit) => hit.id)).toEqual([ + 'first', + 'second', + ]) + }) +}) + describe('Mem0 adapter', () => { it('is type-compatible with both current Mem0 clients', () => { const hosted = null as unknown as MemoryClient @@ -198,122 +268,38 @@ describe('Mem0 adapter', () => { expect(searchOptions[1]).toMatchObject({ filters: { user_id: 'user-1' }, topK: 1 }) }) - it('waits for the official hosted Mem0 event to succeed', async () => { - let addOptions: Record | undefined - const eventIds: string[] = [] - const adapter = createMem0MemoryAdapter({ - mode: 'hosted', - ingestionTimeoutMs: 50, - pollIntervalMs: 1, - client: { - async add(_messages, options) { - addOptions = options - return { status: 'PENDING', eventId: 'event-1' } - }, - async search() { - return { results: [] } - }, - }, - async getEvent(eventId) { - eventIds.push(eventId) - return eventIds.length === 1 - ? { id: eventId, status: 'RUNNING', results: [] } - : { - id: eventId, - status: 'SUCCEEDED', - results: [ - { - id: 'memory-visible', - memory: 'Launch is Friday.', - metadata: { memoryKind: 'fact' }, - }, - ], - } - }, - }) - - const result = await adapter.write({ - id: 'source-event', - kind: 'fact', - text: 'Launch is Friday.', - scope: { userId: 'user-1' }, - }) - - const metadata = addOptions?.metadata as Record - expect(metadata.agentKnowledgeWriteId).toBe('source-event') - expect(metadata.agentKnowledgeWriteAttemptId).toBeUndefined() - expect(eventIds).toEqual(['event-1', 'event-1']) - expect(result).toMatchObject({ - accepted: true, - id: 'memory-visible', - metadata: { consistency: 'visible', queued: false }, - }) - }) - - it('treats a successful hosted Mem0 event with no results as a no-op', async () => { + it('rejects legacy asynchronous hosted responses the current SDK cannot produce', async () => { const adapter = createMem0MemoryAdapter({ mode: 'hosted', client: { async add() { - return { status: 'PENDING', eventId: 'event-noop' } - }, - async search() { - return { results: [] } - }, - }, - async getEvent(eventId) { - return { id: eventId, status: 'SUCCEEDED', results: [] } - }, - }) - - await expect(adapter.write({ kind: 'fact', text: 'No durable fact.' })).resolves.toMatchObject({ - accepted: false, - id: 'event-noop', - }) - }) - - it('surfaces a failed hosted Mem0 event', async () => { - const adapter = createMem0MemoryAdapter({ - mode: 'hosted', - client: { - async add() { - return { status: 'PENDING', eventId: 'event-failed' } + return { status: 'PENDING', eventId: 'event-1' } }, async search() { return { results: [] } }, }, - async getEvent(eventId) { - return { id: eventId, status: 'FAILED', message: 'extraction failed' } - }, }) - await expect(adapter.write({ kind: 'fact', text: 'Remember this.' })).rejects.toThrow( - 'Mem0 event event-failed failed: extraction failed', - ) + await expect( + adapter.write({ + kind: 'fact', + text: 'Remember this.', + scope: { userId: 'user-1' }, + }), + ).rejects.toThrow('mem0ai 3.x must return a memory array') }) - it('requires fresh attempt branches for visible hosted Mem0', async () => { + it('requires fresh attempt branches for hosted Mem0', async () => { const client: Mem0ClientLike = { async add() { - return { status: 'PENDING', eventId: 'event-1' } + return [] }, async search() { return { results: [] } }, } - const visible = createMem0MemoryAdapter({ - mode: 'hosted', - client, - async getEvent(eventId) { - return { id: eventId, status: 'SUCCEEDED', results: [] } - }, - }) - const queued = createMem0MemoryAdapter({ - mode: 'hosted', - consistency: 'queued', - client, - }) + const visible = createMem0MemoryAdapter({ mode: 'hosted', client }) expect(() => createAgentMemoryBranch({ adapter: visible, branchId: 'mem0-resumable' })).toThrow( "must use lifetime='attempt'", @@ -332,9 +318,6 @@ describe('Mem0 adapter', () => { snapshot, }), ).toThrow('attempt snapshots cannot be resumed') - expect(() => createAgentMemoryBranch({ adapter: queued, branchId: 'mem0-queued' })).toThrow( - 'queued hosted Mem0 writes can become visible after branch cleanup', - ) }) it('does not journal an empty provider result as an accepted write', async () => { @@ -351,7 +334,11 @@ describe('Mem0 adapter', () => { }) await expect( - adapter.write({ kind: 'fact', text: 'No provider mutation' }), + adapter.write({ + kind: 'fact', + text: 'No provider mutation', + scope: { userId: 'user-1' }, + }), ).resolves.toMatchObject({ accepted: false }) }) @@ -368,7 +355,7 @@ describe('Mem0 adapter', () => { return [ { id: text.startsWith('Forget') ? 'memory-old' : 'memory-new', - event: text.startsWith('Forget') ? 'DELETE' : 'ADD', + metadata: { event: text.startsWith('Forget') ? 'DELETE' : 'ADD' }, }, ] }, @@ -401,8 +388,8 @@ describe('Mem0 adapter', () => { expect(mem0MemoryAdapterIdentity({ ...base, defaultScope: { tenantId: 'tenant-a' } })).not.toBe( mem0MemoryAdapterIdentity({ ...base, defaultScope: { tenantId: 'tenant-b' } }), ) - expect(mem0MemoryAdapterIdentity({ ...base, consistency: 'visible' })).not.toBe( - mem0MemoryAdapterIdentity({ ...base, consistency: 'queued' }), + expect(mem0MemoryAdapterIdentity({ ...base, ingestionTimeoutMs: 100 })).not.toBe( + mem0MemoryAdapterIdentity({ ...base, ingestionTimeoutMs: 200 }), ) }) @@ -435,6 +422,64 @@ describe('Mem0 adapter', () => { }) }) + it('rejects direct OSS operations without a provider entity scope', async () => { + let providerCalls = 0 + const adapter = createMem0MemoryAdapter({ + mode: 'oss', + client: { + async add() { + providerCalls += 1 + return { results: [] } + }, + async search() { + providerCalls += 1 + return { results: [] } + }, + }, + }) + + await expect(adapter.write({ kind: 'fact', text: 'No owner.' })).rejects.toThrow( + 'requires scope.userId, scope.agentId, scope.runId, or scope.namespace', + ) + await expect(adapter.search('No owner.')).rejects.toThrow( + 'requires scope.userId, scope.agentId, scope.runId, or scope.namespace', + ) + expect(providerCalls).toBe(0) + }) + + it('rejects unscoped hosted operations before calling Mem0', async () => { + let providerCalls = 0 + const adapter = createMem0MemoryAdapter({ + mode: 'hosted', + appId: 'support-app', + client: { + async add() { + providerCalls += 1 + return [] + }, + async search() { + providerCalls += 1 + return { results: [] } + }, + async getAll() { + providerCalls += 1 + return { results: [] } + }, + async delete() { + providerCalls += 1 + return { message: 'deleted' } + }, + }, + }) + + await expect(adapter.write({ kind: 'fact', text: 'No owner.' })).rejects.toThrow( + 'Mem0 hosted write requires', + ) + await expect(adapter.search('No owner.')).rejects.toThrow('Mem0 hosted search requires') + await expect(adapter.clear?.()).rejects.toThrow('refusing an unscoped Mem0 clear') + expect(providerCalls).toBe(0) + }) + it('deletes only memories matching a complete scoped filter', async () => { const rows = new Set(['memory-1', 'memory-2']) const getAllOptions: Record[] = [] @@ -563,6 +608,81 @@ describe('Mem0 adapter', () => { expect(searchCalls).toBeGreaterThan(1) }) + it('waits for a fresh narrower id-less write when clearing a broader Mem0 scope', async () => { + const writtenAt = Date.now() + let deleted = false + let getAllCalls = 0 + let deleteCalls = 0 + const visible = () => !deleted && Date.now() - writtenAt >= 5 + const adapter = createMem0MemoryAdapter({ + mode: 'hosted', + ingestionTimeoutMs: 15, + pollIntervalMs: 1, + client: { + async add() { + return [{ event: 'ADD' }] + }, + async search() { + return { results: visible() ? [{ id: 'late-memory', memory: 'late fact' }] : [] } + }, + async getAll() { + getAllCalls += 1 + return { results: visible() ? [{ id: 'late-memory', memory: 'late fact' }] : [] } + }, + async delete(memoryId) { + expect(memoryId).toBe('late-memory') + deleteCalls += 1 + deleted = true + return { message: 'deleted' } + }, + }, + }) + + await adapter.write({ + kind: 'fact', + text: 'late fact', + scope: { userId: 'user-1', agentId: 'agent-1', namespace: 'branch-1' }, + }) + await adapter.clear?.({ userId: 'user-1' }) + + expect(deleteCalls).toBe(1) + expect(getAllCalls).toBeGreaterThan(1) + }) + + it('expires pending write probes after the configured visibility window', async () => { + let searchCalls = 0 + const adapter = createMem0MemoryAdapter({ + mode: 'hosted', + ingestionTimeoutMs: 1, + pollIntervalMs: 1, + client: { + async add() { + return [{ event: 'ADD' }] + }, + async search() { + searchCalls += 1 + return { results: [] } + }, + async getAll() { + return { results: [] } + }, + async delete() { + return { message: 'deleted' } + }, + }, + }) + + await adapter.write({ + kind: 'fact', + text: 'bounded pending fact', + scope: { userId: 'user-1' }, + }) + await new Promise((resolve) => setTimeout(resolve, 5)) + await adapter.clear?.({ userId: 'user-1' }) + + expect(searchCalls).toBe(0) + }) + it('refuses to clear Mem0 without an exact identity scope', async () => { let providerCalls = 0 const adapter = createMem0MemoryAdapter({ @@ -826,7 +946,7 @@ describe('Graphiti adapter', () => { client: { async callTool(request) { if (request.name === 'search_memory_facts') { - groupIds.push(String((request.arguments?.group_ids as string[])[0])) + groupIds.push(String((request.arguments?.group_ids as string[] | undefined)?.[0])) return { structuredContent: { facts: [] } } } return { structuredContent: { episodes: [] } } @@ -1899,7 +2019,7 @@ describe('agent memory experiments', () => { const clear = adapter.clear! adapter.clear = async (scope) => { clearCalls += 1 - if (clearCalls > 1) return new Promise(() => {}) + if (clearCalls === 1) return new Promise(() => {}) await clear(scope) } const startedAt = Date.now() @@ -1971,6 +2091,107 @@ describe('agent memory experiments', () => { expect(costLedger.summary().totalCostUsd).toBe(0) }) + it('charges a factory failure after dedicated provider provisioning starts', async () => { + const storage = inMemoryCampaignStorage() + const costLedger = createRunCostLedger({ + storage, + runDir: '/runs/create-adapter-provider-cost', + costCeilingUsd: 1, + }) + + await expect( + runAgentMemoryExperiment({ + experimentId: 'create-adapter-provider-cost', + sequences: [ + { + id: 'history', + family: 'first-party', + steps: [{ id: 'probe', probes: [{ id: 'probe', query: 'x', referenceAnswer: 'x' }] }], + }, + ], + candidates: [ + { + id: 'paid-memory', + ref: 'paid-memory:v1', + externalCostUsdPerSequence: 0.25, + createAdapter({ markExternalCall }) { + markExternalCall() + throw new Error('provider provisioning failed') + }, + }, + ], + runDir: '/runs/create-adapter-provider-cost', + storage, + costLedger, + }), + ).rejects.toThrow('memory experiment cleanup failed after dispatch') + + expect(costLedger.summary()).toMatchObject({ totalCalls: 1, totalCostUsd: 0.25 }) + }) + + it('records provider cleanup before a paid-call receipt can be interrupted', async () => { + const storage = inMemoryCampaignStorage() + const append = storage.append!.bind(storage) + let interruptReceipt = true + storage.append = (path, value, expectedBytes) => { + if ( + interruptReceipt && + path.endsWith('/cost-ledger.jsonl') && + value.includes('"status":"settled"') && + value.includes('agent-knowledge:memory-experiment') + ) { + interruptReceipt = false + throw new Error('simulated process exit before execute receipt') + } + return append(path, value, expectedBytes) + } + const purposes: string[] = [] + const sequence = { + id: 'history', + family: 'first-party' as const, + steps: [ + { + id: 'remember', + scope: { agentId: 'worker' }, + writes: [{ kind: 'fact' as const, text: 'durable fact' }], + probes: [{ id: 'recall', query: 'durable', referenceAnswer: 'durable fact' }], + }, + ], + } + const run = () => + runAgentMemoryExperiment({ + experimentId: 'execute-receipt-crash', + sequences: [sequence], + candidates: [ + { + id: 'memory', + ref: 'memory:v1', + externalCostUsdPerSequence: 0.1, + createAdapter({ purpose }) { + purposes.push(purpose) + return createScopedTestAdapter(`memory:${purpose}`) + }, + }, + ], + runDir: '/runs/execute-receipt-crash', + storage, + costCeiling: 1, + }) + + const interrupted = await run() + expect(interrupted.campaign.aggregates).toMatchObject({ + cellsFailed: 1, + cost: { accountingComplete: false, unresolvedCalls: 1 }, + }) + expect( + storage.read('/runs/execute-receipt-crash/memory-attempts.jsonl')?.trim().split('\n'), + ).toHaveLength(2) + + const result = await run() + expect(purposes).toEqual(['execute', 'execute']) + expect(result.rows[0]).toMatchObject({ cellsFailed: 0, totalCostUsd: 0.2 }) + }) + it('accounts for parallel candidates against one shared dollar limit', async () => { let releaseFirstCalls: (() => void) | undefined const firstCallsReady = new Promise((resolve) => { @@ -2090,6 +2311,83 @@ describe('agent memory experiments', () => { }) }) + it('reports the reserved cost of an interrupted prior cell attempt', async () => { + const storage = inMemoryCampaignStorage() + const runDir = '/runs/interrupted-cell-cost-reporting' + const sequence = { + id: 'history', + family: 'first-party' as const, + steps: [ + { + id: 'remember', + scope: { agentId: 'worker' }, + writes: [{ kind: 'fact' as const, text: 'resumed fact' }], + probes: [{ id: 'recall', query: 'resumed', referenceAnswer: 'resumed fact' }], + }, + ], + } + const candidate = { + id: 'memory', + ref: 'memory:v1', + externalCostUsdPerSequence: 0.1, + createAdapter: () => createScopedTestAdapter('memory'), + } + const scenarioId = buildAgentMemorySequenceScenarios([sequence], [candidate])[0]!.id + const abandonedLedger = createRunCostLedger({ storage, runDir, costCeilingUsd: 1 }) + const controller = new AbortController() + let releaseProvider!: () => void + let reportStarted!: () => void + const started = new Promise((resolve) => { + reportStarted = resolve + }) + const abandoned = abandonedLedger.runPaidCall({ + callId: 'prior-cell-attempt', + channel: 'agent', + phase: 'memory.experiment', + actor: 'agent-knowledge:memory-experiment:memory', + model: 'memory', + signal: controller.signal, + tags: { + runDir, + scenarioId, + cellId: 'prior-cell', + rep: '0', + runAttemptId: 'prior-attempt', + }, + maximumCharge: { externallyEnforcedMaximumUsd: 0.1 }, + async execute() { + reportStarted() + return await new Promise((resolve) => { + releaseProvider = () => resolve('late result') + }) + }, + receipt: () => ({ + model: 'memory', + inputTokens: 0, + outputTokens: 0, + actualCostUsd: 0.05, + }), + }) + await started + controller.abort(new Error('simulated process exit')) + await abandoned + + const resumedLedger = createRunCostLedger({ storage, runDir, costCeilingUsd: 1 }) + const result = await runAgentMemoryExperiment({ + experimentId: 'interrupted-cell-cost-reporting', + sequences: [sequence], + candidates: [candidate], + runDir, + storage, + costLedger: resumedLedger, + }) + + expect(result.rows[0]).toMatchObject({ candidateId: 'memory', totalCostUsd: 0.2 }) + expect(resumedLedger.summary()).toMatchObject({ totalCalls: 2, totalCostUsd: 0.2 }) + releaseProvider() + await abandonedLedger.waitForIdle() + }) + it('reconciles interrupted paid calls from every parallel memory branch before resuming', async () => { const storage = inMemoryCampaignStorage() const runDir = '/runs/parallel-interrupted-cost-recovery' @@ -2551,7 +2849,7 @@ describe('agent memory experiments', () => { expect(result.rows[0]).toMatchObject({ cellsFailed: 1 }) expect(result.campaign.cells[0]?.error).toContain('dispatch exceeded 5ms') - expect(clears).toBeGreaterThanOrEqual(2) + expect(clears).toBe(1) expect(rows).toEqual([]) }) @@ -2586,7 +2884,7 @@ describe('agent memory experiments', () => { }, async clear() { clears += 1 - if (clears > 1) throw new Error('provider cleanup unavailable') + throw new Error('provider cleanup unavailable') }, } const run = runAgentMemoryExperiment({ @@ -2680,7 +2978,7 @@ describe('agent memory experiments', () => { async clear(scope) { clearCalls += 1 operations.push(`clear:${executionLabel}`) - if (executionLabel === 'first' && clearCalls > 1) { + if (executionLabel === 'first' && clearCalls === 1) { throw new Error('provider cleanup unavailable') } rows.delete(scope?.namespace ?? '') @@ -2725,6 +3023,82 @@ describe('agent memory experiments', () => { ]) }) + it('closes and disposes a recovery adapter that arrives after its factory timeout', async () => { + const storage = inMemoryCampaignStorage() + const runDir = '/runs/late-recovery-adapter' + storage.write( + `${runDir}/memory-attempts.jsonl`, + `${JSON.stringify({ + schema: 2, + status: 'started', + branchId: 'unfinished-branch', + candidateId: 'memory', + candidateRef: 'memory:v1', + sequenceId: 'history', + rep: 0, + seed: 42, + cleanupBranches: true, + externalCostUsdPerSequence: 0, + externalRecoveryCostUsdPerAttempt: 0, + recordedAt: '2026-01-01T00:00:00.000Z', + recovery: false, + })}\n`, + ) + const sequence = { + id: 'history', + family: 'first-party' as const, + steps: [ + { + id: 'remember', + scope: { agentId: 'worker' }, + writes: [{ kind: 'fact' as const, text: 'fact' }], + probes: [{ id: 'recall', query: 'fact', referenceAnswer: 'fact' }], + }, + ], + } + let resolveCreation!: (adapter: AgentMemoryAdapter) => void + const creation = new Promise((resolve) => { + resolveCreation = resolve + }) + let reportDisposed!: () => void + const disposed = new Promise((resolve) => { + reportDisposed = resolve + }) + let closeCalls = 0 + let disposeCalls = 0 + const lateAdapter = createScopedTestAdapter('memory-provider') + lateAdapter.close = async () => { + closeCalls += 1 + } + + await expect( + runAgentMemoryExperiment({ + experimentId: 'late-recovery-adapter', + sequences: [sequence], + candidates: [ + { + id: 'memory', + ref: 'memory:v1', + createAdapter: ({ purpose }) => + purpose === 'recovery' ? creation : createScopedTestAdapter('memory-provider'), + async disposeAdapter(adapter) { + expect(adapter).toBe(lateAdapter) + disposeCalls += 1 + reportDisposed() + }, + }, + ], + runDir, + storage, + cleanupTimeoutMs: 10, + }), + ).rejects.toThrow("memory: abandoned memory branch 'unfinished-branch' recovery failed") + + resolveCreation(lateAdapter) + await disposed + expect({ closeCalls, disposeCalls }).toEqual({ closeCalls: 1, disposeCalls: 1 }) + }) + it('reconciles a crash after provider recovery but before its cost receipt', async () => { const storage = inMemoryCampaignStorage() const append = storage.append!.bind(storage) @@ -2758,7 +3132,7 @@ describe('agent memory experiments', () => { adapter.clear = async (scope) => { clearCalls += 1 if (purpose === 'recovery') recoveryClears += 1 - if (failThisExecution && clearCalls === 2) { + if (failThisExecution && clearCalls === 1) { throw new Error('leave the first branch active') } await clear(scope) @@ -2800,7 +3174,7 @@ describe('agent memory experiments', () => { await expect(run()).rejects.toThrow('failed to persist') expect( storage.read('/runs/recovery-receipt-crash/memory-attempts.jsonl')?.trim().split('\n'), - ).toHaveLength(1) + ).toHaveLength(2) const result = await run() const costLedger = createRunCostLedger({ @@ -2809,13 +3183,14 @@ describe('agent memory experiments', () => { costCeilingUsd: 1, }) - expect(result.rows[0]).toMatchObject({ candidateId: 'crash-recoverable', cellsFailed: 0 }) - expect(recoveryClears).toBe(2) - expect(costLedger.summary()).toMatchObject({ - unresolvedCalls: 0, - totalCostUsd: 0.4, - accountingComplete: true, + expect(result.rows[0]).toMatchObject({ + candidateId: 'crash-recoverable', + cellsFailed: 0, }) + expect(result.rows[0]?.totalCostUsd).toBeCloseTo(0.3) + expect(recoveryClears).toBe(1) + expect(costLedger.summary()).toMatchObject({ unresolvedCalls: 0, accountingComplete: true }) + expect(costLedger.summary().totalCostUsd).toBeCloseTo(0.3) }) it('recovers independent abandoned branches in parallel before retrying them', async () => { @@ -2963,7 +3338,7 @@ describe('agent memory experiments', () => { storage.write( `${runDir}/memory-attempts.jsonl`, `${JSON.stringify({ - schema: 1, + schema: 2, status: 'started', branchId: 'retired-branch', candidateId: 'retired', @@ -2972,6 +3347,8 @@ describe('agent memory experiments', () => { rep: 0, seed: 1, cleanupBranches: true, + externalCostUsdPerSequence: 0, + externalRecoveryCostUsdPerAttempt: 0.1, recordedAt: '2026-01-01T00:00:00.000Z', recovery: false, })}\n`, @@ -2995,6 +3372,7 @@ describe('agent memory experiments', () => { { id: 'retired', ref: 'retired:v1', + externalRecoveryCostUsdPerAttempt: 0.1, createAdapter({ purpose }) { purposes.push(`retired:${purpose}`) const adapter = createScopedTestAdapter('retired') @@ -3008,14 +3386,70 @@ describe('agent memory experiments', () => { runDir, storage, resumable: false, + costCeiling: 1, }) expect(result.rows).toHaveLength(1) expect(result.rows[0]?.candidateId).toBe('active') + expect(result.rows[0]?.totalCostUsd).toBe(0) + expect(result).toMatchObject({ totalCostUsd: 0.1, unrankedRecoveryCostUsd: 0.1 }) expect(purposes).toEqual(['retired:recovery', 'active:execute']) expect(retiredClears).toBeGreaterThan(0) }) + it('refuses to hide an unfinished branch when candidate cost settings change', async () => { + const storage = inMemoryCampaignStorage() + const runDir = '/runs/changed-recovery-costs' + const sequence = { + id: 'history', + family: 'first-party' as const, + steps: [{ id: 'probe', probes: [{ id: 'recall', query: 'fact', referenceAnswer: 'fact' }] }], + } + storage.write( + `${runDir}/memory-attempts.jsonl`, + `${JSON.stringify({ + schema: 2, + status: 'started', + branchId: 'unfinished-branch', + candidateId: 'memory', + candidateRef: 'memory:v1', + sequenceId: sequence.id, + rep: 0, + seed: 1, + cleanupBranches: true, + externalCostUsdPerSequence: 0, + externalRecoveryCostUsdPerAttempt: 0, + recordedAt: '2026-01-01T00:00:00.000Z', + recovery: false, + })}\n`, + ) + let adapterCreates = 0 + + await expect( + runAgentMemoryExperiment({ + experimentId: 'changed-recovery-costs', + sequences: [sequence], + candidates: [ + { + id: 'memory', + ref: 'memory:v1', + externalCostUsdPerSequence: 0.1, + createAdapter() { + adapterCreates += 1 + return createScopedTestAdapter('memory') + }, + }, + ], + runDir, + storage, + costCeiling: 1, + }), + ).rejects.toThrow('candidate cost settings changed') + + expect(adapterCreates).toBe(0) + expect(storage.read(`${runDir}/memory-attempts.jsonl`)?.trim().split('\n')).toHaveLength(1) + }) + it('refuses a recovery backlog larger than maxRecoveryAttempts', async () => { const storage = inMemoryCampaignStorage() const runDir = '/runs/recovery-backlog-limit' @@ -3034,7 +3468,7 @@ describe('agent memory experiments', () => { `${runDir}/memory-attempts.jsonl`, `${[ { - schema: 1, + schema: 2, status: 'started', branchId: 'branch-1', candidateId: 'memory', @@ -3043,11 +3477,13 @@ describe('agent memory experiments', () => { rep: 0, seed: 1, cleanupBranches: true, + externalCostUsdPerSequence: 0, + externalRecoveryCostUsdPerAttempt: 0, recordedAt: '2026-01-01T00:00:00.000Z', recovery: false, }, { - schema: 1, + schema: 2, status: 'started', branchId: 'branch-2', candidateId: 'memory', @@ -3056,6 +3492,8 @@ describe('agent memory experiments', () => { rep: 0, seed: 2, cleanupBranches: true, + externalCostUsdPerSequence: 0, + externalRecoveryCostUsdPerAttempt: 0, recordedAt: '2026-01-01T00:00:00.000Z', recovery: false, }, @@ -3087,6 +3525,65 @@ describe('agent memory experiments', () => { expect(providerCreates).toBe(0) }) + it('bounds repeated provider recovery across process restarts', async () => { + const storage = inMemoryCampaignStorage() + const runDir = '/runs/recovery-retry-limit' + const sequence = { + id: 'history', + family: 'first-party' as const, + steps: [{ id: 'probe', probes: [{ id: 'recall', query: 'fact', referenceAnswer: 'fact' }] }], + } + storage.write( + `${runDir}/memory-attempts.jsonl`, + `${JSON.stringify({ + schema: 2, + status: 'started', + branchId: 'unfinished-branch', + candidateId: 'memory', + candidateRef: 'memory:v1', + sequenceId: sequence.id, + rep: 0, + seed: 1, + cleanupBranches: true, + externalCostUsdPerSequence: 0, + externalRecoveryCostUsdPerAttempt: 0, + recordedAt: '2026-01-01T00:00:00.000Z', + recovery: false, + })}\n`, + ) + let recoveryCreates = 0 + const run = () => + runAgentMemoryExperiment({ + experimentId: 'recovery-retry-limit', + sequences: [sequence], + candidates: [ + { + id: 'memory', + ref: 'memory:v1', + createAdapter({ purpose }) { + if (purpose === 'recovery') recoveryCreates += 1 + throw new Error('provider recovery unavailable') + }, + }, + ], + runDir, + storage, + maxRecoveryRetriesPerAttempt: 2, + }) + + await expect(run()).rejects.toThrow( + "abandoned memory branch 'unfinished-branch' recovery failed", + ) + await expect(run()).rejects.toThrow( + "abandoned memory branch 'unfinished-branch' recovery failed", + ) + await expect(run()).rejects.toThrow('exhausted 2 recovery attempts') + expect(recoveryCreates).toBe(2) + expect( + storage.read(`${runDir}/memory-recovery-attempts.jsonl`)?.trim().split('\n'), + ).toHaveLength(2) + }) + it('allows one controller per run while its history workers run in parallel', async () => { const storage = inMemoryCampaignStorage() let reportStarted: (() => void) | undefined @@ -3245,14 +3742,14 @@ describe('agent memory experiments', () => { await expect(run()).rejects.toThrow('controller ownership expired') expect(searches).toBe(0) - expect(clears).toBe(1) + expect(clears).toBe(0) expect(closes).toBe(1) expect(factVisible).toBe(true) const result = await run() expect(result.rows[0]).toMatchObject({ candidateId: 'memory', cellsFailed: 0 }) expect(purposes).toEqual(['execute', 'recovery', 'execute']) - expect(clears).toBe(4) + expect(clears).toBe(2) expect(closes).toBe(3) expect(factVisible).toBe(false) @@ -3380,7 +3877,7 @@ describe('agent memory experiments', () => { async clear(scope) { clears += 1 await base.clear?.(scope) - if (clears > 1) throw new Error('provider clear failed') + throw new Error('provider clear failed') }, async close() { closes += 1 @@ -3402,7 +3899,7 @@ describe('agent memory experiments', () => { }) await expect(run).rejects.toThrow('memory experiment cleanup failed after dispatch') - expect(clears).toBe(2) + expect(clears).toBe(1) expect(closes).toBe(1) expect(disposals).toBe(1) }) @@ -3462,7 +3959,7 @@ describe('agent memory experiments', () => { }) expect(dirtyRows).toBe(0) - expect(clears).toBe(2) + expect(clears).toBe(1) expect(result.rows[0]).toMatchObject({ candidateId: 'complete', cellsFailed: 0, @@ -3838,7 +4335,7 @@ describe('agent memory improvement', () => { expect( JSON.parse(storage.read('/runs/improve-team-memory/memory-improvement-manifest.json') ?? '{}') .identity?.schema, - ).toBe(5) + ).toBe(6) const resumed = await runAgentMemoryImprovement(options) expect(proposalCalls).toBe(1)