diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..ec74c2f --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,37 @@ +# Changelog + +## 4.0.0 + +### Breaking Changes + +- `runAgentMemoryImprovement()` now activates a measured winner through `activation.readCurrent()` and atomic `activation.compareAndSet()` instead of `onPromote`. +- `AgentMemoryActivation.receiptPath` is now `journalPath` because the file is an append-only activation record. +- `mem0MemoryAdapterIdentity()` and `graphitiMemoryAdapterIdentity()` require a stable, non-secret `backendRef` so two deployments cannot share a cache identity. +- Mem0 hosted mode now follows the synchronous array response from `mem0ai` 3.x; the unsupported queued-event options were removed. +- Paid benchmark and improvement work now defaults to a zero dollar limit and requires an explicit `costCeiling` or `maxTotalCostUsd`. +- In-flight run directories from releases before 4.0 are not migrated; archive or clear them before upgrading because 4.0 rejects older attempt records. + +### Added + +- Official-client adapters for Mem0 hosted and open-source deployments, Graphiti MCP, and Neo4j Agent Memory. +- Isolated memory branches with snapshots, replayable forks, private, team, and shared visibility, and ordered writes per agent. +- Parallel multi-track memory experiments and configuration search on `agent-eval`, with fresh-history comparison before activation. +- Durable controller ownership, interrupted-attempt cleanup, retired-candidate recovery, bounded cleanup work, and conservative recovery cost reconciliation. +- Complete candidate cost attribution across interrupted retries, plus explicit unranked recovery spend for retired benchmark candidates. +- Exact scoped Mem0 deletion with list and search convergence checks. + +### Fixed + +- Mem0 and direct Neo4j operations reject provider scopes they cannot enforce before any provider call. +- Hosted Mem0 `appId` is an additional filter and cannot authorize an unscoped whole-application read or delete. +- Mem0 cleanup tracks fresh writes until they become visible and confirms deletion from both list and search indexes. +- Broader Mem0 cleanup scopes wait for delayed writes created under matching narrower scopes. +- Mem0 pending-write probes expire after the configured visibility window instead of accumulating for the adapter lifetime. +- Direct Neo4j reasoning writes reject combined session and run scopes because the SDK can enforce only one conversation identifier. +- Timed-out provider work blocks close or reuse of the same adapter until the original operation settles. +- Recovery retries are reserved durably before provider work and stop after three failures per attempt by default. +- Direct memory benchmarks account for billable adapter provisioning and reconnects in the shared dollar limit. +- Fully cached benchmark resumes skip adapter creation and add no provider charge. +- Execute and recovery adapter factories receive abort signals and are bounded by the configured timeout. +- Adapters returned after a timed-out factory call are closed, and experiment adapters also run their configured disposal callback. +- Reported dollar totals are normalized to twelve decimal places instead of exposing binary floating-point artifacts. diff --git a/README.md b/README.md index 5a51a47..bbdd55e 100644 --- a/README.md +++ b/README.md @@ -11,9 +11,9 @@ This package turns raw sources and generated markdown knowledge into a versionab - [Common uses](#common-uses): wiki, RAG, memory, new KBs, existing KBs, and parallel agents - [CLI](#cli): `init` → `source-add` → `index` → `search` → `lint` - [Design](#design): the invariants, including immutable sources, cited claims, and deterministic graph output -- [Benchmark harness](#benchmark-harness): BEIR/MTEB/qrels, RAG answer, hallucination, KB-improvement cases +- [Benchmark runner](#benchmark-runner): BEIR/MTEB/qrels, RAG answer, hallucination, KB-improvement cases - [Agent-Eval integration](#agent-eval-integration): retrieval eval, readiness bundles, and release reports -- [Memory adapters](#memory-adapters): generic memory contract plus Neo4j Agent Memory bridge +- [Memory adapters](#memory-adapters): Mem0, Graphiti, Neo4j, branching, and automatic improvement - [Research loop](#research-loop): `runKnowledgeResearchLoop` plus the control-loop adapter - [Runtime integration](#runtime-integration): how agent runners plug into the pure KB loop - [Pluggable knowledge sources](#pluggable-knowledge-sources): live authorities → eval re-runs @@ -39,8 +39,8 @@ Two ways in, depending on what you're doing: - *"Expose the lower-level RAG lifecycle phases"* → `runRagKnowledgeImprovementLoop` in the [Agent-Eval integration](#agent-eval-integration) section. It exposes retrieval tuning, gap diagnosis, knowledge acquisition/update, answer-quality checks, and promotion as one typed lifecycle. - *"Evaluate RAG answers or a wiki/KB"* → `ragAnswerQualityJudge`, `createRagAnswerQualityHook`, and `scoreKnowledgeBaseIndex` in the [Agent-Eval integration](#agent-eval-integration) section. - - *"Run standard RAG/KB benchmarks"* → `runKnowledgeBenchmarkSuite` in the [Benchmark harness](#benchmark-harness) section. - - *"Does this candidate KB actually improve task success?"* → run an [agent-eval improvement loop](#agent-eval-integration) over KB variants, then `knowledgeReleaseReport` for the promotion decision. + - *"Run standard RAG/KB benchmarks"* → `runKnowledgeBenchmarkSuite` in the [Benchmark runner](#benchmark-runner) section. + - *"Does this candidate KB improve task success?"* → run an [agent-eval improvement loop](#agent-eval-integration) over KB variants, then `knowledgeReleaseReport` for the promotion decision. - *"Keep live authorities fresh"* → [pluggable sources](#pluggable-knowledge-sources) + `detectChanges` → eval re-runs. Storage stays consumer-owned via `KbStore` (`MemoryKbStore`, `FileSystemKbStore`, or your own D1/Postgres). Every primitive below is source-grounded: claims cite immutable source records, and lint fails on un-grounded citations. @@ -51,10 +51,10 @@ Storage stays consumer-owned via `KbStore` (`MemoryKbStore`, `FileSystemKbStore` |---|---|---| | LLM wiki or docs KB | Source records, cited markdown pages, lint, readiness scoring, candidate promotion | `improveKnowledgeBase` or `runKnowledgeResearchLoop` | | RAG | Retrieval eval, source-span labels, answer quality checks, missing/stale/noisy-source diagnosis | `runRagKnowledgeImprovementLoop` | -| Memory DB | Adapter contract that turns memory hits into source-like evidence and benchmark rows | `/memory` plus `runMemoryAdapterBenchmark` | +| Memory DB | Provider adapters, isolated branches, sharing policies, ordered histories, and fresh comparison | `/memory` plus `runAgentMemoryExperiment` | | From scratch | Empty KB layout, source ingestion, write-block application, indexing, readiness checks | CLI `init` or `runKnowledgeResearchLoop` | | Existing KB | Candidate workspace, resume state, base-hash conflict check, exact approved promotion | `improveKnowledgeBase` | -| Parallel agents | Per-run locks, isolated candidates, frozen candidate handoff, retry with the same `runId` | `improveKnowledgeBase` with runtime workers | +| Parallel agents | Isolated KB candidates or memory branches, ordered per-agent writes, concurrent histories, and explicit activation | `improveKnowledgeBase` or `runAgentMemoryImprovement` | | One-call agent job | Runtime supervisor plus all KB mechanics above | `runKnowledgeImprovementJob` from `@tangle-network/agent-runtime/knowledge` | ## CLI @@ -132,7 +132,7 @@ from `@tangle-network/agent-knowledge`. - Zod schemas define the stable wire shape. - Graph/search/lint are deterministic and fast. - `searchKnowledge` returns hits with three score fields. `score` and - `rrfScore` are the raw reciprocal-rank-fusion value (typically 0.01–0.05); + `rrfScore` are the raw reciprocal-rank-fusion value (typically 0.01 to 0.05); use them when intent matters or when fusing across queries. `normalizedScore` is the same value scaled into [0, 1] relative to the top hit *in this result set* (top hit = 1, others = score / topScore). Use it @@ -151,7 +151,7 @@ The `/memory` subpath exports an optional memory adapter contract. Use it to bridge episodic or graph-native memory systems into the same source-grounded readiness/eval machinery without making `agent-knowledge` own the database. -## Benchmark Harness +## Benchmark Runner Use `runKnowledgeBenchmarkSuite()` when the product goal is to run a fixed RAG/KB benchmark pack and get one report across retrieval, answer quality, hallucination, and candidate-KB improvement cases. The module also exports `INDUSTRY_RAG_BENCHMARKS`, a compact manifest for BEIR, MTEB retrieval, MS MARCO, TREC DL, MIRACL, LoTTE, BRIGHT, CRAG, HotpotQA, KILT, RAGTruth, FaithBench, and first-party KB-improvement tasks. @@ -171,17 +171,18 @@ import { runMemoryAdapterBenchmark, runKnowledgeBenchmarkSuite, } from '@tangle-network/agent-knowledge/benchmarks' -import { createNeo4jAgentMemoryAdapter } from '@tangle-network/agent-knowledge/memory' await runKnowledgeBenchmarkSuite({ cases: buildIndustryRagBenchmarkSmokeCases(), runDir: '.agent-knowledge/benchmark-runs/industry-smoke', + respondRef: 'industry-rag-smoke:v1', respond: respondToIndustryRagBenchmarkSmokeCase, }) await runKnowledgeBenchmarkSuite({ cases: buildIndustryMemoryBenchmarkSmokeCases(), runDir: '.agent-knowledge/benchmark-runs/memory-smoke', + respondRef: 'industry-memory-smoke:v1', respond: ({ case: testCase }) => { if (!isKnowledgeMemoryBenchmarkCase(testCase)) return { answer: '' } return respondToIndustryMemoryBenchmarkSmokeCase({ case: testCase }) @@ -191,18 +192,15 @@ await runKnowledgeBenchmarkSuite({ const memoryRanking = await runMemoryAdapterBenchmark({ cases: buildFirstPartyMemoryLifecycleBenchmarkCases(), runDir: '.agent-knowledge/benchmark-runs/memory-ranking', + costCeiling: 5, candidates: [ - { id: 'no-memory', createAdapter: () => createNoopMemoryBenchmarkAdapter() }, + { id: 'no-memory', ref: 'no-memory:v1', createAdapter: () => createNoopMemoryBenchmarkAdapter() }, { id: 'in-memory', + ref: 'in-memory:v1', createAdapter: () => createInMemoryBenchmarkAdapter(), searchLimit: 1, }, - { - id: 'neo4j-agent-memory', - createAdapter: () => createNeo4jAgentMemoryAdapter({ client: neo4jMemoryClient }), - searchLimit: 5, - }, ], }) @@ -220,6 +218,8 @@ const cases = buildRetrievalBenchmarkCasesFromQrels({ const result = await runKnowledgeBenchmarkSuite({ cases, runDir: '.agent-knowledge/benchmark-runs/beir-nfcorpus-smoke', + respondRef: 'product-retriever:v3', + costCeiling: 5, respond: async ({ case: testCase }) => { if (testCase.taskKind !== 'retrieval') return { hits: [] } const hits = await retrieveFromYourKb(testCase.query) @@ -230,7 +230,29 @@ const result = await runKnowledgeBenchmarkSuite({ console.log(result.report.score.mean) ``` +`runMemoryAdapterBenchmark()` is for adapters that enforce every supplied scope and support exact scoped deletion. +Every cell records its fresh provider namespace before the first write and deletes it after measurement, so concurrent repetitions and retries cannot share memory. +On restart, unfinished scopes are deleted before any new benchmark cell runs. +The candidate factory receives `purpose: 'execute' | 'recovery'`, `signal`, and `markExternalCall()`; both paths must reconnect to the same provider and adapter identity. +Pass `signal` to provider connection and provisioning calls so timeout cancellation reaches the SDK. +Set `adapterId` when the returned adapter ID differs from the candidate ID. +The declared ID lets a fully cached resume skip adapter creation entirely; a returned mismatch fails before any case runs. +Keep local construction free. +When construction provisions or reconnects billable provider state, set `adapterCreationCostUsd` and call `markExternalCall()` immediately before the external action. +Normal read and write work belongs in adapter methods so it runs inside per-case cost accounting. +Use `runAgentMemoryExperiment()` below when an agent profile needs ordered multi-step histories, branch snapshots, runtime callbacks, or automatic configuration improvement. +Set `cleanupTimeoutMs` to bound each provider cleanup and close operation; the default is 180 seconds. +Set `costUsdPerCase` for normal provider work, `adapterCreationCostUsd` for billable construction or reconnects, and `recoveryCostUsdPerAttempt` when deleting an interrupted case can add another provider charge. +One `costCeiling` covers every candidate and recovery call in the comparison, and `totalCostUsd` reports the complete comparison cost. +`unrankedRecoveryCostUsd` identifies cleanup spend from retired candidates that cannot appear in ranking rows. +The default `costCeiling` is zero, so paid work is refused until the caller sets a limit. +Keep removed implementations in `recoveryCandidates` until their unfinished scopes are gone, and use `maxRecoveryAttempts` to reject an unexpectedly large backlog before provider work starts. +Each failed recovery is recorded before its provider call. +`maxRecoveryRetriesPerAttempt` defaults to three, and `recoveryLogPath` identifies the durable retry history returned by the run. +The default filesystem storage uses an OS lock; custom storage requires `controllerMode: 'process-local'` or a distributed `acquireRunLease` implementation. + Billable responders must execute paid work through `context.cost.runPaidCall()`; `costUsd` on an artifact is display-only. +Change `respondRef` whenever the model, prompt, retrieval, tools, or runtime behavior changes; resumable runs reject a missing reference instead of reusing ambiguous cached rows. Use `buildRetrievalBenchmarkCasesFromQrels()` for qrels-backed retrieval datasets. The smoke pack proves that every declared benchmark family is wired through the runner; full BEIR, MTEB, MS MARCO, TREC DL, MIRACL, LoTTE, BRIGHT, CRAG, HotpotQA, KILT, RAGTruth, and FaithBench runs should pass real dataset rows through the same case shapes. Use `KnowledgeAnswerBenchmarkCase` for CRAG/HotpotQA/KILT-style answer checks and RAGTruth/FaithBench-style hallucination checks by encoding required claims, forbidden claims, and expected source IDs. @@ -239,7 +261,8 @@ Use `KnowledgeMemoryBenchmarkCase` for memory systems. Memory cases carry ordered events plus current required facts, stale forbidden facts, expected event IDs, and expected actor IDs. The built-in memory scorer reports current fact recall, stale-memory safety, event recall, and actor recall, which maps to LoCoMo, LongMemEval, Memora, MemoryAgentBench, MemoryBank-style personalization, GroupMemBench, and first-party memory lifecycle checks. Use `runMemoryAdapterBenchmark()` to rank real memory systems that implement `AgentMemoryAdapter`. -This is the path for Neo4j Agent Memory, Mem0, Zep, Letta, Graphiti, a vector store, or a product-owned memory backend. +Built-in adapters cover Neo4j Agent Memory, Mem0 hosted and open source, and the official Graphiti MCP server. +Zep, Letta, a vector store, or a product-owned backend can implement the same small contract. ## Agent-Eval Integration @@ -364,7 +387,7 @@ That keeps the public API from reporting a fake “RAG improved” result when t Use retrieval eval when the question is whether a retrieval/RAG config can find the right knowledge before an agent reasons over it. The labels should name stable pages, source records, anchors, or source spans, not ephemeral chunk IDs. -The completion roadmap is in [`docs/eval/rag-eval-roadmap.md`](docs/eval/rag-eval-roadmap.md). +Benchmark coverage and completion criteria are documented in [`docs/eval/rag-eval-roadmap.md`](docs/eval/rag-eval-roadmap.md). ```ts import { runRetrievalImprovementLoop } from '@tangle-network/agent-knowledge' @@ -392,7 +415,7 @@ Pass a custom `retrieve` function to `buildRetrievalEvalDispatch` when the confi The built-in fallback uses `searchKnowledge` over the local deterministic index. Use `buildRetrievalEvalDispatch`, `retrievalRecallJudge`, and `retrievalParameterSweepProposer` directly only when you need custom `agent-eval` wiring. -To answer whether a candidate knowledge base actually improves agent task success, run an `@tangle-network/agent-eval` improvement loop (`runImprovementLoop`) over your KB variants on a real task corpus; each run is scored into a `RunRecord`. +To answer whether a candidate knowledge base improves agent task success, run an `@tangle-network/agent-eval` improvement loop (`runImprovementLoop`) over your KB variants on a real task corpus; each run is scored into a `RunRecord`. Use `knowledgeReleaseReport()` before promotion: pass the candidate and baseline `RunRecord[]` (plus optional `ReleaseTraceEvidence` and the gate decision) and it folds them into a `ReleaseConfidenceScorecard` and a `KnowledgeRelease` using `agent-eval`'s release gates and `RunRecord` validation. @@ -429,35 +452,278 @@ Pass `readiness.report` to `blockingKnowledgeEval()` from ## Memory Adapters -`agent-knowledge` does not store operational memory itself. It defines the -contract that lets a runtime read/write memory through any backend, then turn -memory hits into `SourceRecord` evidence for readiness, linting, and eval gates. +`agent-knowledge` does not replace a memory database. +It gives existing memory systems one contract, isolated branches, ordered multi-agent histories, comparable measurements, and an automatic improvement loop. + +Use the provider adapter that matches the system you already run: ```ts import { + createGraphitiMemoryAdapter, + createMem0MemoryAdapter, createNeo4jAgentMemoryAdapter, - memoryHitToSourceRecord, } from '@tangle-network/agent-knowledge/memory' -const memory = createNeo4jAgentMemoryAdapter({ client: neo4jMemoryClient }) +const mem0Hosted = createMem0MemoryAdapter({ + client: mem0Client, + mode: 'hosted', + backendRef: 'mem0:production-account', +}) + +const mem0Oss = createMem0MemoryAdapter({ + client: mem0OssClient, + mode: 'oss', + backendRef: 'mem0:local-postgres', +}) + +const graphiti = createGraphitiMemoryAdapter({ + client: graphitiMcpClient, + backendRef: 'graphiti:production-cluster', +}) + +const neo4j = createNeo4jAgentMemoryAdapter({ + client: neo4jMemoryClient, + transport: 'rest', + contextMode: 'search', +}) +``` + +The Graphiti defaults match the current official server tools: `add_memory`, `search_memory_facts`, `search_nodes`, `get_episodes`, and `clear_graph`. +Set `toolNames` explicitly when your server lists different names. +The adapter never retries a write under another name because that could enqueue it twice. +Visible writes expand the episode scan when Graphiti truncates a long group and fail explicitly at `episodeScanLimit` instead of reporting a false success. +Hosted Mem0 follows the synchronous memory-array response from `mem0ai` 3.x and rejects legacy asynchronous event responses instead of treating them as successful writes. +Mem0 cleanup requires `getAll` plus per-memory `delete`, refuses an empty identity scope, and waits until both list and search results stop returning deleted IDs. +Every Mem0 operation requires `userId`, `agentId`, `runId`, or `namespace`. +`appId`, tenant, team, session, and tag filters refine that identity but cannot replace it. +Fresh-write probes expire after `ingestionTimeoutMs`, so a long-lived adapter retains only its current visibility window rather than its lifetime write history. +The adapter does not call Mem0's account-wide `deleteAll` method. +Use a stable, non-secret `backendRef` in Mem0 and Graphiti adapter identities so caches and dispatch keys cannot alias two accounts or clusters. + +Each adapter uses the same `search`, `getContext`, and `write` method shape, but provider capabilities remain explicit. + +| Adapter | Searchable through this adapter | Writable through this adapter | +| --- | --- | --- | +| Mem0 hosted or OSS | messages, observations, facts, entities, preferences, reasoning traces | all six kinds | +| Graphiti MCP | extracted facts and entities | all six kinds as episodes | +| Neo4j hosted REST | messages, observations, entities | messages, observations, entities, conversation reasoning steps | +| Neo4j bridge | messages, observations, entities, preferences, similar reasoning traces | all six kinds | + +Neo4j bridge facts are graph writes. +The official `@neo4j-labs/agent-memory` 0.4 API has no corresponding fact-search method, so fact-only retrieval experiments should use another provider or a custom Neo4j query adapter. +Mem0 and Graphiti apply tenant, user, agent, team, run, session, namespace, and tag filters per request, subject to Mem0's required provider identity above. +Neo4j Agent Memory experiments require disposable external state per branch because the SDK cannot clear every memory kind atomically. +On a shared Neo4j client, the adapter rejects scope fields the selected SDK method cannot enforce before making a provider call. +Conversation messages can use `sessionId`, and reasoning writes can use either `sessionId` or `runId`; other direct operations are unscoped unless a dedicated client is used. +Pass `branchId` only after the caller has provisioned one physically isolated instance for that exact branch. +Provider-specific identifiers remain internal to the adapter. +Custom adapters must declare `branchIsolation: { mode: 'scoped' }` only when every operation enforces the supplied scope; undeclared adapters are rejected by branch execution. + +```ts +const neo4jInstances = new WeakMap() + +const neo4jCandidate = { + id: 'neo4j', + ref: 'neo4j-agent-memory:0.4.0:rest', + policy: { read: ['shared'], write: 'shared' }, + async createAdapter({ branchId, purpose, signal, markExternalCall }: { + branchId: string + purpose: 'execute' | 'recovery' + signal: AbortSignal + markExternalCall(): void + }) { + markExternalCall() + const client = purpose === 'recovery' + ? await openDedicatedNeo4jMemory({ branchId, transport: 'rest', signal }) + : await provisionDedicatedNeo4jMemory({ branchId, transport: 'rest', signal }) + if (!client) return null + const adapter = createNeo4jAgentMemoryAdapter({ client, transport: 'rest', branchId }) + neo4jInstances.set(adapter, branchId) + return adapter + }, + async disposeAdapter(adapter: object) { + const branchId = neo4jInstances.get(adapter) + if (!branchId) throw new Error('unknown Neo4j branch') + await destroyDedicatedNeo4jMemory({ branchId }) + }, +} +``` + +The adapter requires `transport: 'rest' | 'bridge'` and rejects unsupported searches or writes before making a provider call. +`contextMode: 'search'` is the default and preserves query-based behavior across providers. +Set `contextMode: 'native'` with hosted REST to use Neo4j's whole-conversation context for message and observation memory. + +Use a branch when agents or candidate configurations must not share state accidentally: + +```ts +import { + createAgentMemoryBranch, + createMem0MemoryAdapter, +} from '@tangle-network/agent-knowledge/memory' + +const memory = createAgentMemoryBranch({ + adapter: mem0Oss, + branchId: 'candidate-17', + policy: { read: ['private', 'team'], write: 'team' }, + baseScope: { tenantId: 'acme', teamId: 'research' }, +}) -const context = await memory.getContext('What does this user prefer?', { - scope: { userId: 'user-123', sessionId: 'session-456' }, +await memory.write({ + kind: 'fact', + text: 'The launch date is Friday.', + scope: { agentId: 'researcher' }, +}) + +const context = await memory.getContext('When is launch?', { + scope: { agentId: 'builder' }, limit: 5, }) -const sourceRecords = context.hits.map((hit) => - memoryHitToSourceRecord(hit, { scope: { userId: 'user-123' } }), -) +const snapshot = await memory.snapshot() +await memory.close?.() +const reopenedMem0Oss = createMem0MemoryAdapter({ + client: mem0OssClient, + mode: 'oss', + id: snapshot.adapterId, +}) +const resumed = createAgentMemoryBranch({ + adapter: reopenedMem0Oss, + branchId: 'candidate-17', + snapshot, +}) +``` + +Private writes stay visible only to one agent. +Team writes are shared inside one team. +Shared writes are visible to every agent in the branch. +`fork()` replays accepted writes into a child branch, including a branch backed by a different provider. +One adapter object can have only one live handle for a given branch ID in a process. +Close the prior handle before resuming it, and use the run lease to prevent distributed controllers from owning the same durable run at once. + +Use `runAgentMemoryExperiment()` to compare known providers or configurations across ordered histories. +`buildAgentMemorySequencesFromBenchmarkCases()` feeds existing LoCoMo, LongMemEval, GroupMemBench, or first-party memory cases into the same runner without changing their event order. +Use `runAgentMemoryImprovement()` when a proposer should search configurations, measure them on training histories, compare the winner with the baseline on fresh histories, and activate it only after a statistically supported win. + +```ts +const result = await runAgentMemoryImprovement({ + experimentId: 'support-memory-v3', + trainSequences, + holdoutSequences, + seeds: [ + { + config: currentConfig, + track: 'conservative', + proposer: 'conservative', + vision: 'Improve recall without increasing stale-memory reuse.', + }, + { + config: graphConfig, + track: 'graph', + proposer: 'graph', + vision: 'Improve temporal and multi-actor reasoning.', + }, + ], + proposer: configProposer, + proposers: { + conservative: conservativeConfigProposer, + graph: graphConfigProposer, + }, + improvementRef: 'memory-stack@8f2c1a7', + budget: { maxSteps: 12 }, + maxTotalCostUsd: 25, + candidateConcurrency: 4, + sequenceConcurrency: 8, + createCandidate: ({ config }) => buildMemoryCandidate(config), + executeStep: ({ memory, step, context }) => + runProfileStep({ memory, step, context }), + executeStepRef: 'support-profile@8f2c1a7', + runDir: '.agent-knowledge/memory-runs/support-v3', + activation: { + ref: 'config-store:support-memory:v1', + readCurrent: () => configStore.read('support-memory'), + compareAndSet: ({ + activationId, + expectedSurfaceHash, + config, + surfaceHash, + }) => configStore.compareAndSet({ + key: 'support-memory', + operationId: activationId, + expectedHash: expectedSurfaceHash, + nextHash: surfaceHash, + value: config, + }), + }, +}) ``` -The Neo4j adapter is runtime dependency-free: pass the real -`@neo4j-labs/agent-memory` client in products, or a fake client in tests. CI -typechecks against `@neo4j-labs/agent-memory@0.4.0` and covers the published -TypeScript SDK surface: `shortTerm.addMessage/searchMessages/getContext`, -`longTerm.addEntity/addPreference/addFact/searchEntities/searchPreferences`, -and `reasoning.getSimilarTraces`. Generic `search` / `getContext` and -snake_case bridge-style methods remain supported for non-hosted clients. +`candidateConcurrency` controls how many configurations are measured at once. +`sequenceConcurrency` controls how many independent histories run at once. +Every experiment candidate must set `ref` to a version or commit that changes whenever its provider configuration or adapter behavior changes. +The first seed is always the deployed baseline. +Each seed starts an independent track with its own objective and optional proposer, and a branch inherits its parent track's proposer unless the governor chooses another one. +Candidate factories should perform local construction only when possible. +When a dedicated instance must be provisioned or reconnected, call `markExternalCall()` immediately before that action. +Set each candidate's `externalCostUsdPerSequence` to a conservative memory-provider charge; it is counted once an adapter method or a marked factory action attempts provider work, while a local factory failure costs zero. +Set `externalRecoveryCostUsdPerAttempt` when reconnecting and deleting an interrupted history adds a separate provider charge. +Recovery uses the same durable run-wide cost account as candidate execution, proposers, profile steps, and the governor, so a resumed run cannot exceed `maxTotalCostUsd` by treating cleanup as free work. +`runAgentMemoryExperiment()` reports all attempt and recovery spend in `totalCostUsd`, with retired-candidate cleanup separated as `unrankedRecoveryCostUsd`. +Cleanup completion is journaled before a paid-call receipt. +If a process exits between those writes, resume charges the reserved maximum without repeating provider cleanup and continues only after the account is complete. +The default `maxTotalCostUsd` is zero, so model and paid provider calls require an explicit limit. +Model calls inside `executeStep` should use `context.cost.runPaidCall()` so `maxTotalCostUsd` counts both sources before starting more work. +The same budget is supplied to named proposers as `context.costLedger` and to the governor as `context.costLedger`; the standard `agent-eval` proposers charge their model calls there. +Steps inside one history remain ordered, and writes from one agent remain ordered while independent agents can write concurrently. +Runtime callbacks may write only to scopes declared on a step, write, probe, or `sequence.cleanupScopes`; this lets an interrupted retry clear every actor and session it could have touched. +Search never changes the live configuration. +Only `activation.compareAndSet` can activate the fresh-history winner. +The activation target must provide an exact current read and an atomic compare-and-set from the measured baseline to the measured winner. +The durable activation journal is written before that operation. +After a crash, the runner reads the live configuration and records the completed activation without applying it twice. +Change `activation.ref` whenever the target or activation behavior changes. +The default filesystem path permits one controller and many parallel workers, persists candidate history, and resumes without adding more than `maxSteps` candidate nodes. +Custom storage must set `controllerMode: 'process-local'` when all controllers share one process, or provide `acquireRunLease` for distributed controllers. +Independent workers still use `candidateConcurrency` and `sequenceConcurrency`. +Every measured cell receives a fresh provider branch ID for that execution attempt. +This prevents a late asynchronous write from a crashed worker from entering a retried cell. +`experimentRunId` keeps the logical experiment identity stable across workers without reusing physical branch state. +Timed-out histories enter bounded branch cleanup before `runAgentMemoryExperiment()` returns. +The run waits for cleanup to succeed or for `cleanupTimeoutMs` to expire, and either cleanup failure leaves the attempt available for recovery. +The runner appends a `started` event before adapter creation and a `cleaned` event only after provider cleanup and disposal succeed. +On resume it recovers every unfinished attempt before starting new cells. +Independent unfinished attempts recover up to `maxConcurrency` at once, and no new cell starts until every cleanup succeeds. +Recovery refuses more than `maxRecoveryAttempts`, which defaults to 1000. +Recovery also refuses a fourth failed cleanup for the same attempt by default. +Set `maxRecoveryRetriesPerAttempt` to change that limit; its durable retry record is returned as `recoveryLogPath`. +`runAgentMemoryImprovement()` forwards the same option to every training and fresh-history experiment. +Pass retired implementations through `recoveryCandidates` until their recorded attempts are cleaned. +`createAdapter` receives `purpose: 'recovery'` during that pass so a dedicated-instance candidate can reconnect to the existing branch instead of provisioning another one. +It may return `null` during recovery when the recorded attempt never created external state. +Hosted Mem0 and Graphiti recovery waits for their configured ingestion timeout before deletion so an accepted asynchronous write can become visible first. +That visibility wait is bounded by `cleanupTimeoutMs`; a longer required delay fails before new provider work starts. +The default filesystem storage survives process restarts and uses an OS lock. +Custom durable storage must implement atomic `append`; process-local custom storage opts in with `controllerMode: 'process-local'`, and distributed controllers provide `acquireRunLease`. +Provider SDK calls should also have their own network timeout because a caller-side cleanup timeout cannot cancel an arbitrary third-party promise. +Candidate factories receive an `AbortSignal` and must pass it into provider connection or provisioning calls. +The persisted run identity includes `budget.maxSteps`; start a new `runDir` to enlarge a completed search instead of mutating its history. +Promotion is blocked when a configured critical dimension is missing or incomplete on either fresh-history arm. +Improvement runs clear each scoped branch before and after a measured history by default, including timeout and thrown-error cleanup. +Set `cleanupBranches: false` only when the candidate creates and disposes a physically isolated provider instance per cell. +The current Neo4j SDK has no complete all-memory clear operation, so a Neo4j improvement run must use disposable per-cell service instances with `cleanupBranches: false` and `disposeAdapter`; a persistent shared workspace is rejected because failed retries could read partial state. +If recovery cannot confirm deletion, the resumed run stops before launching new provider work and leaves the attempt active for another retry. + +The package stays independent of `agent-runtime`. +The `executeStep` callback is where a runtime profile, coding agent, research agent, or deterministic worker uses the branch. +This keeps provider and measurement code reusable while the profile owns what agents do and what they choose to remember. + +The Neo4j adapter accepts the real `@neo4j-labs/agent-memory` client and rejects unsafe branch reuse unless `branchId` identifies caller-provisioned isolated state. +The Mem0 adapter typechecks against `mem0ai` hosted and open-source clients and accepts only the current hosted SDK response shape. +The Graphiti adapter calls the official MCP tools and waits for queued episodes to become visible before evaluating them by default. +Hosted Mem0 and Graphiti visible writes are safe only in `lifetime: 'attempt'` branches because their writes may outlive a worker process. +`runAgentMemoryExperiment()` creates those fresh attempt branches automatically. +Attempt snapshots are audit and fork inputs, not restart points; use `forkAgentMemoryBranchSnapshot()` to replay one into a fresh branch. +Mem0 OSS and caller-provisioned disposable instances can use resumable branches. +Graphiti `consistency: 'queued'` is available only for direct ingestion and is rejected by branch execution. ## Research Loop @@ -549,11 +815,11 @@ with its own pass (`driverResearches: true`), and gates on the readiness check. Both are yours (no creds). The loop owns the deterministic mechanics (indexing, applying write blocks, scoring readiness) and stops once no blocking gap remains. -Does the verifying driver actually earn its keep? See -[docs/two-agent-research-ab.md](docs/two-agent-research-ab.md) for an equal-compute -A/B (9 ML topics, `glm-5.2`): the two-agent loop admits ~2.33 fewer sources per topic -at identical coverage, though most of that win is de-duplication, not relevance -filtering. Honest caveats and how to reproduce included. +An equal-compute comparison across 9 ML topics using `glm-5.2` is documented in +[docs/two-agent-research-ab.md](docs/two-agent-research-ab.md). +The two-agent loop admitted about 2.33 fewer sources per topic at identical coverage. +Most of that difference came from de-duplication rather than relevance filtering. +The document includes limitations and reproduction steps. ```ts import { diff --git a/package.json b/package.json index 8be44de..548577d 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@tangle-network/agent-knowledge", - "version": "3.2.1", + "version": "4.0.0", "description": "Source-grounded, eval-gated knowledge growth primitives for agents.", "homepage": "https://github.com/tangle-network/agent-knowledge#readme", "repository": { @@ -52,6 +52,7 @@ "dist", "docs", "AGENTS.md", + "CHANGELOG.md", "README.md" ], "publishConfig": { @@ -69,19 +70,20 @@ "verify:package": "node scripts/verify-package.mjs" }, "dependencies": { - "@tangle-network/agent-eval": "^0.122.7", + "@tangle-network/agent-eval": "^0.122.8", "@tangle-network/agent-interface": "^0.31.0", "proper-lockfile": "4.1.2", - "zod": "^4.3.6" + "zod": "^4.4.3" }, "devDependencies": { - "@biomejs/biome": "^2.4.15", + "@biomejs/biome": "^2.5.4", "@neo4j-labs/agent-memory": "0.4.0", "@tangle-network/sandbox": "^0.9.7", "@types/node": "^25.6.0", "@types/proper-lockfile": "4.1.4", + "mem0ai": "3.1.0", "tsup": "^8.0.0", - "tsx": "^4.22.4", + "tsx": "^4.23.1", "typescript": "^5.7.0", "vitest": "^3.0.0" }, diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 4a0c904..78194ea 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -13,8 +13,8 @@ importers: .: dependencies: '@tangle-network/agent-eval': - specifier: ^0.122.7 - version: 0.122.7(typescript@5.9.3) + specifier: ^0.122.8 + version: 0.122.8(typescript@5.9.3) '@tangle-network/agent-interface': specifier: ^0.31.0 version: 0.31.0 @@ -22,42 +22,48 @@ importers: specifier: 4.1.2 version: 4.1.2 zod: - specifier: ^4.3.6 - version: 4.4.2 + specifier: ^4.4.3 + version: 4.4.3 devDependencies: '@biomejs/biome': - specifier: ^2.4.15 - version: 2.4.15 + specifier: ^2.5.4 + version: 2.5.4 '@neo4j-labs/agent-memory': specifier: 0.4.0 version: 0.4.0 '@tangle-network/sandbox': specifier: ^0.9.7 - version: 0.9.7(viem@2.48.8(typescript@5.9.3)(zod@4.4.2)) + version: 0.9.7(viem@2.48.8(typescript@5.9.3)(zod@4.4.3)) '@types/node': specifier: ^25.6.0 version: 25.6.0 '@types/proper-lockfile': specifier: 4.1.4 version: 4.1.4 + mem0ai: + specifier: 3.1.0 + version: 3.1.0(@anthropic-ai/sdk@0.40.1)(@azure/identity@4.13.1)(@azure/search-documents@12.2.0)(@cloudflare/workers-types@4.20260702.1)(@google/genai@1.52.0)(@langchain/core@1.2.3(@opentelemetry/api@1.9.1)(ws@8.21.0))(@mistralai/mistralai@1.15.1)(@qdrant/js-client-rest@1.18.0(typescript@5.9.3))(@supabase/supabase-js@2.110.7)(@types/jest@29.5.14)(@types/pg@8.11.0)(better-sqlite3@12.11.1)(cloudflare@4.5.0)(compromise@14.16.0)(groq-sdk@0.3.0)(mongodb@7.2.0)(natural@8.1.1(@opentelemetry/api@1.9.1))(ollama@0.5.18)(pg@8.22.0)(redis@5.12.1(@opentelemetry/api@1.9.1))(ws@8.21.0) tsup: specifier: ^8.0.0 - version: 8.5.1(postcss@8.5.13)(tsx@4.22.4)(typescript@5.9.3)(yaml@2.8.4) + version: 8.5.1(postcss@8.5.13)(tsx@4.23.1)(typescript@5.9.3)(yaml@2.8.4) tsx: - specifier: ^4.22.4 - version: 4.22.4 + specifier: ^4.23.1 + version: 4.23.1 typescript: specifier: ^5.7.0 version: 5.9.3 vitest: specifier: ^3.0.0 - version: 3.2.4(@types/node@25.6.0)(tsx@4.22.4)(yaml@2.8.4) + version: 3.2.4(@types/node@25.6.0)(tsx@4.23.1)(yaml@2.8.4) packages: '@adraffy/ens-normalize@1.11.1': resolution: {integrity: sha512-nhCBV3quEgesuf7c7KYfperqSS14T8bYuvJ8PcLJp6znkZpFc0AuW4qBtr8eKVyPPe/8RSr7sglCWPU5eaxwKQ==} + '@anthropic-ai/sdk@0.40.1': + resolution: {integrity: sha512-DJMWm8lTEM9Lk/MSFL+V+ugF7jKOn0M2Ujvb5fN8r2nY14aHbGPZ1k6sgjL+tpJ3VuOGJNG+4R83jEpOuYPv8w==} + '@asteasolutions/zod-to-openapi@8.5.0': resolution: {integrity: sha512-SABbKiObg5dLRiTFnqiW1WWwGcg1BJfmHtT2asIBnBHg6Smy/Ms2KHc650+JI4Hw7lSkdiNebEGXpwoxfben8Q==} peerDependencies: @@ -72,59 +78,132 @@ packages: zod: optional: true - '@biomejs/biome@2.4.15': - resolution: {integrity: sha512-j5VH3a/h/HXTKBM50MDMxRCzkeLv9S2XJcW2WgnZT1+xyisi+0bISrXR82gCX+8S9lvK0skEvHJRN+3Ktr2hlw==} + '@azure/abort-controller@2.2.0': + resolution: {integrity: sha512-fNAjWnA/nZ2jz31kxR/AqRaUT8ewHBw/WuBIosK0moMy1C9e5ValbDfFdIxJzVOOYaYkV/b2F1S4H/aHiqfVQg==} + engines: {node: '>=22.0.0'} + + '@azure/core-auth@1.11.0': + resolution: {integrity: sha512-IUZydyTUkDnYdstOW9pFOOUQlBjAepK5teihDE3x6yxsPJs/hsAaaYpeGxdxrgtOiJbBKSjKW7MDk7AEhb4LRg==} + engines: {node: '>=22.0.0'} + + '@azure/core-client@1.11.0': + resolution: {integrity: sha512-JjQWO6akOck45PH/XBrxzsQGAiKrfFl4m5iggJ0ItMIz5omRufOXWpqCPpdjKN3vKDzlSUvFjaMb7Zwf0gvAdA==} + engines: {node: '>=22.0.0'} + + '@azure/core-http-compat@2.5.0': + resolution: {integrity: sha512-BoSmXPx2er1Ai+wKlDvj29jIQespCNBwEmKyZVHO2kEFsWbGjAjwMCGzug3DJM5/QYIV3vej0S1zcU5bq9fa8w==} + engines: {node: '>=22.0.0'} + peerDependencies: + '@azure/core-client': ^1.10.0 + '@azure/core-rest-pipeline': ^1.22.0 + + '@azure/core-paging@1.7.0': + resolution: {integrity: sha512-7GEAoIsaoBr6KELNRb8nypowCqvk8dnCHFCYg4XD4lOQGY2GqjQg5IhkRjyBFRO18CGSMq05PaNqSOE9GQro3g==} + engines: {node: '>=22.0.0'} + + '@azure/core-rest-pipeline@1.25.0': + resolution: {integrity: sha512-bMs8ekJLjX8wPV+9IPBges1SLPyuDtE9g5gLDWOpxzKcoOFQnpLGkbcT1tdw3FaAmDS1gnPmMmJ6y/T5B96kIA==} + engines: {node: '>=22.0.0'} + + '@azure/core-tracing@1.4.0': + resolution: {integrity: sha512-eGwxD0AtncrxeBM4tG8R55Pc3rdX1hNW2WibJAgYpCVA6E93mvvVH+LcssoVjOBrSKWS55yEIHsk0X8ctHmfOQ==} + engines: {node: '>=22.0.0'} + + '@azure/core-util@1.14.0': + resolution: {integrity: sha512-9n2pWK61veAuN0V20t9lOuoV4CFMdyAZ1ygZzvBGk/pBBJRib/PjL9PLXa/aI2CcPpyHfqVsxxqLCYl6uZlfDw==} + engines: {node: '>=22.0.0'} + + '@azure/identity@4.13.1': + resolution: {integrity: sha512-5C/2WD5Vb1lHnZS16dNQRPMjN6oV/Upba+C9nBIs15PmOi6A3ZGs4Lr2u60zw4S04gi+u3cEXiqTVP7M4Pz3kw==} + engines: {node: '>=20.0.0'} + + '@azure/logger@1.4.0': + resolution: {integrity: sha512-rbAE25KUfjU/s3XHUdJgceoCP5dEOpMx85J04kF+QMdta73XkuG9JGHHinch+XIoKpBdqljin+KqURpJriSzLA==} + engines: {node: '>=22.0.0'} + + '@azure/msal-browser@5.17.1': + resolution: {integrity: sha512-zBhRGzABKSI7hfWh5EaZmril5ybZ7imBN1qEZl5sDTaelr+l8SnPjZO50Q4dnKnm347YPIlBMSnXKZyh3Yu5DQ==} + engines: {node: '>=0.8.0'} + + '@azure/msal-common@16.11.2': + resolution: {integrity: sha512-yDhtBOGDCdK9ipQ9g3+wmlMEPnZx2pXaDicDd9jYyR1L+7lEbvEohTDmF5qejZDutZY3m9pWPxeYxzNC701A2w==} + engines: {node: '>=0.8.0'} + + '@azure/msal-node@5.4.1': + resolution: {integrity: sha512-yqgoyOIMCH7TNaSLMBTP+4LUlbMMf1zgC8nzOFG95lmW82CmsAEtUT0J93e4BdqDcnX5qle/9X+yb7A8Mw9M0g==} + engines: {node: '>=20'} + + '@azure/search-documents@12.2.0': + resolution: {integrity: sha512-4+Qw+qaGqnkdUCq/vEFzk/bkROogTvdbPb1fmI8poxNfDDN1q2WHxBmhI7CYwesrBj1yXC4i5E0aISBxZqZi0g==} + engines: {node: '>=20.0.0'} + + '@babel/code-frame@7.29.7': + resolution: {integrity: sha512-Aup7aUOfpbAUg2ROOJN6Iw5f9DMBlzu0mIkm/malLQFN/YQgO48wCj0Kxa3sEHJvPVFg7siR+qRInwXd2qhQKw==} + engines: {node: '>=6.9.0'} + + '@babel/helper-validator-identifier@7.29.7': + resolution: {integrity: sha512-qehxGkRj55h/ff8EMaJ+cYhyaKlHIxqYDn682wQD7RNp9UujOQsHog2uS0r2vzr4pW+sXf90NeeayjcNaX3fFg==} + engines: {node: '>=6.9.0'} + + '@biomejs/biome@2.5.4': + resolution: {integrity: sha512-xy5FNE5kQJKyK5MR1gJy6ztXYx4WBAbYGlK04lMEgmyPRWKybY9NFwiG9yo0XdzOU8Xvhj41u034J1ywfoWfMw==} engines: {node: '>=14.21.3'} hasBin: true - '@biomejs/cli-darwin-arm64@2.4.15': - resolution: {integrity: sha512-rF3PPqLq1yoST79zaQbDjVJwsuIeci/O+9bgNmC5QpgOqz6aqYuzA4abyAGx+mgyiDXn4A049xAN8gijbuR1Qg==} + '@biomejs/cli-darwin-arm64@2.5.4': + resolution: {integrity: sha512-4o3NFRobXHynkgcFVrlZsoDAFtF2ldlEGN8sORSws5ZQqyY4PXnPUIylu4ksfyHuwkfvDREuWh3JK+niRwGq3w==} engines: {node: '>=14.21.3'} cpu: [arm64] os: [darwin] - '@biomejs/cli-darwin-x64@2.4.15': - resolution: {integrity: sha512-/5KHXYMfSJs1fNXiX30xFtI8JcCFV6zaVVLxOa0M2sfqBKHkpQhRTv94yxQWxeTY2lzo2OuTlNvPC+hDQt2wcQ==} + '@biomejs/cli-darwin-x64@2.5.4': + resolution: {integrity: sha512-D32P5HkU2Y6PySuC/WsVDTOgsDwVFmujzhhhOQjajtATpVWFDXuVd3oRbsWNSEA+aaFzyzZm22szsyydBYlSyQ==} engines: {node: '>=14.21.3'} cpu: [x64] os: [darwin] - '@biomejs/cli-linux-arm64-musl@2.4.15': - resolution: {integrity: sha512-ZPcxznxm0pogHBLZhYntyR3sR+MrZjqJIKEr7ZqVen0Rl+P/4upVmfYXjftizi9RoqZntg33fv/1fbdhbYXpEQ==} + '@biomejs/cli-linux-arm64-musl@2.5.4': + resolution: {integrity: sha512-Rpm5/AT1m+DlJmUoYvS4/vXc+0tXJPJ2NQz25TGPyHVF5JrWy75PE0GH6kVxsKtQDuCH4OgzquZq0R4kj/wCVg==} engines: {node: '>=14.21.3'} cpu: [arm64] os: [linux] - '@biomejs/cli-linux-arm64@2.4.15': - resolution: {integrity: sha512-owaAMZD/T4LrD0ELNCk0Km3qrRHuM0X6EAyVE1FSqGY0rbLoiDLrO4Us2tllm6cAeB2Ioa9C2C08NZPdr8+0Ug==} + '@biomejs/cli-linux-arm64@2.5.4': + resolution: {integrity: sha512-pSEfW7B8kTsXUjUxC1xVVK+y85Ht3C5XxZ9gclmC7/3Ku9Vqz8jmI7k0p/BNIjQ6t4sFERI2sFeH73ybiZl6YQ==} engines: {node: '>=14.21.3'} cpu: [arm64] os: [linux] - '@biomejs/cli-linux-x64-musl@2.4.15': - resolution: {integrity: sha512-CNq/9W38SYSH023lfcQ4KKU8K0YX8T//FZUhcgtMMRABDojx5XsMV7jlweAvGSl389wJQB29Qo6Zb/a+jdvt+w==} + '@biomejs/cli-linux-x64-musl@2.5.4': + resolution: {integrity: sha512-aby/PohmmgbShcHqFsZVzG8H6D98+P+A6xRWRrQcLW1pCjabcov5UUlke4UqNQBYTkDQav+jB4zyyDDeKB2GaA==} engines: {node: '>=14.21.3'} cpu: [x64] os: [linux] - '@biomejs/cli-linux-x64@2.4.15': - resolution: {integrity: sha512-0jj7THz12GbUOLmMibktK6DZjqz2zV64KFxyBtcFTKPiiOIY0a7vns1elpO1dERvxpsZ5ik0oFfz0oGwFde1+g==} + '@biomejs/cli-linux-x64@2.5.4': + resolution: {integrity: sha512-FNxojWJkL7EajAuzBgoLe0T2G0y112M4lBrDIFl/DomFTx8yqenYOIdsRLNXvOvBBofE8hJi85LjzLmBDpY7/Q==} engines: {node: '>=14.21.3'} cpu: [x64] os: [linux] - '@biomejs/cli-win32-arm64@2.4.15': - resolution: {integrity: sha512-ouhkYdlhp/1GghEJPdWwD/Vi3gQ1nFxuSpMolWsbq3Lsq3QUR4jl6UdhhscdCugKU5vOEuMiJhvKj66O0OCq+w==} + '@biomejs/cli-win32-arm64@2.5.4': + resolution: {integrity: sha512-emoXexPZIPAZkz2RKmA95WJUqK3I5MJNYtwEbL5ESciRzhmFMMyekDhNG8hpeOaK+ZGRDxAU4wvGuA5IHQ0h0w==} engines: {node: '>=14.21.3'} cpu: [arm64] os: [win32] - '@biomejs/cli-win32-x64@2.4.15': - resolution: {integrity: sha512-zBrGq5mx5wwpnow4+2BxUvleDM+GNd4sLbPaMapsSLQLD0NGRCquqPBTgN+7XkUteHvj7M+BstuI8tmnV7+HgQ==} + '@biomejs/cli-win32-x64@2.5.4': + resolution: {integrity: sha512-U1jaluLw1qQc2Tx7/CeSoL9N5XcqIH+GWjpUAy1ouB5nVjSCMNO+NNHdY3RAs8zxNurLWAdj6pehQdCA2zyU+Q==} engines: {node: '>=14.21.3'} cpu: [x64] os: [win32] + '@cfworker/json-schema@4.1.1': + resolution: {integrity: sha512-gAmrUZSGtKc3AiBL71iNWxDsyUC5uMaKKGdvzYsBoTW/xi42JQHl7eKV2OYzCUqvc+D2RCcf7EXY2iCyFIk6og==} + + '@cloudflare/workers-types@4.20260702.1': + resolution: {integrity: sha512-mOhf5TUEB1m2vPrxtqoIGfz0fUC9xyxRDx5gWHy5s+OCo6dcV+g7wI1R7gYCMFohhqF/2y2xeKVwMwCJjfn/WA==} + '@esbuild/aix-ppc64@0.27.7': resolution: {integrity: sha512-EKX3Qwmhz1eMdEJokhALr0YiD0lhQNwDqkPYyPhiSwKrh7/4KRjQc04sZ8db+5DVVnZ1LmbNDI1uAMPEUBnQPg==} engines: {node: '>=18'} @@ -437,12 +516,33 @@ packages: cpu: [x64] os: [win32] + '@google/genai@1.52.0': + resolution: {integrity: sha512-gwSvbpiN/17O9TbsqSsE/OzZcpv5Fo4RQjdngGgogtuB9RsyJ8ZHhX5KjHj1bp5N9snN2eK8LDGXSaWW2hof8Q==} + engines: {node: '>=20.0.0'} + peerDependencies: + '@modelcontextprotocol/sdk': ^1.25.2 + peerDependenciesMeta: + '@modelcontextprotocol/sdk': + optional: true + '@hono/node-server@2.0.1': resolution: {integrity: sha512-jI9yMDyFpqBeSighf/zlXnQG/nl9AyBc6aAgy4XtxJMyt/CNyJpvPfzDD+bCc2zAOmhhqtF6TnmIaY+xV4mIrw==} engines: {node: '>=20'} peerDependencies: hono: 4.12.30 + '@jest/expect-utils@29.7.0': + resolution: {integrity: sha512-GlsNBWiFQFCVi9QVSx7f5AgMeLxe9YCCs5PuP2O2LdjDAA8Jh9eX7lA1Jq/xdXw3Wb3hyvlFNfZIfcRetSzYcA==} + engines: {node: ^14.15.0 || ^16.10.0 || >=18.0.0} + + '@jest/schemas@29.6.3': + resolution: {integrity: sha512-mo5j5X+jIZmJQveBKeS/clAueipV7KgiX1vMgCxam1RNYiqE1w62n0/tJJnHtjW8ZHcQco5gY85jA3mi0L+nSA==} + engines: {node: ^14.15.0 || ^16.10.0 || >=18.0.0} + + '@jest/types@29.6.3': + resolution: {integrity: sha512-u3UPsIilWKOM3F9CXtrG8LEJmNxwoCQC/XVj4IKYXvvpx7QIi/Kg1LI5uDmDpKlac62NUtX7eLjRh+jVZcLOzw==} + engines: {node: ^14.15.0 || ^16.10.0 || >=18.0.0} + '@jridgewell/gen-mapping@0.3.13': resolution: {integrity: sha512-2kkt/7niJ6MgEPxF0bYdQ6etZaA+fQvDcLKckhy1yIQOzaoKjBBjSj63/aLVjYE3qhRt5dvM+uUyfCg6UKCBbA==} @@ -456,6 +556,16 @@ packages: '@jridgewell/trace-mapping@0.3.31': resolution: {integrity: sha512-zzNR+SdQSDJzc8joaeP8QQoCQr8NuYx2dIIytl1QeBEZHJ9uW6hebsrYgbz8hJwUQao3TWCMtmfV8Nu1twOLAw==} + '@langchain/core@1.2.3': + resolution: {integrity: sha512-F+L5SsciykwDl7eDxacnhDTcWe1IF6jetzfkvI5PPfq6ogWHO7xcjU90SGh/3lqbbS0tgun+qF01KIqxawrCsA==} + engines: {node: '>=20'} + + '@mistralai/mistralai@1.15.1': + resolution: {integrity: sha512-fb995eiz3r0KsBGtRjFV+/iLbX+UpfalxpF+YitT3R6ukrPD4PN+FGwwmYcRFhNAzVzDUtTVxQYnjQWEnwV5nw==} + + '@mongodb-js/saslprep@1.4.12': + resolution: {integrity: sha512-QAfAMwNgnYxZ2C6D1HgeP7Gc4i/uvJRim415PCIL9ptRxWMNbWeLBYb2/9R4pGKny/s1FVu2JA2cxCUBUOggrA==} + '@neo4j-labs/agent-memory@0.4.0': resolution: {integrity: sha512-iKUi+STQm0DQqDUDx0cR486RTMCHlPfmmHBqjyN/gaVXSBlaTzYaLp63D5O/qP5tDc4E4iqiMfMITLUabllTaw==} engines: {node: '>=20.0.0'} @@ -484,6 +594,79 @@ packages: resolution: {integrity: sha512-gLyJlPHPZYdAk1JENA9LeHejZe1Ti77/pTeFm/nMXmQH/HFZlcS/O2XJB+L8fkbrNSqhdtlvjBVjxwUYanNH5Q==} engines: {node: '>=8.0.0'} + '@protobufjs/aspromise@1.1.2': + resolution: {integrity: sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ==} + + '@protobufjs/base64@1.1.2': + resolution: {integrity: sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg==} + + '@protobufjs/codegen@2.0.5': + resolution: {integrity: sha512-zgXFLzW3Ap33e6d0Wlj4MGIm6Ce8O89n/apUaGNB/jx+hw+ruWEp7EwGUshdLKVRCxZW12fp9r40E1mQrf/34g==} + + '@protobufjs/eventemitter@1.1.1': + resolution: {integrity: sha512-vW1GmwMZNnL+gMRaovlh9yZX74kc+TTU3FObkkurpMaRtBfLP3ldjS9KQWlwZgraRE0+dheEEoAxdzcJQ8eXZg==} + + '@protobufjs/fetch@1.1.1': + resolution: {integrity: sha512-GpptLrs57adMSuHi3VNj0mAF8dwh36LMaYF6XyJ6JMWlVsc+t42tm1HSEDmOs3A8fC9yyeisgLhsTVQokOZ0zw==} + + '@protobufjs/float@1.0.2': + resolution: {integrity: sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ==} + + '@protobufjs/path@1.1.2': + resolution: {integrity: sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA==} + + '@protobufjs/pool@1.1.0': + resolution: {integrity: sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw==} + + '@protobufjs/utf8@1.1.2': + resolution: {integrity: sha512-b1UQwcEZ4yCnMCD8DAL1VlbvBJE9/IX4FTIp7BG1xYpf29SLazLSrqUkj4w7Y5y7cCVP6E5tcqqcI0xemPkHug==} + + '@qdrant/js-client-rest@1.18.0': + resolution: {integrity: sha512-/0dqX5uV9chC1DnYSnU4gNMrDqse/pt6hHg3Rqqpl5isH7xl1xSNvffjzBoxycDD79luWn7Ho6Rh/61sOs5DNw==} + engines: {node: '>=18.17.0', pnpm: '>=8'} + peerDependencies: + typescript: '>=4.7' + + '@qdrant/openapi-typescript-fetch@1.2.6': + resolution: {integrity: sha512-oQG/FejNpItrxRHoyctYvT3rwGZOnK4jr3JdppO/c78ktDvkWiPXPHNsrDf33K9sZdRb6PR7gi4noIapu5q4HA==} + engines: {node: '>=18.0.0', pnpm: '>=8'} + + '@redis/bloom@5.12.1': + resolution: {integrity: sha512-PUUfv+ms7jgPSBVoo/DN4AkPHj4D5TZSd6SbJX7egzBplkYUcKmHRE8RKia7UtZ8bSQbLguLvxVO+asKtQfZWA==} + engines: {node: '>= 18.19.0'} + peerDependencies: + '@redis/client': ^5.12.1 + + '@redis/client@5.12.1': + resolution: {integrity: sha512-7aPGWeqA3uFm43o19umzdl16CEjK/JQGtSXVPevplTaOU3VJA/rseBC1QvYUz9lLDIMBimc4SW/zrW4S89BaCA==} + engines: {node: '>= 18.19.0'} + peerDependencies: + '@node-rs/xxhash': ^1.1.0 + '@opentelemetry/api': '>=1 <2' + peerDependenciesMeta: + '@node-rs/xxhash': + optional: true + '@opentelemetry/api': + optional: true + + '@redis/json@5.12.1': + resolution: {integrity: sha512-eOze75esLve4vfqDel7aMX08CNaiLLQS2fV8mpRN9NxPe1rVR4vQyYiW/OgtGUysF6QOr9ANhfxABKNOJfXdKg==} + engines: {node: '>= 18.19.0'} + peerDependencies: + '@redis/client': ^5.12.1 + + '@redis/search@5.12.1': + resolution: {integrity: sha512-ItlxbxC9cKI6IU1TLWoczwJCRb6TdmkEpWv05UrPawqaAnWGRu3rcIqsc5vN483T2fSociuyV1UkWIL5I4//2w==} + engines: {node: '>= 18.19.0'} + peerDependencies: + '@redis/client': ^5.12.1 + + '@redis/time-series@5.12.1': + resolution: {integrity: sha512-c6JL6E3EcZJuNqKFz+KM+l9l5mpcQiKvTwgA3blt5glWJ8hjDk0yeHN3beE/MpqYIQ8UEX44ItQzgkE/gCBELQ==} + engines: {node: '>= 18.19.0'} + peerDependencies: + '@redis/client': ^5.12.1 + '@rollup/rollup-android-arm-eabi@4.60.2': resolution: {integrity: sha512-dnlp69efPPg6Uaw2dVqzWRfAWRnYVb1XJ8CyyhIbZeaq4CA5/mLeZ1IEt9QqQxmbdvagjLIm2ZL8BxXv5lH4Yw==} cpu: [arm] @@ -627,14 +810,47 @@ packages: '@scure/bip39@2.2.0': resolution: {integrity: sha512-T/Bj/YvYMNkIPq6EENO6/rcs2e7qTNuyoUXf0KBFDmp0ZDu0H2X4Lq6yC3i0c8PcWkov5EbW+yQZZbdMmk154A==} + '@sinclair/typebox@0.27.12': + resolution: {integrity: sha512-hhyNJ+nbR6ZR7pToHvllEFun9TL0sbL+tk/ON75lo+Xas054uez98qRbsuNt7MBCyZKK4+8Yli/OAGZhmfBZ/g==} + + '@standard-schema/spec@1.1.0': + resolution: {integrity: sha512-l2aFy5jALhniG5HgqrD6jXLi/rUWrKvqN/qJx6yoJsgKhblVd+iqqU4RCXavm/jPityDo5TCvKMnpjKnOriy0w==} + + '@supabase/auth-js@2.110.7': + resolution: {integrity: sha512-M5Bpl4hCv6kHcOO/xM06Dyfg1mYLHljMkp1plhzG9IRZPc3czvyMsSN1XpL5+GKisOKM3lSN59zhpcm6sMVXfA==} + engines: {node: '>=22.0.0'} + + '@supabase/functions-js@2.110.7': + resolution: {integrity: sha512-megYmexlYEoR/0qlsr4Snh9wtzAodO7MAri3NMevZrXzNvQRKlvmTcSBoKGLQEPDakgDZMqbMdf9DwoZz6qfoA==} + engines: {node: '>=22.0.0'} + + '@supabase/phoenix@0.4.5': + resolution: {integrity: sha512-aAn9H9ovVyeApKy11OWOrrOGq8DV68yWeH4ud2lN9fzn4aO8Zb5GLL9m1pUg9nLqIcT+ZDfAcsZe0E/nqdv2lw==} + + '@supabase/postgrest-js@2.110.7': + resolution: {integrity: sha512-ban6YV0djhVaqVYezlOARKLIuOBSvLLhyQVZjA2nxPrtswhxHCl1+gI4giFgI9ATQAaMNbUZb4JXiuL5lEA/5g==} + engines: {node: '>=22.0.0'} + + '@supabase/realtime-js@2.110.7': + resolution: {integrity: sha512-AMtZjyFA2gsmjuxopPNS/sRznLQHG0Ht5x+ytTPTOh3vAcOTUlVRLx7gW4/CONNnbb3PKOkE+HmM35HOSbmomQ==} + engines: {node: '>=22.0.0'} + + '@supabase/storage-js@2.110.7': + resolution: {integrity: sha512-2tcDE8cjEDy1uKxKavBpKQod1JdMV1jDXQag48TCa+kycmJOltc0yVabC0BUlhOwAl6WykXU2aOsH3ELMtZrmQ==} + engines: {node: '>=22.0.0'} + + '@supabase/supabase-js@2.110.7': + resolution: {integrity: sha512-AnfO3A230Shy6RMO7cya3Wl1OcXnABJrzH8vP+fY7/RFjhzcchB7DjKkkTIAntlwekD+GkSFzEvt2tC+D4Fp8w==} + engines: {node: '>=22.0.0'} + '@tangle-network/agent-core@0.3.8': resolution: {integrity: sha512-bZfVpdiFjXbcQwSxSQABdtXEmUuapWChCcXrHkP6fAnwqEy9hWEfeZLlMZUy+1XaOfpTaMLUOEAwYhXfN018wQ==} '@tangle-network/agent-core@0.4.11': resolution: {integrity: sha512-5B1IjrJ8xDR7w8Hv/MSk2ixul6NEJQ5Ftzo+z6l7ipeFpc0yaf1+Kkml4HDUEmGW/rqdrNpBf9/MpT7i/SY0PA==} - '@tangle-network/agent-eval@0.122.7': - resolution: {integrity: sha512-uOYVynC/+BnlNqZsKRG8NXz/wqeGB/AUImOPb/juUDQMIDzbkmxw9CiZGDmWH7kjCX7Nb8Z5zfpzO3VCTpO74Q==} + '@tangle-network/agent-eval@0.122.8': + resolution: {integrity: sha512-7v0us+6zpcR+VKqt9olG7C7j4KlTctvwhlHgm2qHU+FBorEJFcJs6JDqWMw+3h7t9ba/07muIb0AfuvSsmJJ8w==} engines: {node: '>=20'} hasBin: true @@ -688,15 +904,61 @@ packages: '@types/estree@1.0.8': resolution: {integrity: sha512-dWHzHa2WqEXI/O1E9OjrocMTKJl2mSrEolh1Iomrv6U+JuNwaHXsXx9bLu5gG7BUWFIN0skIQJQ/L1rIex4X6w==} + '@types/istanbul-lib-coverage@2.0.6': + resolution: {integrity: sha512-2QF/t/auWm0lsy8XtKVPG19v3sSOQlJe/YHZgfjb/KBBHOGSV+J2q/S671rcq9uTBrLAXmZpqJiaQbMT+zNU1w==} + + '@types/istanbul-lib-report@3.0.3': + resolution: {integrity: sha512-NQn7AHQnk/RSLOxrBbGyJM/aVQ+pjj5HCgasFxc0K/KhoATfQ/47AyUl15I2yBUpihjmas+a+VJBOqecrFH+uA==} + + '@types/istanbul-reports@3.0.4': + resolution: {integrity: sha512-pk2B1NWalF9toCRu6gjBzR69syFjP4Od8WRAX+0mmf9lAjCRicLOWc+ZrxZHx/0XRjotgkF9t6iaMJ+aXcOdZQ==} + + '@types/jest@29.5.14': + resolution: {integrity: sha512-ZN+4sdnLUbo8EVvVc2ao0GFW6oVrQRPn4K2lglySj7APvSrgzxHiNNK99us4WDMi57xxA2yggblIAMNhXOotLQ==} + + '@types/node-fetch@2.6.13': + resolution: {integrity: sha512-QGpRVpzSaUs30JBSGPjOg4Uveu384erbHBoT1zeONvyCfwQxIkUshLAOqN/k9EjGviPRmWTTe6aH2qySWKTVSw==} + + '@types/node@18.19.130': + resolution: {integrity: sha512-GRaXQx6jGfL8sKfaIDD6OupbIHBr9jv7Jnaml9tB7l4v068PAOXqfcujMMo5PhbIs6ggR1XODELqahT2R8v0fg==} + '@types/node@25.6.0': resolution: {integrity: sha512-+qIYRKdNYJwY3vRCZMdJbPLJAtGjQBudzZzdzwQYkEPQd+PJGixUL5QfvCLDaULoLv+RhT3LDkwEfKaAkgSmNQ==} + '@types/node@25.9.5': + resolution: {integrity: sha512-OScDchr2fwuUmWdf4kZ9h7PcJiYDVInhJizG/biAq3cAvqwYktuy/TYGGdZNMtNTFUP7rnb0NU4TUdm82kt4Rg==} + + '@types/pg@8.11.0': + resolution: {integrity: sha512-sDAlRiBNthGjNFfvt0k6mtotoVYVQ63pA8R4EMWka7crawSR60waVYR0HAgmPRs/e2YaeJTD/43OoZ3PFw80pw==} + '@types/proper-lockfile@4.1.4': resolution: {integrity: sha512-uo2ABllncSqg9F1D4nugVl9v93RmjxF6LJzQLMLDdPaXCUIDPeOJ21Gbqi43xNKzBi/WQ0Q0dICqufzQbMjipQ==} + '@types/retry@0.12.0': + resolution: {integrity: sha512-wWKOClTTiizcZhXnPY4wikVAwmdYHp8q6DmC+EJUzAMsycb7HB32Kh9RN4+0gExjmPmZSAQjgURXIGATPegAvA==} + '@types/retry@0.12.5': resolution: {integrity: sha512-3xSjTp3v03X/lSQLkczaN9UIEwJMoMCA1+Nb5HfbJEQWogdeQIyVtTvxPXDQjZ5zws8rFQfVfRdz03ARihPJgw==} + '@types/stack-utils@2.0.3': + resolution: {integrity: sha512-9aEbYZ3TbYMznPdcdr3SmIrLXwC/AKZXQeCf9Pgao5CKb8CyHuEX5jzWPTkvregvhRJHcpRO6BFoGW9ycaOkYw==} + + '@types/webidl-conversions@7.0.3': + resolution: {integrity: sha512-CiJJvcRtIgzadHCYXw7dqEnMNRjhGZlYK05Mj9OyktqV8uVT8fD2BFOB7S1uwBE3Kj2Z+4UyPmFw/Ixgw/LAlA==} + + '@types/whatwg-url@13.0.0': + resolution: {integrity: sha512-N8WXpbE6Wgri7KUSvrmQcqrMllKZ9uxkYWMt+mCSGwNc0Hsw9VQTW7ApqI4XNrx6/SaM2QQJCzMPDEXE058s+Q==} + + '@types/yargs-parser@21.0.3': + resolution: {integrity: sha512-I4q9QU9MQv4oEOz4tAHJtNz1cwuLxn2F3xcc2iV5WdqLPpUnj30aUuxt1mAxYTG+oe8CZMV/+6rU4S4gRDzqtQ==} + + '@types/yargs@17.0.35': + resolution: {integrity: sha512-qUHkeCyQFxMXg79wQfTtfndEC+N9ZZg76HJftDJp+qH2tV7Gj4OJi7l+PiWwJ+pWtW8GwSmqsDj/oymhrTWXjg==} + + '@typespec/ts-http-runtime@0.3.7': + resolution: {integrity: sha512-JVUD8X2tfDMWjcjLs4yVxxVrS8yR5vnh386GAXT9Qj79nBxxXSaHFQZg5FweLmT8HlPQ3kii6noUB+Z9RN7DvQ==} + engines: {node: '>=22.0.0'} + '@vitest/expect@3.2.4': resolution: {integrity: sha512-Io0yyORnB6sikFlt8QW5K7slY4OjqNX9jmJQ02QDda8lyM6B5oNgVWoSoKPac8/kgnCUzuHQKrSLtu/uOqqrig==} @@ -737,18 +999,95 @@ packages: zod: optional: true + abort-controller@3.0.0: + resolution: {integrity: sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==} + engines: {node: '>=6.5'} + acorn@8.16.0: resolution: {integrity: sha512-UVJyE9MttOsBQIDKw1skb9nAwQuR5wuGD3+82K6JgJlm/Y+KI92oNsMNGZCYdDsVtRHSak0pcV5Dno5+4jh9sw==} engines: {node: '>=0.4.0'} hasBin: true + afinn-165-financialmarketnews@3.0.0: + resolution: {integrity: sha512-0g9A1S3ZomFIGDTzZ0t6xmv4AuokBvBmpes8htiyHpH7N4xDmvSQL6UxL/Zcs2ypRb3VwgCscaD8Q3zEawKYhw==} + + afinn-165@2.0.2: + resolution: {integrity: sha512-mJ/RLUfpXfQA6bzugv+bBsc/QYkVrKaLYeS8fWBpKbTCsonv4iuV9ET0fgReEunm9vKLkaNgnekuSNlTC3WQ1Q==} + + agent-base@6.0.2: + resolution: {integrity: sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==} + engines: {node: '>= 6.0.0'} + + agent-base@7.1.4: + resolution: {integrity: sha512-MnA+YT8fwfJPgBx3m60MNqakm30XOkyIoH1y6huTQvC0PwZG7ki8NacLBcrPbNoo8vEZy7Jpuk7+jMO+CUovTQ==} + engines: {node: '>= 14'} + + agentkeepalive@4.6.0: + resolution: {integrity: sha512-kja8j7PjmncONqaTsB8fQ+wE2mSU2DJ9D4XKoJ5PFWIdRMa6SLSN1ff4mOr4jCbfRSsxR4keIiySJU0N9T5hIQ==} + engines: {node: '>= 8.0.0'} + + ansi-styles@4.3.0: + resolution: {integrity: sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==} + engines: {node: '>=8'} + + ansi-styles@5.2.0: + resolution: {integrity: sha512-Cxwpt2SfTzTtXcfOlzGEee8O+c+MmUgGrNiBcXnuWxuFJHe6a5Hz7qwhwe5OgaSYI0IJvkLqWX1ASG+cJOkEiA==} + engines: {node: '>=10'} + any-promise@1.3.0: resolution: {integrity: sha512-7UvmKalWRt1wgjL1RrGxoSJW/0QZFIegpeGvZG9kjp8vrRu55XTHbwnqq2GpXm9uLbcuhxm3IqX9OB4MZR1b2A==} + apparatus@0.0.10: + resolution: {integrity: sha512-KLy/ugo33KZA7nugtQ7O0E1c8kQ52N3IvD/XgIh4w/Nr28ypfkwDfA67F1ev4N1m5D+BOk1+b2dEJDfpj/VvZg==} + engines: {node: '>=0.2.6'} + assertion-error@2.0.1: resolution: {integrity: sha512-Izi8RQcffqCeNVgFigKli1ssklIbpHnCYc6AknXGYoB6grJqyeby7jv12JUQgmTAnIDnbck1uxksT4dzN3PWBA==} engines: {node: '>=12'} + asynckit@0.4.0: + resolution: {integrity: sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==} + + axios@1.18.1: + resolution: {integrity: sha512-3nTvFlvpn9Zu/RkHUqtc7/+al4UpRW5az71ap5zccp6e8RAYEzhMTecX8Dz1wWDYrPpUoB1HAQEGEAEvUr7S9g==} + + base-64@0.1.0: + resolution: {integrity: sha512-Y5gU45svrR5tI2Vt/X9GPd3L0HNIKzGu202EjxrXMpuc2V2CiKgemAbUUsqYmZJvPtCXoUKjNZwBJzsNScUbXA==} + + base64-js@1.5.1: + resolution: {integrity: sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==} + + better-sqlite3@12.11.1: + resolution: {integrity: sha512-dq9AtApgg5PGFtBzPFSBl3HZQjHok5gaQCM6zh2Yk0aSmDCs1CbnVI8/HgASQkNKsWFpseIO9beg5xxpYhbIfA==} + engines: {node: 20.x || 22.x || 23.x || 24.x || 25.x || 26.x} + + bignumber.js@9.3.1: + resolution: {integrity: sha512-Ko0uX15oIUS7wJ3Rb30Fs6SkVbLmPBAKdlm7q9+ak9bbIeFf0MwuBsQV6z7+X768/cHsfg+WlysDWJcmthjsjQ==} + + bindings@1.5.0: + resolution: {integrity: sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ==} + + bl@4.1.0: + resolution: {integrity: sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==} + + braces@3.0.3: + resolution: {integrity: sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==} + engines: {node: '>=8'} + + bson@7.3.1: + resolution: {integrity: sha512-h/C0qe6857pQhcSJHLfsR1uYGj98Ge3wKAD3Ed9KqH3wcVh+BM4Jq4xISD7vs9OPuT07n+q3QQVjslJ286j6ag==} + engines: {node: '>=20.19.0'} + + buffer-equal-constant-time@1.0.1: + resolution: {integrity: sha512-zRpUiDwd/xk6ADqPMATG8vc9VPrkck7T07OIx0gnjmJAnHnTVXNQG3vfvWNuiZIkwu9KrKdA1iJKfsfTVxE6NA==} + + buffer@5.7.1: + resolution: {integrity: sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==} + + bundle-name@4.1.0: + resolution: {integrity: sha512-tjwM5exMg6BGRI+kNmTntNsvdZS1X8BFYS6tnJ2hdH0kVxM6/eVZ2xy+FqStSWvYmtfFMDLIxurorHwDKfDz5Q==} + engines: {node: '>=18'} + bundle-require@5.1.0: resolution: {integrity: sha512-3WrrOuZiyaaZPWiEt4G3+IffISVC9HYlWueJEBWED4ZH4aIAC2PnkdnuRrR94M+w6yGWn4AglWtJtBI8YqvgoA==} engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} @@ -759,10 +1098,21 @@ packages: resolution: {integrity: sha512-b6Ilus+c3RrdDk+JhLKUAQfzzgLEPy6wcXqS7f/xe1EETvsDP6GORG7SFuOs6cID5YkqchW/LXZbX5bc8j7ZcQ==} engines: {node: '>=8'} + call-bind-apply-helpers@1.0.2: + resolution: {integrity: sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==} + engines: {node: '>= 0.4'} + chai@5.3.3: resolution: {integrity: sha512-4zNhdJD/iOjSH0A05ea+Ke6MU5mmpQcbQsSOkgdaUMJ9zTlDTD/GYlwohmIE2u0gaxHYiVHEn1Fw9mZ/ktJWgw==} engines: {node: '>=18'} + chalk@4.1.2: + resolution: {integrity: sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==} + engines: {node: '>=10'} + + charenc@0.0.2: + resolution: {integrity: sha512-yrLQ/yVUFXkzg7EDQsPieE/53+0RlaWTs+wBrvW36cyilJ2SaDWfl4Yj7MtLTXleV9uEKefbAGUPv2/iWSooRA==} + check-error@2.1.3: resolution: {integrity: sha512-PAJdDJusoxnwm1VwW07VWwUN1sl7smmC3OKggvndJFadxxDRyFJBX/ggnu/KE4kQAB7a3Dp8f/YXC1FlUprWmA==} engines: {node: '>= 16'} @@ -771,6 +1121,31 @@ packages: resolution: {integrity: sha512-Qgzu8kfBvo+cA4962jnP1KkS6Dop5NS6g7R5LFYJr4b8Ub94PPQXUksCw9PvXoeXPRRddRNC5C1JQUR2SMGtnA==} engines: {node: '>= 14.16.0'} + chownr@1.1.4: + resolution: {integrity: sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==} + + ci-info@3.9.0: + resolution: {integrity: sha512-NIxF55hv4nSqQswkAeiOi1r83xy8JldOFDTWiug55KBu9Jnblncd2U6ViHmYgHf01TPZS77NJBhBMKdWj9HQMQ==} + engines: {node: '>=8'} + + cloudflare@4.5.0: + resolution: {integrity: sha512-fPcbPKx4zF45jBvQ0z7PCdgejVAPBBCZxwqk1k7krQNfpM07Cfj97/Q6wBzvYqlWXx/zt1S9+m8vnfCe06umbQ==} + + cluster-key-slot@1.1.2: + resolution: {integrity: sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA==} + engines: {node: '>=0.10.0'} + + color-convert@2.0.1: + resolution: {integrity: sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==} + engines: {node: '>=7.0.0'} + + color-name@1.1.4: + resolution: {integrity: sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==} + + combined-stream@1.0.8: + resolution: {integrity: sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==} + engines: {node: '>= 0.8'} + commander@14.0.3: resolution: {integrity: sha512-H+y0Jo/T1RZ9qPP4Eh1pkcQcLRglraJaSLoyOtHxu6AapkjWVCy2Sit1QQ4x3Dng8qDlSsZEet7g5Pq06MvTgw==} engines: {node: '>=20'} @@ -779,6 +1154,10 @@ packages: resolution: {integrity: sha512-NOKm8xhkzAjzFx8B2v5OAHT+u5pRQc2UCa2Vq9jYL/31o2wi9mxBA7LIFs3sV5VSC49z6pEhfbMULvShKj26WA==} engines: {node: '>= 6'} + compromise@14.16.0: + resolution: {integrity: sha512-4DFYl/Hl7sW4XWUDfx9S5vxqyYKpZDwwqrpXsQv5acdbVP+joKceIcIaLb0lhVWUpDBV0OnExk/o/dnYUwXnhQ==} + engines: {node: '>=12.0.0'} + confbox@0.1.8: resolution: {integrity: sha512-RMtmw0iFkeR4YV+fUOSucriAQNb9g8zFR52MWCtl+cCZOFRNL6zeB395vPzFhEjjn4fMxXudmELnl/KF/WrK6w==} @@ -786,6 +1165,13 @@ packages: resolution: {integrity: sha512-5IKcdX0nnYavi6G7TtOhwkYzyjfJlatbjMjuLSfE2kYT5pMDOilZ4OvMhi637CcDICTmz3wARPoyhqyX1Y+XvA==} engines: {node: ^14.18.0 || >=16.10.0} + crypt@0.0.2: + resolution: {integrity: sha512-mCxBlsHFYh9C+HVpiEacem8FEBnMXgU9gy4zmNC+SXAZNB/1idgp/aulFJ4FgCi7GPEVbfyng092GqL2k2rmow==} + + data-uri-to-buffer@4.0.1: + resolution: {integrity: sha512-0R9ikRb668HB7QDxT1vkpuUBtqc53YyAwMwGeUFKRojY/NWKvdZ+9UYtRfGmhqNbRkTSVpMbmyhXipFFv2cb/A==} + engines: {node: '>= 12'} + debug@4.4.3: resolution: {integrity: sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==} engines: {node: '>=6.0'} @@ -795,13 +1181,82 @@ packages: supports-color: optional: true + decompress-response@6.0.0: + resolution: {integrity: sha512-aW35yZM6Bb/4oJlZncMH2LCoZtJXTRxES17vE3hoRiowU2kWHaJKFkSBDnDR+cm9J+9QhXmREyIfv0pji9ejCQ==} + engines: {node: '>=10'} + deep-eql@5.0.2: resolution: {integrity: sha512-h5k/5U50IJJFpzfL6nO9jaaumfjO/f2NjK/oYB2Djzm4p9L+3T9qWpZqZ2hAbLPuuYq9wrU08WQyBTL5GbPk5Q==} engines: {node: '>=6'} + deep-extend@0.6.0: + resolution: {integrity: sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==} + engines: {node: '>=4.0.0'} + + default-browser-id@5.0.1: + resolution: {integrity: sha512-x1VCxdX4t+8wVfd1so/9w+vQ4vx7lKd2Qp5tDRutErwmR85OgmfX7RlLRMWafRMY7hbEiXIbudNrjOAPa/hL8Q==} + engines: {node: '>=18'} + + default-browser@5.5.0: + resolution: {integrity: sha512-H9LMLr5zwIbSxrmvikGuI/5KGhZ8E2zH3stkMgM5LpOWDutGM2JZaj460Udnf1a+946zc7YBgrqEWwbk7zHvGw==} + engines: {node: '>=18'} + + define-lazy-prop@3.0.0: + resolution: {integrity: sha512-N+MeXYoqr3pOgn8xfyRPREN7gHakLYjhsHhWGT3fWAiL4IkAt0iDw14QiiEm2bE30c5XX5q0FtAA3CK5f9/BUg==} + engines: {node: '>=12'} + + delayed-stream@1.0.0: + resolution: {integrity: sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==} + engines: {node: '>=0.4.0'} + + detect-libc@2.1.2: + resolution: {integrity: sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==} + engines: {node: '>=8'} + + diff-sequences@29.6.3: + resolution: {integrity: sha512-EjePK1srD3P08o2j4f0ExnylqRs5B9tJjcp9t1krH2qRi8CCdsYfwe9JgSLurFBWwq4uOlipzfk5fHNvwFKr8Q==} + engines: {node: ^14.15.0 || ^16.10.0 || >=18.0.0} + + digest-fetch@1.3.0: + resolution: {integrity: sha512-CGJuv6iKNM7QyZlM2T3sPAdZWd/p9zQiRNS9G+9COUCwzWFTs0Xp8NF5iePx7wtvhDykReiRRrSeNb4oMmB8lA==} + + dotenv@17.4.2: + resolution: {integrity: sha512-nI4U3TottKAcAD9LLud4Cb7b2QztQMUEfHbvhTH09bqXTxnSie8WnjPALV/WMCrJZ6UV/qHJ6L03OqO3LcdYZw==} + engines: {node: '>=12'} + + dunder-proto@1.0.1: + resolution: {integrity: sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==} + engines: {node: '>= 0.4'} + + ecdsa-sig-formatter@1.0.11: + resolution: {integrity: sha512-nagl3RYrbNv6kQkeJIpt6NJZy8twLB/2vtz6yN9Z4vRKHN4/QZJIEbqohALSgwKdnksuY3k5Addp5lg8sVoVcQ==} + + efrt@2.7.0: + resolution: {integrity: sha512-/RInbCy1d4P6Zdfa+TMVsf/ufZVotat5hCw3QXmWtjU+3pFEOvOQ7ibo3aIxyCJw2leIeAMjmPj+1SLJiCpdrQ==} + engines: {node: '>=12.0.0'} + + end-of-stream@1.4.5: + resolution: {integrity: sha512-ooEGc6HP26xXq/N+GCGOT0JKCLDGrq2bQUZrQ7gyrJiZANJ/8YDTxTpQBXGMn+WbIQXNVpyWymm7KYVICQnyOg==} + + es-define-property@1.0.1: + resolution: {integrity: sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==} + engines: {node: '>= 0.4'} + + es-errors@1.3.0: + resolution: {integrity: sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==} + engines: {node: '>= 0.4'} + es-module-lexer@1.7.0: resolution: {integrity: sha512-jEQoCwk8hyb2AZziIOLhDqpm5+2ww5uIE6lkO/6jcOCusfk6LhMHpXXfBLXTZ7Ydyt0j4VoUQv6uGNYbdW+kBA==} + es-object-atoms@1.1.2: + resolution: {integrity: sha512-HWcBoN6NileqtSydK2FqHbS/LoDd2pqrnQHLyJzBj4kOp/ky2MWMN694xOfkK8/SnUsW2DH7EfyVlydKCsm1Zw==} + engines: {node: '>= 0.4'} + + es-set-tostringtag@2.1.0: + resolution: {integrity: sha512-j6vWzfrGVfyXxge+O0x5sh6cvxAog0a/4Rdd2K36zCMV5eJ+/+tOAngRO8cODMNWbVRdVlmGZQL2YS3yR8bIUA==} + engines: {node: '>= 0.4'} + esbuild@0.27.7: resolution: {integrity: sha512-IxpibTjyVnmrIQo5aqNpCgoACA/dTKLTlhMHihVHhdkxKyPO1uBBthumT0rdHmcsk9uMonIWS0m4FljWzILh3w==} engines: {node: '>=18'} @@ -812,16 +1267,42 @@ packages: engines: {node: '>=18'} hasBin: true + escape-string-regexp@2.0.0: + resolution: {integrity: sha512-UpzcLCXolUWcNu5HtVMHYdXJjArjsF9C0aNnquZYY4uW/Vu0miy5YoWvbV345HauVvcAUnpRuhMMcqTcGOY2+w==} + engines: {node: '>=8'} + estree-walker@3.0.3: resolution: {integrity: sha512-7RUKfXgSMMkzt6ZuXmqapOurLGPPfgj6l9uRZ7lRGolvk0y2yocc35LdcxKC5PQZdn2DMqioAQ2NoWcrTKmm6g==} + event-target-shim@5.0.1: + resolution: {integrity: sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==} + engines: {node: '>=6'} + + eventemitter3@4.0.7: + resolution: {integrity: sha512-8guHBZCwKnFhYdHr2ysuRWErTwhoN2X8XELRlrRwpmfeY2jjuUN4taQMsULKUVo1K4DvZl+0pgfyoysHxvmvEw==} + eventemitter3@5.0.1: resolution: {integrity: sha512-GWkBvjiSZK87ELrYOSESUYeVIc9mvLLf/nXalMOS5dYrgZq9o5OVkbZAVM06CVxYsCwH9BDZFPlQTlPA1j4ahA==} + events@3.3.0: + resolution: {integrity: sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q==} + engines: {node: '>=0.8.x'} + + expand-template@2.0.3: + resolution: {integrity: sha512-XYfuKMvj4O35f/pOXLObndIRvyQ+/+6AhODh+OKWj9S9498pHHn/IMszH+gt0fBCRWMNfk1ZSp5x3AifmnI2vg==} + engines: {node: '>=6'} + expect-type@1.3.0: resolution: {integrity: sha512-knvyeauYhqjOYvQ66MznSMs83wmHrCycNEN6Ao+2AeYEfxUIkuiVxdEa1qlGEPK+We3n0THiDciYSsCcgW/DoA==} engines: {node: '>=12.0.0'} + expect@29.7.0: + resolution: {integrity: sha512-2Zks0hf1VLFYI1kbh0I5jP3KHHyCHpkfyHBzsSXRFgl/Bg9mWYfMW8oD+PdMPlEwy5HNsR9JutYy6pMeOh61nw==} + engines: {node: ^14.15.0 || ^16.10.0 || >=18.0.0} + + extend@3.0.2: + resolution: {integrity: sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==} + fdir@6.5.0: resolution: {integrity: sha512-tIbYtZbucOs0BRGqPJkshJUYdL+SDH7dVM8gjy+ERp3WAUjLEFJE+02kanyHtwjWOnwrKYBiwAmM0p4kLJAnXg==} engines: {node: '>=12.0.0'} @@ -831,33 +1312,240 @@ packages: picomatch: optional: true + fetch-blob@3.2.0: + resolution: {integrity: sha512-7yAQpD2UMJzLi1Dqv7qFYnPbaPx7ZfFK6PiIxQ4PfkGPyNyl2Ugx+a/umUonmKqjhM4DnfbMvdX6otXq83soQQ==} + engines: {node: ^12.20 || >= 14.13} + + file-uri-to-path@1.0.0: + resolution: {integrity: sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw==} + + fill-range@7.1.1: + resolution: {integrity: sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==} + engines: {node: '>=8'} + fix-dts-default-cjs-exports@1.0.1: resolution: {integrity: sha512-pVIECanWFC61Hzl2+oOCtoJ3F17kglZC/6N94eRWycFgBH35hHx0Li604ZIzhseh97mf2p0cv7vVrOZGoqhlEg==} + follow-redirects@1.16.0: + resolution: {integrity: sha512-y5rN/uOsadFT/JfYwhxRS5R7Qce+g3zG97+JrtFZlC9klX/W5hD7iiLzScI4nZqUS7DNUdhPgw4xI8W2LuXlUw==} + engines: {node: '>=4.0'} + peerDependencies: + debug: '*' + peerDependenciesMeta: + debug: + optional: true + + form-data-encoder@1.7.2: + resolution: {integrity: sha512-qfqtYan3rxrnCk1VYaA4H+Ms9xdpPqvLZa6xmMgFvhO32x7/3J/ExcTd6qpxM0vH2GdMI+poehyBZvqfMTto8A==} + + form-data@4.0.6: + resolution: {integrity: sha512-vKatAh4SlVfgbv+YtmhiRjhEMJsYpsG1Y2rMQtR+SVSbytsSD1YGzDIcrAJmdFec88u/+VoGmxnl+80gL1tRCQ==} + engines: {node: '>= 6'} + + formdata-node@4.4.1: + resolution: {integrity: sha512-0iirZp3uVDjVGt9p49aTaqjk84TrglENEDuqfdlZQ1roC9CWlPk6Avf8EEnZNcAqPonwkG35x4n3ww/1THYAeQ==} + engines: {node: '>= 12.20'} + + formdata-polyfill@4.0.10: + resolution: {integrity: sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g==} + engines: {node: '>=12.20.0'} + + fs-constants@1.0.0: + resolution: {integrity: sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==} + fsevents@2.3.3: resolution: {integrity: sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==} engines: {node: ^8.16.0 || ^10.6.0 || >=11.0.0} os: [darwin] + function-bind@1.1.2: + resolution: {integrity: sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==} + + gaxios@7.2.0: + resolution: {integrity: sha512-CUVb4wcYe+771XevyH6HtGmXFAGGKkIC3kswAP8Z1JCe0j80JMaTPZH930DWFrvo0atjh18Arc0pEyUCWa5bfg==} + engines: {node: '>=18'} + + gcp-metadata@8.1.2: + resolution: {integrity: sha512-zV/5HKTfCeKWnxG0Dmrw51hEWFGfcF2xiXqcA3+J90WDuP0SvoiSO5ORvcBsifmx/FoIjgQN3oNOGaQ5PhLFkg==} + engines: {node: '>=18'} + + get-intrinsic@1.3.0: + resolution: {integrity: sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==} + engines: {node: '>= 0.4'} + + get-proto@1.0.1: + resolution: {integrity: sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==} + engines: {node: '>= 0.4'} + + github-from-package@0.0.0: + resolution: {integrity: sha512-SyHy3T1v2NUXn29OsWdxmK6RwHD+vkj3v8en8AOBZ1wBQ/hCAQ5bAQTD02kW4W9tUp/3Qh6J8r9EvntiyCmOOw==} + + google-auth-library@10.9.0: + resolution: {integrity: sha512-xtvUqvINPhTaBm7nXqlYPcrMHJPm1lCNdSovxnKKhTm+4JsvQ+KGVYJViLoH9Yxu8w+T0Qv5HubzYT9BLrppJg==} + engines: {node: '>=18'} + + google-logging-utils@1.1.3: + resolution: {integrity: sha512-eAmLkjDjAFCVXg7A1unxHsLf961m6y17QFqXqAXGj/gVkKFrEICfStRfwUlGNfeCEjNRa32JEWOUTlYXPyyKvA==} + engines: {node: '>=14'} + + gopd@1.2.0: + resolution: {integrity: sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==} + engines: {node: '>= 0.4'} + graceful-fs@4.2.11: resolution: {integrity: sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==} + grad-school@0.0.5: + resolution: {integrity: sha512-rXunEHF9M9EkMydTBux7+IryYXEZinRk6g8OBOGDBzo/qWJjhTxy86i5q7lQYpCLHN8Sqv1XX3OIOc7ka2gtvQ==} + engines: {node: '>=8.0.0'} + + groq-sdk@0.3.0: + resolution: {integrity: sha512-Cdgjh4YoSBE2X4S9sxPGXaAy1dlN4bRtAaDZ3cnq+XsxhhN9WSBeHF64l7LWwuD5ntmw7YC5Vf4Ff1oHCg1LOg==} + + has-flag@4.0.0: + resolution: {integrity: sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==} + engines: {node: '>=8'} + + has-symbols@1.1.0: + resolution: {integrity: sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==} + engines: {node: '>= 0.4'} + + has-tostringtag@1.0.2: + resolution: {integrity: sha512-NqADB8VjPFLM2V0VvHUewwwsw0ZWBaIdgo+ieHtK3hasLz4qeCRjYcqfB6AQrBggRKppKF8L52/VqdVsO47Dlw==} + engines: {node: '>= 0.4'} + + hasown@2.0.4: + resolution: {integrity: sha512-T2UbfbBEF32wiepXIsMlTW9+dDYC6wMh/t/vYA4tuOMKqWz/n3vr1NFSxQiyP+zk2mXsoMA/i/7qV6LKut1t1A==} + engines: {node: '>= 0.4'} + hono@4.12.30: resolution: {integrity: sha512-emn+JoJjrN9YTpRDS5it/UI2SO9BAE37T6I3d963RxcZ81G9A4pr2SZTEiiaiKbzx+NKRg5BZ89fCL7gCJCUog==} engines: {node: '>=16.9.0'} + http-proxy-agent@7.0.2: + resolution: {integrity: sha512-T1gkAiYYDWYx3V5Bmyu7HcfcvL7mUrTWiM6yOfa3PIphViJ/gFPbvidQ+veqSOHci/PxBcDabeUNCzpOODJZig==} + engines: {node: '>= 14'} + + https-proxy-agent@5.0.1: + resolution: {integrity: sha512-dFcAjpTQFgoLMzC2VwU+C/CbS7uRL0lWmxDITmqm7C+7F0Odmj6s9l6alZc6AELXhrnggM2CeWSXHGOdX2YtwA==} + engines: {node: '>= 6'} + + https-proxy-agent@7.0.6: + resolution: {integrity: sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw==} + engines: {node: '>= 14'} + + humanize-ms@1.2.1: + resolution: {integrity: sha512-Fl70vYtsAFb/C06PTS9dZBo7ihau+Tu/DNCk/OyHhea07S+aeMWpFFkUaXRa8fI+ScZbEI8dfSxwY7gxZ9SAVQ==} + + iceberg-js@0.8.1: + resolution: {integrity: sha512-1dhVQZXhcHje7798IVM+xoo/1ZdVfzOMIc8/rgVSijRK38EDqOJoGula9N/8ZI5RD8QTxNQtK/Gozpr+qUqRRA==} + engines: {node: '>=20.0.0'} + + ieee754@1.2.1: + resolution: {integrity: sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==} + + inherits@2.0.4: + resolution: {integrity: sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==} + + ini@1.3.8: + resolution: {integrity: sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==} + + is-buffer@1.1.6: + resolution: {integrity: sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w==} + + is-docker@3.0.0: + resolution: {integrity: sha512-eljcgEDlEns/7AXFosB5K/2nCM4P7FQPkGc/DWLy5rmFEWvZayGrik1d9/QIY5nJ4f9YsVvBkA6kJpHn9rISdQ==} + engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} + hasBin: true + + is-inside-container@1.0.0: + resolution: {integrity: sha512-KIYLCCJghfHZxqjYBE7rEy0OBuTd5xCHS7tHVgvCLkx7StIoaxwNW3hCALgEUjFfeRk+MG/Qxmp/vtETEF3tRA==} + engines: {node: '>=14.16'} + hasBin: true + + is-number@7.0.0: + resolution: {integrity: sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==} + engines: {node: '>=0.12.0'} + + is-wsl@3.1.1: + resolution: {integrity: sha512-e6rvdUCiQCAuumZslxRJWR/Doq4VpPR82kqclvcS0efgt430SlGIk05vdCN58+VrzgtIcfNODjozVielycD4Sw==} + engines: {node: '>=16'} + isows@1.0.7: resolution: {integrity: sha512-I1fSfDCZL5P0v33sVqeTDSpcstAg/N+wF5HS033mogOVIp4B+oHC7oOCsA3axAbBSGTJ8QubbNmnIRN/h8U7hg==} peerDependencies: ws: 8.21.0 + jest-diff@29.7.0: + resolution: {integrity: sha512-LMIgiIrhigmPrs03JHpxUh2yISK3vLFPkAodPeo0+BuF7wA2FoQbkEg1u8gBYBThncu7e1oEDUfIXVuTqLRUjw==} + engines: {node: ^14.15.0 || ^16.10.0 || >=18.0.0} + + jest-get-type@29.6.3: + resolution: {integrity: sha512-zrteXnqYxfQh7l5FHyL38jL39di8H8rHoecLH3JNxH3BwOrBsNeabdap5e0I23lD4HHI8W5VFBZqG4Eaq5LNcw==} + engines: {node: ^14.15.0 || ^16.10.0 || >=18.0.0} + + jest-matcher-utils@29.7.0: + resolution: {integrity: sha512-sBkD+Xi9DtcChsI3L3u0+N0opgPYnCRPtGcQYrgXmR+hmt/fYfWAL0xRXYU8eWOdfuLgBe0YCW3AFtnRLagq/g==} + engines: {node: ^14.15.0 || ^16.10.0 || >=18.0.0} + + jest-message-util@29.7.0: + resolution: {integrity: sha512-GBEV4GRADeP+qtB2+6u61stea8mGcOT4mCtrYISZwfu9/ISHFJ/5zOMXYbpBE9RsS5+Gb63DW4FgmnKJ79Kf6w==} + engines: {node: ^14.15.0 || ^16.10.0 || >=18.0.0} + + jest-util@29.7.0: + resolution: {integrity: sha512-z6EbKajIpqGKU56y5KBUgy1dt1ihhQJgWzUlZHArA/+X2ad7Cb5iF+AK1EWVL/Bo7Rz9uurpqw6SiBCefUbCGA==} + engines: {node: ^14.15.0 || ^16.10.0 || >=18.0.0} + joycon@3.1.1: resolution: {integrity: sha512-34wB/Y7MW7bzjKRjUKTa46I2Z7eV62Rkhva+KkopW7Qvv/OSWBqvkSY7vusOPrNuZcUG3tApvdVgNB8POj3SPw==} engines: {node: '>=10'} + js-tiktoken@1.0.21: + resolution: {integrity: sha512-biOj/6M5qdgx5TKjDnFT1ymSpM5tbd3ylwDtrQvFQSu0Z7bBYko2dF+W/aUkXUPuk6IVpRxk/3Q2sHOzGlS36g==} + + js-tokens@4.0.0: + resolution: {integrity: sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==} + js-tokens@9.0.1: resolution: {integrity: sha512-mxa9E9ITFOt0ban3j6L5MpjwegGz6lBQmM1IJkWeBZGcMxto50+eWdjC/52xDbS2vy0k7vIMK0Fe2wfL9OQSpQ==} + json-bigint@1.0.0: + resolution: {integrity: sha512-SiPv/8VpZuWbvLSMtTDU8hEfrZWg/mH/nV/b4o0CYbSxu1UIQPLdwKOCIyLQX+VIPO5vrLX3i8qtqFyhdPSUSQ==} + + jsonwebtoken@9.0.3: + resolution: {integrity: sha512-MT/xP0CrubFRNLNKvxJ2BYfy53Zkm++5bX9dtuPbqAeQpTVe0MQTFhao8+Cp//EmJp244xt6Drw/GVEGCUj40g==} + engines: {node: '>=12', npm: '>=6'} + + jwa@2.0.1: + resolution: {integrity: sha512-hRF04fqJIP8Abbkq5NKGN0Bbr3JxlQ+qhZufXVr0DvujKy93ZCbXZMHDL4EOtodSbCWxOqR8MS1tXA5hwqCXDg==} + + jws@4.0.1: + resolution: {integrity: sha512-EKI/M/yqPncGUUh44xz0PxSidXFr/+r0pA70+gIYhjv+et7yxM+s29Y+VGDkovRofQem0fs7Uvf4+YmAdyRduA==} + + kareem@3.3.0: + resolution: {integrity: sha512-kpSuLD3/7RenBnjnJdOHXCKC8dTd1JzeOiJhN0necWWci6cC+qX+VuwPnMVgb+a4+KNJSfgqahpnfWaeDXCimw==} + engines: {node: '>=18.0.0'} + + langsmith@0.8.3: + resolution: {integrity: sha512-3y1PRTmY24wiN8NiMb0CcNI2LwgovsKAz+ymVmBSmUx4nXL+GlmPY6Ju7FGeaPV1DMfQs9SjceNZHIfY2XYMKQ==} + peerDependencies: + '@opentelemetry/api': '*' + '@opentelemetry/exporter-trace-otlp-proto': '*' + '@opentelemetry/sdk-trace-base': '*' + openai: '*' + ws: 8.21.0 + peerDependenciesMeta: + '@opentelemetry/api': + optional: true + '@opentelemetry/exporter-trace-otlp-proto': + optional: true + '@opentelemetry/sdk-trace-base': + optional: true + openai: + optional: true + ws: + optional: true + lilconfig@3.1.3: resolution: {integrity: sha512-/vlFKAoH5Cgt3Ie+JLhRbwOsCQePABiU3tJ1egGvyQ+33R/vcwM2Zl2QR/LzjsBeItPt3oSVXapn+m4nQDvpzw==} engines: {node: '>=14'} @@ -869,30 +1557,282 @@ packages: resolution: {integrity: sha512-IXO6OCs9yg8tMKzfPZ1YmheJbZCiEsnBdcB03l0OcfK9prKnJb96siuHCr5Fl37/yo9DnKU+TLpxzTUspw9shg==} engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} + lodash.includes@4.3.0: + resolution: {integrity: sha512-W3Bx6mdkRTGtlJISOvVD/lbqjTlPPUDTMnlXZFnVwi9NKJ6tiAk6LVdlhZMm17VZisqhKcgzpO5Wz91PCt5b0w==} + + lodash.isboolean@3.0.3: + resolution: {integrity: sha512-Bz5mupy2SVbPHURB98VAcw+aHh4vRV5IPNhILUCsOzRmsTmSQ17jIuqopAentWoehktxGd9e/hbIXq980/1QJg==} + + lodash.isinteger@4.0.4: + resolution: {integrity: sha512-DBwtEWN2caHQ9/imiNeEA5ys1JoRtRfY3d7V9wkqtbycnAmTvRRmbHKDV4a0EYc678/dia0jrte4tjYwVBaZUA==} + + lodash.isnumber@3.0.3: + resolution: {integrity: sha512-QYqzpfwO3/CWf3XP+Z+tkQsfaLL/EnUlXWVkIk5FUPc4sBdTehEqZONuyRt2P67PXAk+NXmTBcc97zw9t1FQrw==} + + lodash.isplainobject@4.0.6: + resolution: {integrity: sha512-oSXzaWypCMHkPC3NvBEaPHf0KsA5mvPrOPgQWDsbg8n7orZ290M0BmC/jgRZ4vcJ6DTAhjrsSYgdsW/F+MFOBA==} + + lodash.isstring@4.0.1: + resolution: {integrity: sha512-0wJxfxH1wgO3GrbuP+dTTk7op+6L41QCXbGINEmD+ny/G/eCqGzxyCsh7159S+mgDDcoarnBw6PC1PS5+wUGgw==} + + lodash.once@4.1.1: + resolution: {integrity: sha512-Sb487aTOCr9drQVL8pIxOzVhafOjZN9UU54hiN8PU3uAiSV7lx1yYNpbNmex2PK6dSJoNTSJUUswT651yww3Mg==} + + long@5.3.2: + resolution: {integrity: sha512-mNAgZ1GmyNhD7AuqnTG3/VQ26o760+ZYBPKjPvugO8+nLbYfX6TVpJPseBvopbdY+qpZ/lKUnmEc1LeZYS3QAA==} + loupe@3.2.1: resolution: {integrity: sha512-CdzqowRJCeLU72bHvWqwRBBlLcMEtIvGrlvef74kMnV2AolS9Y8xUv1I0U/MNAWMhBlKIoyuEgoJ0t/bbwHbLQ==} magic-string@0.30.21: resolution: {integrity: sha512-vd2F4YUyEXKGcLHoq+TEyCjxueSeHnFxyyjNp80yg0XV4vUhnDer/lvvlqM/arB5bXQN5K2/3oinyCRyx8T2CQ==} - mlly@1.8.2: - resolution: {integrity: sha512-d+ObxMQFmbt10sretNDytwt85VrbkhhUA/JBGm1MPaWJ65Cl4wOgLaB1NYvJSZ0Ef03MMEU/0xpPMXUIQ29UfA==} - - ms@2.1.3: - resolution: {integrity: sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==} + math-intrinsics@1.1.0: + resolution: {integrity: sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==} + engines: {node: '>= 0.4'} - mz@2.7.0: - resolution: {integrity: sha512-z81GNO7nnYMEhrGh9LeymoE4+Yr0Wn5McHIZMK5cfQCl+NDX08sCZgUc9/6MHni9IWuFLm1Z3HTCXu2z9fN62Q==} + md5@2.3.0: + resolution: {integrity: sha512-T1GITYmFaKuO91vxyoQMFETst+O71VUPEU3ze5GNzDm0OWdP8v1ziTaAEPUr/3kLsY3Sftgz242A1SetQiDL7g==} + + mem0ai@3.1.0: + resolution: {integrity: sha512-N/ESxO2wWlQMvWdjNgTK9kuPYVdMYJglSR14BsDfuNMvK9YrigNdZ0nDtuOO4EtJ+Yge6YQnGJ8uuJDbKbJaXg==} + engines: {node: '>=18'} + peerDependencies: + '@anthropic-ai/sdk': ^0.40.1 + '@aws-sdk/client-bedrock-runtime': '>=3.0.0 <3.968.0' + '@aws-sdk/client-neptune-graph': '>=3.0.0 <3.968.0' + '@aws-sdk/client-s3vectors': 3.967.0 + '@azure/identity': ^4.0.0 + '@azure/search-documents': ^12.0.0 + '@cloudflare/workers-types': ^4.20250504.0 + '@databricks/sql': ^1.16.0 + '@elastic/elasticsearch': ^9.0.0 + '@google-cloud/aiplatform': ^6.8.0 + '@google/genai': ^1.40.0 + '@huggingface/transformers': ^3.0.0 || ^4.0.0 + '@langchain/core': ^1.1.47 + '@mistralai/mistralai': ^1.5.2 + '@mochow/mochow-sdk-node': ^2.1.5 + '@opensearch-project/opensearch': ^3.5.1 + '@pinecone-database/pinecone': ^8.0.0 + '@qdrant/js-client-rest': ^1.18.0 + '@supabase/supabase-js': ^2.49.1 + '@turbopuffer/turbopuffer': ^2.0.0 + '@types/jest': 29.5.14 + '@types/pg': 8.11.0 + '@upstash/vector': ^1.2.3 + '@zilliz/milvus2-sdk-node': ^2.4.0 || ^3.0.0 + better-sqlite3: ^12.6.2 + cassandra-driver: 4.8.0 + chromadb: ^3.5.0 + cloudflare: ^4.2.0 + cohere-ai: ^7.17.0 || ^8.0.0 + compromise: ^14.0.0 + fastembed: ^2.1.0 + groq-sdk: 0.3.0 + iovalkey: ^0.3.3 + mongodb: ^7.0.0 + mysql2: ^3.0.0 + natural: ^8.0.1 + ollama: ^0.5.14 + pg: 8.11.3 + redis: ^4.6.13 + weaviate-client: ^3.0.0 + zeroentropy: ^0.1.0-alpha.10 + peerDependenciesMeta: + '@aws-sdk/client-bedrock-runtime': + optional: true + '@aws-sdk/client-neptune-graph': + optional: true + '@aws-sdk/client-s3vectors': + optional: true + '@databricks/sql': + optional: true + '@elastic/elasticsearch': + optional: true + '@google-cloud/aiplatform': + optional: true + '@huggingface/transformers': + optional: true + '@mochow/mochow-sdk-node': + optional: true + '@opensearch-project/opensearch': + optional: true + '@pinecone-database/pinecone': + optional: true + '@turbopuffer/turbopuffer': + optional: true + '@upstash/vector': + optional: true + '@zilliz/milvus2-sdk-node': + optional: true + cassandra-driver: + optional: true + chromadb: + optional: true + cohere-ai: + optional: true + fastembed: + optional: true + iovalkey: + optional: true + mongodb: + optional: true + mysql2: + optional: true + weaviate-client: + optional: true + zeroentropy: + optional: true + + memjs@1.3.2: + resolution: {integrity: sha512-qUEg2g8vxPe+zPn09KidjIStHPtoBO8Cttm8bgJFWWabbsjQ9Av9Ky+6UcvKx6ue0LLb/LEhtcyQpRyKfzeXcg==} + engines: {node: '>=0.10.0'} + + memory-pager@1.5.0: + resolution: {integrity: sha512-ZS4Bp4r/Zoeq6+NLJpP+0Zzm0pR8whtGPf1XExKLJBAczGMnSi3It14OiNCStjQjM6NU1okjQGSxgEZN8eBYKg==} + + micromatch@4.0.8: + resolution: {integrity: sha512-PXwfBhYu0hBCPw8Dn0E+WDYb7af3dSLVWKi3HGv84IdF4TyFoC0ysxFd0Goxw7nSv4T/PzEJQxsYsEiFCKo2BA==} + engines: {node: '>=8.6'} + + mime-db@1.52.0: + resolution: {integrity: sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==} + engines: {node: '>= 0.6'} + + mime-types@2.1.35: + resolution: {integrity: sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==} + engines: {node: '>= 0.6'} + + mimic-response@3.1.0: + resolution: {integrity: sha512-z0yWI+4FDrrweS8Zmt4Ej5HdJmky15+L2e6Wgn3+iK5fWzb6T3fhNFq2+MeTRb064c6Wr4N/wv0DzQTjNzHNGQ==} + engines: {node: '>=10'} + + minimist@1.2.8: + resolution: {integrity: sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==} + + mkdirp-classic@0.5.3: + resolution: {integrity: sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==} + + mlly@1.8.2: + resolution: {integrity: sha512-d+ObxMQFmbt10sretNDytwt85VrbkhhUA/JBGm1MPaWJ65Cl4wOgLaB1NYvJSZ0Ef03MMEU/0xpPMXUIQ29UfA==} + + mongodb-connection-string-url@7.0.2: + resolution: {integrity: sha512-ZoS07RoFqpKYQwAk59qmrx8+jJHNHU30UjlU96QktiGn1ltvDr+vCznLX5DiUBLEpMAHatHNWV1nM/74ul66kA==} + engines: {node: '>=20.19.0'} + + mongodb@7.2.0: + resolution: {integrity: sha512-F/2+BMZtLVhY30ioZp0dAmZ+IRZMBqI+nrv6t5+9/1AIwCa8sMRC3jBf81lpxMhnZgqq8CoUD503Z1oZWq1/sw==} + engines: {node: '>=20.19.0'} + peerDependencies: + '@aws-sdk/credential-providers': ^3.806.0 + '@mongodb-js/zstd': ^7.0.0 + gcp-metadata: ^7.0.1 + kerberos: ^7.0.0 + mongodb-client-encryption: '>=7.0.0 <7.1.0' + snappy: ^7.3.2 + socks: ^2.8.6 + peerDependenciesMeta: + '@aws-sdk/credential-providers': + optional: true + '@mongodb-js/zstd': + optional: true + gcp-metadata: + optional: true + kerberos: + optional: true + mongodb-client-encryption: + optional: true + snappy: + optional: true + socks: + optional: true + + mongoose@9.7.4: + resolution: {integrity: sha512-nuSYGUWWzNd4EAbGYxE469wPTL+kmxb5+91YvCvMkJ08rvNRht/usZUU3LuFuk7rDutF2QWBZHPHuzM8TxXApA==} + engines: {node: '>=20.19.0'} + + mpath@0.9.0: + resolution: {integrity: sha512-ikJRQTk8hw5DEoFVxHG1Gn9T/xcjtdnOKIU1JTmGjZZlg9LST2mBLmcX3/ICIbgJydT2GOc15RnNy5mHmzfSew==} + engines: {node: '>=4.0.0'} + + mquery@6.0.0: + resolution: {integrity: sha512-b2KQNsmgtkscfeDgkYMcWGn9vZI9YoXh802VDEwE6qc50zxBFQ0Oo8ROkawbPAsXCY1/Z1yp0MagqsZStPWJjw==} + engines: {node: '>=20.19.0'} + + ms@2.1.3: + resolution: {integrity: sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==} + + mustache@4.2.0: + resolution: {integrity: sha512-71ippSywq5Yb7/tVYyGbkBggbU8H3u5Rz56fH60jGFgr8uHwxs+aSKeqmluIVzM0m0kB7xQjKS6qPfd0b2ZoqQ==} + hasBin: true + + mz@2.7.0: + resolution: {integrity: sha512-z81GNO7nnYMEhrGh9LeymoE4+Yr0Wn5McHIZMK5cfQCl+NDX08sCZgUc9/6MHni9IWuFLm1Z3HTCXu2z9fN62Q==} nanoid@3.3.12: resolution: {integrity: sha512-ZB9RH/39qpq5Vu6Y+NmUaFhQR6pp+M2Xt76XBnEwDaGcVAqhlvxrl3B2bKS5D3NH3QR76v3aSrKaF/Kiy7lEtQ==} engines: {node: ^10 || ^12 || ^13.7 || ^14 || >=15.0.1} hasBin: true + napi-build-utils@2.0.0: + resolution: {integrity: sha512-GEbrYkbfF7MoNaoh2iGG84Mnf/WZfB0GdGEsM8wz7Expx/LlWf5U8t9nvJKXSp3qr5IsEbK04cBGhol/KwOsWA==} + + natural@8.1.1: + resolution: {integrity: sha512-Ucb+lsUcGxUqu3rn8cwHjT6gJQosO63nIX/aBQXB3+IDkNbFV7PuviysO+Rzz3aKn7PZhPj3bNF4PS9gDVjYCQ==} + engines: {node: '>=0.4.10'} + + node-abi@3.94.0: + resolution: {integrity: sha512-W5ZNO5KRPB5TkYmGVD9F6YqhsglXJzE6etpbmT+f6EQElhiX/UTG551cnsRGvLG3fyZEg9HwaDmNmj5nwJ4z9g==} + engines: {node: '>=10'} + + node-domexception@1.0.0: + resolution: {integrity: sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==} + engines: {node: '>=10.5.0'} + deprecated: Use your platform's native DOMException instead + + node-fetch@2.7.0: + resolution: {integrity: sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==} + engines: {node: 4.x || >=6.0.0} + peerDependencies: + encoding: ^0.1.0 + peerDependenciesMeta: + encoding: + optional: true + + node-fetch@3.3.2: + resolution: {integrity: sha512-dRB78srN/l6gqWulah9SrxeYnxeddIG30+GOqK/9OlLVyLg3HPnr6SqOWTWOXKRwC2eGYCkZ59NNuSgvSrpgOA==} + engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} + object-assign@4.1.1: resolution: {integrity: sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg==} engines: {node: '>=0.10.0'} + obuf@1.1.2: + resolution: {integrity: sha512-PX1wu0AmAdPqOL1mWhqmlOd8kOIZQwGZw6rh7uby9fTc5lhaOWFLX3I6R1hrF9k3zUY40e6igsLGkDXK92LJNg==} + + ollama@0.5.18: + resolution: {integrity: sha512-lTFqTf9bo7Cd3hpF6CviBe/DEhewjoZYd9N/uCe7O20qYTvGqrNOFOBDj3lbZgFWHUgDv5EeyusYxsZSLS8nvg==} + + once@1.4.0: + resolution: {integrity: sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==} + + open@10.2.0: + resolution: {integrity: sha512-YgBpdJHPyQ2UE5x+hlSXcnejzAvD0b22U2OuAP+8OnlJT+PjWPxtgmGqKKc+RgTM63U9gN0YzrYc71R2WT/hTA==} + engines: {node: '>=18'} + + openai@4.104.0: + resolution: {integrity: sha512-p99EFNsA/yX6UhVO93f5kJsDRLAg+CTA2RBqdHK4RtK8u5IJw32Hyb2dTGKbnnFmnuoBv5r7Z2CURI9sGZpSuA==} + hasBin: true + peerDependencies: + ws: 8.21.0 + zod: ^3.23.8 + peerDependenciesMeta: + ws: + optional: true + zod: + optional: true + openapi3-ts@4.5.0: resolution: {integrity: sha512-jaL+HgTq2Gj5jRcfdutgRGLosCy/hT8sQf6VOy+P+g36cZOjI1iukdPnijC+4CmeRzg/jEllJUboEic2FhxhtQ==} @@ -904,6 +1844,22 @@ packages: typescript: optional: true + p-finally@1.0.0: + resolution: {integrity: sha512-LICb2p9CB7FS+0eR1oqWnHhp0FljGLZCWBE9aix0Uye9W8LTQPwMTYVGWQWIw9RdQiDg4+epXQODwIYJtSJaow==} + engines: {node: '>=4'} + + p-queue@6.6.2: + resolution: {integrity: sha512-RwFpb72c/BhQLEXIZ5K2e+AhgNVmIejGlTgiB9MzZ0e93GRvqZ7uSi0dvRF7/XIXDeNkra2fNHBxTyPDGySpjQ==} + engines: {node: '>=8'} + + p-retry@4.6.2: + resolution: {integrity: sha512-312Id396EbJdvRONlngUx0NydfrIQ5lsYu0znKVUzVvArzEIt08V1qhtyESbGVd1FGX7UKtiFp5uwKZdM8wIuQ==} + engines: {node: '>=8'} + + p-timeout@3.2.0: + resolution: {integrity: sha512-rhIwUycgwwKcP9yTOOFK/AKsAopjjCakVqLHePO3CC6Mir1Z99xT+R63jZxAT5lFZLa2inS5h+ZS2GvR99/FBg==} + engines: {node: '>=8'} + pathe@2.0.3: resolution: {integrity: sha512-WUjGcAqP1gQacoQe+OBJsFA7Ld4DyXuUIjZ5cc75cLHvJ7dtNsTugphxIADwspS+AraAUePCKrSVtPLFj/F88w==} @@ -911,9 +1867,55 @@ packages: resolution: {integrity: sha512-//nshmD55c46FuFw26xV/xFAaB5HF9Xdap7HJBBnrKdAd6/GxDBaNA1870O79+9ueg61cZLSVc+OaFlfmObYVQ==} engines: {node: '>= 14.16'} + pg-cloudflare@1.4.0: + resolution: {integrity: sha512-Vo7z/6rrQYxpNRylp4Tlob2elzbh+N/MOQbxFVWCxS7oEx6jF53GTJFxK2WWpKuBRkmiin4Mt+xofFDjx09R0A==} + + pg-connection-string@2.14.0: + resolution: {integrity: sha512-XwWDGcLRGCXAR8F/AM5bG7Q+A3Wm2s6QeEjlOKZLlH3UYcguiqCWKyWXVag5TLTIjR7oOJUY8kcADaZgWPyLeg==} + + pg-int8@1.0.1: + resolution: {integrity: sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw==} + engines: {node: '>=4.0.0'} + + pg-numeric@1.0.2: + resolution: {integrity: sha512-BM/Thnrw5jm2kKLE5uJkXqqExRUY/toLHda65XgFTBTFYZyopbKjBe29Ii3RbkvlsMoFwD+tHeGaCjjv0gHlyw==} + engines: {node: '>=4'} + + pg-pool@3.14.0: + resolution: {integrity: sha512-gKtPkFdQPU3DksooVLi9LsjZxrsBUZIpa+7aVx+LV5pNh0KzP4Zleud2po+ConrxbuXGBJ6Hfer6hdgpIBpBaw==} + peerDependencies: + pg: '>=8.0' + + pg-protocol@1.15.0: + resolution: {integrity: sha512-cq9sECI5s0+uPUXjbz8ioyPJni6RzsRib0US67i5IoTZKw8fNeYlVE7u8F4dG7vEJJtc5wdD1K189lCCUwqWTQ==} + + pg-types@2.2.0: + resolution: {integrity: sha512-qTAAlrEsl8s4OiEQY69wDvcMIdQN6wdz5ojQiOy6YRMuynxenON0O5oCpJI6lshc6scgAY8qvJ2On/p+CXY0GA==} + engines: {node: '>=4'} + + pg-types@4.1.0: + resolution: {integrity: sha512-o2XFanIMy/3+mThw69O8d4n1E5zsLhdO+OPqswezu7Z5ekP4hYDqlDjlmOpYMbzY2Br0ufCwJLdDIXeNVwcWFg==} + engines: {node: '>=10'} + + pg@8.22.0: + resolution: {integrity: sha512-8wih1vVIBMxoUM2oB4soJsD9tDnDpLv4OXBJ+EJzFsvycD+lfyIreC2gGHq78f8jbLLt+bvlPTFdFZfJkOuzAA==} + engines: {node: '>= 16.0.0'} + peerDependencies: + pg-native: '>=3.0.1' + peerDependenciesMeta: + pg-native: + optional: true + + pgpass@1.0.5: + resolution: {integrity: sha512-FdW9r/jQZhSeohs1Z3sI1yxFQNFvMcnmfuj4WBMUTxOrAyLMaTcE1aAMBiTlbMNaXvBCQuVi0R7hd8udDSP7ug==} + picocolors@1.1.1: resolution: {integrity: sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==} + picomatch@2.3.2: + resolution: {integrity: sha512-V7+vQEJ06Z+c5tSye8S+nHUfI51xoXIXjHQ99cQtKUkQqqO1kO/KCJUfZXuB47h/YBlDhah2H3hdUGXn8ie0oA==} + engines: {node: '>=8.6'} + picomatch@4.0.4: resolution: {integrity: sha512-QP88BAKvMam/3NxH6vj2o21R6MjxZUAd6nlwAS/pnGvN9IVLocLHxGYIzFhg6fUQ+5th6P4dv4eW9jX3DSIj7A==} engines: {node: '>=12'} @@ -947,13 +1949,88 @@ packages: resolution: {integrity: sha512-qif0+jGGZoLWdHey3UFHHWP0H7Gbmsk8T5VEqyYFbWqPr1XqvLGBbk/sl8V5exGmcYJklJOhOQq1pV9IcsiFag==} engines: {node: ^10 || ^12 || >=14} + postgres-array@2.0.0: + resolution: {integrity: sha512-VpZrUqU5A69eQyW2c5CA1jtLecCsN2U/bD6VilrFDWq5+5UIEVO7nazS3TEcHf1zuPYO/sqGvUvW62g86RXZuA==} + engines: {node: '>=4'} + + postgres-array@3.0.4: + resolution: {integrity: sha512-nAUSGfSDGOaOAEGwqsRY27GPOea7CNipJPOA7lPbdEpx5Kg3qzdP0AaWC5MlhTWV9s4hFX39nomVZ+C4tnGOJQ==} + engines: {node: '>=12'} + + postgres-bytea@1.0.1: + resolution: {integrity: sha512-5+5HqXnsZPE65IJZSMkZtURARZelel2oXUEO8rH83VS/hxH5vv1uHquPg5wZs8yMAfdv971IU+kcPUczi7NVBQ==} + engines: {node: '>=0.10.0'} + + postgres-bytea@3.0.0: + resolution: {integrity: sha512-CNd4jim9RFPkObHSjVHlVrxoVQXz7quwNFpz7RY1okNNme49+sVyiTvTRobiLV548Hx/hb1BG+iE7h9493WzFw==} + engines: {node: '>= 6'} + + postgres-date@1.0.7: + resolution: {integrity: sha512-suDmjLVQg78nMK2UZ454hAG+OAW+HQPZ6n++TNDUX+L0+uUlLywnoxJKDou51Zm+zTCjrCl0Nq6J9C5hP9vK/Q==} + engines: {node: '>=0.10.0'} + + postgres-date@2.1.0: + resolution: {integrity: sha512-K7Juri8gtgXVcDfZttFKVmhglp7epKb1K4pgrkLxehjqkrgPhfG6OO8LHLkfaqkbpjNRnra018XwAr1yQFWGcA==} + engines: {node: '>=12'} + + postgres-interval@1.2.0: + resolution: {integrity: sha512-9ZhXKM/rw350N1ovuWHbGxnGh/SNJ4cnxHiM0rxE4VN41wsg8P8zWn9hv/buK00RP4WvlOyr/RBDiptyxVbkZQ==} + engines: {node: '>=0.10.0'} + + postgres-interval@3.0.0: + resolution: {integrity: sha512-BSNDnbyZCXSxgA+1f5UU2GmwhoI0aU5yMxRGO8CdFEcY2BQF9xm/7MqKnYoM1nJDk8nONNWDk9WeSmePFhQdlw==} + engines: {node: '>=12'} + + postgres-range@1.1.4: + resolution: {integrity: sha512-i/hbxIE9803Alj/6ytL7UHQxRvZkI9O4Sy+J3HGc4F4oo/2eQAjTSNJ0bfxyse3bH0nuVesCk+3IRLaMtG3H6w==} + + prebuild-install@7.1.3: + resolution: {integrity: sha512-8Mf2cbV7x1cXPUILADGI3wuhfqWvtiLA1iclTDbFRZkgRQS0NqsPZphna9V+HyTEadheuPmjaJMsbzKQFOzLug==} + engines: {node: '>=10'} + deprecated: No longer maintained. Please contact the author of the relevant native addon; alternatives are available. + hasBin: true + + pretty-format@29.7.0: + resolution: {integrity: sha512-Pdlw/oPxN+aXdmM9R00JVC9WVFoCLTKJvDVLgmJ+qAffBMxsV85l/Lu7sNx4zSzPyoL2euImuEwHhOXdEgNFZQ==} + engines: {node: ^14.15.0 || ^16.10.0 || >=18.0.0} + proper-lockfile@4.1.2: resolution: {integrity: sha512-TjNPblN4BwAWMXU8s9AEz4JmQxnD1NNL7bNOY/AKUzyamc379FWASUhc/K1pL2noVb+XmZKLL68cjzLsiOAMaA==} + protobufjs@7.6.5: + resolution: {integrity: sha512-/FPD0nUc9jH6rfFjji9IBqOz4pcSE3CsT1m7Ep6Mdb0LxSUMj8hgl6GomOvZzpNpAqqGaXA0P3VSrZLFzIhQrw==} + engines: {node: '>=12.0.0'} + + proxy-from-env@2.1.0: + resolution: {integrity: sha512-cJ+oHTW1VAEa8cJslgmUZrc+sjRKgAKl3Zyse6+PV38hZe/V6Z14TbCuXcan9F9ghlz4QrFr2c92TNF82UkYHA==} + engines: {node: '>=10'} + + pump@3.0.4: + resolution: {integrity: sha512-VS7sjc6KR7e1ukRFhQSY5LM2uBWAUPiOPa/A3mkKmiMwSmRFUITt0xuj+/lesgnCv+dPIEYlkzrcyXgquIHMcA==} + + punycode@2.3.1: + resolution: {integrity: sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==} + engines: {node: '>=6'} + + rc@1.2.8: + resolution: {integrity: sha512-y3bGgqKj3QBdxLbLkomlohkvsA8gdAiUQlSBJnBhfn+BPxg4bc62d8TcBW15wavDfgexCgccckhcZvywyQYPOw==} + hasBin: true + + react-is@18.3.1: + resolution: {integrity: sha512-/LLMVyas0ljjAtoYiPqYiL8VWXzUUdThrmU5+n20DZv+a+ClRoevUzw5JxU+Ieh5/c87ytoTBV9G1FiKfNJdmg==} + + readable-stream@3.6.2: + resolution: {integrity: sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==} + engines: {node: '>= 6'} + readdirp@4.1.2: resolution: {integrity: sha512-GDhwkLfywWL2s6vEjyhri+eXmfH6j1L7JE27WhqLeYzoh/A3DBaYGEj2H/HFZCn/kMfim73FXxEJTw06WtxQwg==} engines: {node: '>= 14.18.0'} + redis@5.12.1: + resolution: {integrity: sha512-LDsoVvb/CpoV9EN3FXvgvSHNJWuCIzl9MiO3ppOevuGLpSGJhwfQjpEwfFJcQvNSddHADDdZaWx0HnmMxRXG7g==} + engines: {node: '>= 18.19.0'} + resolve-from@5.0.0: resolution: {integrity: sha512-qYg9KP24dD5qka9J47d0aVky0N+b4fTU89LN9iDnjB5waksiC49rvMB0PrUJQGoTmH50XPiqOvAjDfaijGxYZw==} engines: {node: '>=8'} @@ -962,17 +2039,50 @@ packages: resolution: {integrity: sha512-9LkiTwjUh6rT555DtE9rTX+BKByPfrMzEAtnlEtdEwr3Nkffwiihqe2bWADg+OQRjt9gl6ICdmB/ZFDCGAtSow==} engines: {node: '>= 4'} + retry@0.13.1: + resolution: {integrity: sha512-XQBQ3I8W1Cge0Seh+6gjj03LbmRFWuoszgK9ooCpwYIrhhoO80pfq4cUkU5DkknwfOfFteRwlZ56PYOGYyFWdg==} + engines: {node: '>= 4'} + rollup@4.60.2: resolution: {integrity: sha512-J9qZyW++QK/09NyN/zeO0dG/1GdGfyp9lV8ajHnRVLfo/uFsbji5mHnDgn/qYdUHyCkM2N+8VyspgZclfAh0eQ==} engines: {node: '>=18.0.0', npm: '>=8.0.0'} hasBin: true + run-applescript@7.1.0: + resolution: {integrity: sha512-DPe5pVFaAsinSaV6QjQ6gdiedWDcRCbUuiQfQa2wmWV7+xC9bGulGI8+TdRmoFkAPaBXk8CrAbnlY2ISniJ47Q==} + engines: {node: '>=18'} + + safe-buffer@5.2.1: + resolution: {integrity: sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==} + + safe-stable-stringify@2.5.0: + resolution: {integrity: sha512-b3rppTKm9T+PsVCBEOUR46GWI7fdOs00VKZ1+9c1EWDaDMvjQc6tUwuFyIprgGgTcWoVHSKrU8H31ZHA2e0RHA==} + engines: {node: '>=10'} + + semver@7.8.5: + resolution: {integrity: sha512-Y7/KDsb8LjooZpwaqGyulO6DQlksgCncchHGk+sZIY4SBvUocMBEFH5Ur1fI4dV+Jvl0w6cjvucaIi40puRioA==} + engines: {node: '>=10'} + hasBin: true + + sift@17.1.3: + resolution: {integrity: sha512-Rtlj66/b0ICeFzYTuNvX/EF1igRbbnGSvEyT79McoZa/DeGhMyC5pWKOEsZKnpkqtSeovd5FL/bjHWC3CIIvCQ==} + siginfo@2.0.0: resolution: {integrity: sha512-ybx0WO1/8bSBLEWXZvEd7gMW3Sn3JFlW3TvX1nREbDLRNQNaeNN8WK0meBwPdAaOI7TtRRRJn/Es1zhrrCHu7g==} signal-exit@3.0.7: resolution: {integrity: sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ==} + simple-concat@1.0.1: + resolution: {integrity: sha512-cSFtAPtRhljv69IK0hTVZQ+OfE9nePi/rtJmw5UjHeVyVroEqJXP1sFztKUy1qU+xvz3u/sfYJLa947b7nAN2Q==} + + simple-get@4.0.1: + resolution: {integrity: sha512-brv7p5WgH0jmQJr1ZDDfKDOSeWWg+OVypG99A/5vYGPqJ6pxiaHLy8nxtFjBA7oMa01ebA9gfh1uMCFqOuXxvA==} + + slash@3.0.0: + resolution: {integrity: sha512-g9Q1haeby36OSStwb4ntCGGGaKsaVSjQ68fBxoQcutl5fS1vuY18H3wSt3jFyFtrkx+Kz0V1G85A4MyAdDMi2Q==} + engines: {node: '>=8'} + source-map-js@1.2.1: resolution: {integrity: sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA==} engines: {node: '>=0.10.0'} @@ -981,12 +2091,34 @@ packages: resolution: {integrity: sha512-i5uvt8C3ikiWeNZSVZNWcfZPItFQOsYTUAOkcUPGd8DqDy1uOUikjt5dG+uRlwyvR108Fb9DOd4GvXfT0N2/uQ==} engines: {node: '>= 12'} + sparse-bitfield@3.0.3: + resolution: {integrity: sha512-kvzhi7vqKTfkh0PZU+2D2PIllw2ymqJKujUcyPMd9Y75Nv4nPbGJZXNhxsgdQab2BmlDct1YnfQCguEvHr7VsQ==} + + split2@4.2.0: + resolution: {integrity: sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg==} + engines: {node: '>= 10.x'} + + stack-utils@2.0.6: + resolution: {integrity: sha512-XlkWvfIm6RmsWtNJx+uqtKLS8eqFbxUg0ZzLXqY0caEy9l7hruX8IpiDnjsLavoBgqCCR71TqWO8MaXYheJ3RQ==} + engines: {node: '>=10'} + stackback@0.0.2: resolution: {integrity: sha512-1XMJE5fQo1jGH6Y/7ebnwPOBEkIEnT4QF32d5R1+VXdXveM0IBMJt8zfaxX1P3QhVwrYe+576+jkANtSS2mBbw==} std-env@3.10.0: resolution: {integrity: sha512-5GS12FdOZNliM5mAOxFRg7Ir0pWz8MdpYm6AY6VPkGpbA7ZzmbzNcBJQ0GPvvyWgcY7QAhCgf9Uy89I03faLkg==} + stopwords-iso@1.1.0: + resolution: {integrity: sha512-I6GPS/E0zyieHehMRPQcqkiBMJKGgLta+1hREixhoLPqEA0AlVFiC43dl8uPpmkkeRdDMzYRWFWk5/l9x7nmNg==} + engines: {node: '>=0.10.0'} + + string_decoder@1.3.0: + resolution: {integrity: sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==} + + strip-json-comments@2.0.1: + resolution: {integrity: sha512-4gB8na07fecVVkOI6Rs4e7T6NOTki5EmL7TUduTs6bu3EdnSycntVJ4re8kgZA+wx9IueI2Y11bfbgwtzuE0KQ==} + engines: {node: '>=0.10.0'} + strip-literal@3.1.0: resolution: {integrity: sha512-8r3mkIM/2+PpjHoOtiAW8Rg3jJLHaV7xPwG+YRGrv6FP0wwk/toTpATxWYOW0BKdWwl82VT2tFYi5DlROa0Mxg==} @@ -995,6 +2127,24 @@ packages: engines: {node: '>=16 || 14 >=14.17'} hasBin: true + suffix-thumb@5.0.2: + resolution: {integrity: sha512-I5PWXAFKx3FYnI9a+dQMWNqTxoRt6vdBdb0O+BJ1sxXCWtSoQCusc13E58f+9p4MYx/qCnEMkD5jac6K2j3dgA==} + + supports-color@7.2.0: + resolution: {integrity: sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==} + engines: {node: '>=8'} + + sylvester@0.0.21: + resolution: {integrity: sha512-yUT0ukFkFEt4nb+NY+n2ag51aS/u9UHXoZw+A4jgD77/jzZsBoSDHuqysrVCBC4CYR4TYvUJq54ONpXgDBH8tA==} + engines: {node: '>=0.2.6'} + + tar-fs@2.1.5: + resolution: {integrity: sha512-OboTd8mmMhZDNPV+UjQcK9yKAatXu2aJ+r1w4im1Otd4M4fl2hwvdoXUxIYHFTHWK/3y3FarBP70v3vwmGlOxw==} + + tar-stream@2.2.0: + resolution: {integrity: sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ==} + engines: {node: '>=6'} + thenify-all@1.6.0: resolution: {integrity: sha512-RNxQH/qI8/t3thXJDwcstUO4zeqo64+Uy/+sNVRBx4Xn2OX+OZ9oP+iJnNFqplFra2ZUVeKCSa2oVWi3T4uVmA==} engines: {node: '>=0.8'} @@ -1024,6 +2174,17 @@ packages: resolution: {integrity: sha512-azl+t0z7pw/z958Gy9svOTuzqIk6xq+NSheJzn5MMWtWTFywIacg2wUlzKFGtt3cthx0r2SxMK0yzJOR0IES7Q==} engines: {node: '>=14.0.0'} + to-regex-range@5.0.1: + resolution: {integrity: sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==} + engines: {node: '>=8.0'} + + tr46@0.0.3: + resolution: {integrity: sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==} + + tr46@5.1.1: + resolution: {integrity: sha512-hdF5ZgjTqgAntKkklYw0R03MG2x/bSzTtkxmIRw/sTNV8YXsCJ1tfLAX23lhxhHJlEf3CRCOCGGWw3vI3GaSPw==} + engines: {node: '>=18'} + tree-kill@1.2.2: resolution: {integrity: sha512-L0Orpi8qGpRG//Nd+H90vFB+3iHnue1zSSGmNOOCh1GLJ7rUKVwV2HvijphGQS2UmhUZewS9VgvxYIdgr+fG1A==} hasBin: true @@ -1031,6 +2192,9 @@ packages: ts-interface-checker@0.1.13: resolution: {integrity: sha512-Y/arvbn+rrz3JCKl9C4kVNfTfSm2/mEp5FSz5EsZSANGPSlQrpRI5M4PKF+mJnE52jOO90PnPSc3Ur3bTQw0gA==} + tslib@2.8.1: + resolution: {integrity: sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==} + tsup@8.5.1: resolution: {integrity: sha512-xtgkqwdhpKWr3tKPmCkvYmS9xnQK3m3XgxZHwSUjvfTjp7YfXe5tT3GgWi0F2N+ZSMsOeWeZFh7ZZFg5iPhing==} engines: {node: '>=18'} @@ -1050,11 +2214,14 @@ packages: typescript: optional: true - tsx@4.22.4: - resolution: {integrity: sha512-X8EX+XV4QR5xCsrgxaED954zTDfY8KqlDtskKEL0cHhyS/P8b4IFOvGDQpsC9Q1XnLq915wEfwwY/zzskCtmhg==} + tsx@4.23.1: + resolution: {integrity: sha512-GQHnkIfxyx1wYCOS/wonik5MVRZU9hi1TEZmzGZSCJB1y9YgoZ8H6itNE/u4suE+yLmOzuE4E5S4TZ/ZX2wcWQ==} engines: {node: '>=18.0.0'} hasBin: true + tunnel-agent@0.6.0: + resolution: {integrity: sha512-McnNiV1l8RYeY8tBgEpuodCC1mLUdbSN+CYBL7kJsJNInOP8UjDDEwdk6Mw60vdLLrr5NHKZhMAOSrR2NZuQ+w==} + typescript@5.9.3: resolution: {integrity: sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==} engines: {node: '>=14.17'} @@ -1063,9 +2230,33 @@ packages: ufo@1.6.4: resolution: {integrity: sha512-JFNbkD1Svwe0KvGi8GOeLcP4kAWQ609twvCdcHxq1oSL8svv39ZuSvajcD8B+5D0eL4+s1Is2D/O6KN3qcTeRA==} + underscore@1.13.8: + resolution: {integrity: sha512-DXtD3ZtEQzc7M8m4cXotyHR+FAS18C64asBYY5vqZexfYryNNnDc02W4hKg3rdQuqOYas1jkseX0+nZXjTXnvQ==} + + undici-types@5.26.5: + resolution: {integrity: sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==} + undici-types@7.19.2: resolution: {integrity: sha512-qYVnV5OEm2AW8cJMCpdV20CDyaN3g0AjDlOGf1OW4iaDEx8MwdtChUp4zu4H0VP3nDRF/8RKWH+IPp9uW0YGZg==} + undici-types@7.24.6: + resolution: {integrity: sha512-WRNW+sJgj5OBN4/0JpHFqtqzhpbnV0GuB+OozA9gCL7a993SmU+1JBZCzLNxYsbMfIeDL+lTsphD5jN5N+n0zg==} + + undici@6.27.0: + resolution: {integrity: sha512-YmfV3YnEDzXRC5lZ2jWtWWHKGUm1zIt8AhesR1tens+HTNv+YZlN/dp6G727LOvMJ8xjP9Be7Y2Sdr96LDm+pg==} + engines: {node: '>=18.17'} + + util-deprecate@1.0.2: + resolution: {integrity: sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==} + + uuid@11.1.1: + resolution: {integrity: sha512-vIYxrBCC/N/K+Js3qSN88go7kIfNPssr/hHCesKCQNAjmgvYS2oqr69kIufEG+O4+PfezOH4EbIeHCfFov8ZgQ==} + hasBin: true + + uuid@13.0.2: + resolution: {integrity: sha512-vzi9uRZ926x4XV73S/4qQaTwPXM2JBj6/6lI/byHH1jOpCzb0zDbfytgA9LcN/hzb2l7WQSQnxITOVx5un/wGw==} + hasBin: true + viem@2.48.8: resolution: {integrity: sha512-Xj3Nrt66SKtn06kczU91ELn9Difr84ZM5A62BTlaisT5lpgt058i2mBkfMZCXHGb1ocOLjzC2ztPhD0Lvky7uQ==} peerDependencies: @@ -1147,11 +2338,43 @@ packages: jsdom: optional: true + web-streams-polyfill@3.3.3: + resolution: {integrity: sha512-d2JWLCivmZYTSIoge9MsgFCZrt571BikcWGYkjC1khllbTeDlGqZ2D8vD8E/lJa8WGWbb7Plm8/XJYV7IJHZZw==} + engines: {node: '>= 8'} + + web-streams-polyfill@4.0.0-beta.3: + resolution: {integrity: sha512-QW95TCTaHmsYfHDybGMwO5IJIM93I/6vTRk+daHTWFPhwh+C8Cg7j7XyKrwrj8Ib6vYXe0ocYNrmzY4xAAN6ug==} + engines: {node: '>= 14'} + + webidl-conversions@3.0.1: + resolution: {integrity: sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==} + + webidl-conversions@7.0.0: + resolution: {integrity: sha512-VwddBukDzu71offAQR975unBIGqfKZpM+8ZX6ySk8nYhVoo5CYaZyzt3YBvYtRtO+aoGlqxPg/B87NGVZ/fu6g==} + engines: {node: '>=12'} + + whatwg-fetch@3.6.20: + resolution: {integrity: sha512-EqhiFU6daOA8kpjOWTL0olhVOF3i7OrFzSYiGsEMB8GcXS+RrzauAERX65xMeNWVqxA6HXH2m69Z9LaKKdisfg==} + + whatwg-url@14.2.0: + resolution: {integrity: sha512-De72GdQZzNTUBBChsXueQUnPKDkg/5A5zp7pFDuQAj5UFoENpiACU0wlCvzpAGnTkj++ihpKwKyYewn/XNUbKw==} + engines: {node: '>=18'} + + whatwg-url@5.0.0: + resolution: {integrity: sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==} + why-is-node-running@2.3.0: resolution: {integrity: sha512-hUrmaWBdVDcxvYqnyh09zunKzROWjbZTiNy8dBEjkS7ehEDQibXJ7XvlmtbwuTclUiIyN+CyXQD4Vmko8fNm8w==} engines: {node: '>=8'} hasBin: true + wordnet-db@3.1.14: + resolution: {integrity: sha512-zVyFsvE+mq9MCmwXUWHIcpfbrHHClZWZiVOzKSxNJruIcFn2RbY55zkhiAMMxM8zCVSmtNiViq8FsAZSFpMYag==} + engines: {node: '>=0.6.0'} + + wrappy@1.0.2: + resolution: {integrity: sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==} + ws@8.21.0: resolution: {integrity: sha512-Vsp28b7DRcimFQvrqu2Wek3z1iYxDCWqHYB8Qsnk/S4RfaCQzPGPyBNuVjJV3cd6UiKtUtp6sNM77gWvzcCH+g==} engines: {node: '>=10.0.0'} @@ -1164,13 +2387,26 @@ packages: utf-8-validate: optional: true + wsl-utils@0.1.0: + resolution: {integrity: sha512-h3Fbisa2nKGPxCpm89Hk33lBLsnaGBvctQopaBSOW/uIs6FTe1ATyAnKFJrzVs9vpGdsTe73WF3V4lIsk4Gacw==} + engines: {node: '>=18'} + + xtend@4.0.2: + resolution: {integrity: sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==} + engines: {node: '>=0.4'} + yaml@2.8.4: resolution: {integrity: sha512-ml/JPOj9fOQK8RNnWojA67GbZ0ApXAUlN2UQclwv2eVgTgn7O9gg9o7paZWKMp4g0H3nTLtS9LVzhkpOFIKzog==} engines: {node: '>= 14.6'} hasBin: true - zod@4.4.2: - resolution: {integrity: sha512-IynmDyxsEsb9RKzO3J9+4SxXnl2FTFSzNBaKKaMV6tsSk0rw9gYw9gs+JFCq/qk2LCZ78KDwyj+Z289TijSkUw==} + zod-to-json-schema@3.25.2: + resolution: {integrity: sha512-O/PgfnpT1xKSDeQYSCfRI5Gy3hPf91mKVDuYLUHZJMiDFptvP41MSnWofm8dnCm0256ZNfZIM7DSzuSMAFnjHA==} + peerDependencies: + zod: ^3.25.28 || ^4 + + zod@3.25.76: + resolution: {integrity: sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ==} zod@4.4.3: resolution: {integrity: sha512-ytENFjIJFl2UwYglde2jchW2Hwm4GJFLDiSXWdTrJQBIN9Fcyp7n4DhxJEiWNAJMV1/BqWfW/kkg71UDcHJyTQ==} @@ -1179,6 +2415,18 @@ snapshots: '@adraffy/ens-normalize@1.11.1': {} + '@anthropic-ai/sdk@0.40.1': + dependencies: + '@types/node': 18.19.130 + '@types/node-fetch': 2.6.13 + abort-controller: 3.0.0 + agentkeepalive: 4.6.0 + form-data-encoder: 1.7.2 + formdata-node: 4.4.1 + node-fetch: 2.7.0 + transitivePeerDependencies: + - encoding + '@asteasolutions/zod-to-openapi@8.5.0(zod@4.4.3)': dependencies: openapi3-ts: 4.5.0 @@ -1190,41 +2438,160 @@ snapshots: optionalDependencies: zod: 4.4.3 - '@biomejs/biome@2.4.15': - optionalDependencies: - '@biomejs/cli-darwin-arm64': 2.4.15 - '@biomejs/cli-darwin-x64': 2.4.15 - '@biomejs/cli-linux-arm64': 2.4.15 - '@biomejs/cli-linux-arm64-musl': 2.4.15 - '@biomejs/cli-linux-x64': 2.4.15 - '@biomejs/cli-linux-x64-musl': 2.4.15 - '@biomejs/cli-win32-arm64': 2.4.15 - '@biomejs/cli-win32-x64': 2.4.15 + '@azure/abort-controller@2.2.0': + dependencies: + tslib: 2.8.1 + + '@azure/core-auth@1.11.0': + dependencies: + '@azure/abort-controller': 2.2.0 + '@azure/core-util': 1.14.0 + tslib: 2.8.1 + transitivePeerDependencies: + - supports-color + + '@azure/core-client@1.11.0': + dependencies: + '@azure/abort-controller': 2.2.0 + '@azure/core-auth': 1.11.0 + '@azure/core-rest-pipeline': 1.25.0 + '@azure/core-tracing': 1.4.0 + '@azure/core-util': 1.14.0 + '@azure/logger': 1.4.0 + tslib: 2.8.1 + transitivePeerDependencies: + - supports-color + + '@azure/core-http-compat@2.5.0(@azure/core-client@1.11.0)(@azure/core-rest-pipeline@1.25.0)': + dependencies: + '@azure/abort-controller': 2.2.0 + '@azure/core-client': 1.11.0 + '@azure/core-rest-pipeline': 1.25.0 + + '@azure/core-paging@1.7.0': + dependencies: + tslib: 2.8.1 + + '@azure/core-rest-pipeline@1.25.0': + dependencies: + '@azure/abort-controller': 2.2.0 + '@azure/core-auth': 1.11.0 + '@azure/core-tracing': 1.4.0 + '@azure/core-util': 1.14.0 + '@azure/logger': 1.4.0 + '@typespec/ts-http-runtime': 0.3.7 + tslib: 2.8.1 + transitivePeerDependencies: + - supports-color - '@biomejs/cli-darwin-arm64@2.4.15': + '@azure/core-tracing@1.4.0': + dependencies: + tslib: 2.8.1 + + '@azure/core-util@1.14.0': + dependencies: + '@azure/abort-controller': 2.2.0 + '@typespec/ts-http-runtime': 0.3.7 + tslib: 2.8.1 + transitivePeerDependencies: + - supports-color + + '@azure/identity@4.13.1': + dependencies: + '@azure/abort-controller': 2.2.0 + '@azure/core-auth': 1.11.0 + '@azure/core-client': 1.11.0 + '@azure/core-rest-pipeline': 1.25.0 + '@azure/core-tracing': 1.4.0 + '@azure/core-util': 1.14.0 + '@azure/logger': 1.4.0 + '@azure/msal-browser': 5.17.1 + '@azure/msal-node': 5.4.1 + open: 10.2.0 + tslib: 2.8.1 + transitivePeerDependencies: + - supports-color + + '@azure/logger@1.4.0': + dependencies: + '@typespec/ts-http-runtime': 0.3.7 + tslib: 2.8.1 + transitivePeerDependencies: + - supports-color + + '@azure/msal-browser@5.17.1': + dependencies: + '@azure/msal-common': 16.11.2 + + '@azure/msal-common@16.11.2': {} + + '@azure/msal-node@5.4.1': + dependencies: + '@azure/msal-common': 16.11.2 + jsonwebtoken: 9.0.3 + + '@azure/search-documents@12.2.0': + dependencies: + '@azure/core-auth': 1.11.0 + '@azure/core-client': 1.11.0 + '@azure/core-http-compat': 2.5.0(@azure/core-client@1.11.0)(@azure/core-rest-pipeline@1.25.0) + '@azure/core-paging': 1.7.0 + '@azure/core-rest-pipeline': 1.25.0 + '@azure/core-tracing': 1.4.0 + '@azure/core-util': 1.14.0 + '@azure/logger': 1.4.0 + events: 3.3.0 + tslib: 2.8.1 + transitivePeerDependencies: + - supports-color + + '@babel/code-frame@7.29.7': + dependencies: + '@babel/helper-validator-identifier': 7.29.7 + js-tokens: 4.0.0 + picocolors: 1.1.1 + + '@babel/helper-validator-identifier@7.29.7': {} + + '@biomejs/biome@2.5.4': + optionalDependencies: + '@biomejs/cli-darwin-arm64': 2.5.4 + '@biomejs/cli-darwin-x64': 2.5.4 + '@biomejs/cli-linux-arm64': 2.5.4 + '@biomejs/cli-linux-arm64-musl': 2.5.4 + '@biomejs/cli-linux-x64': 2.5.4 + '@biomejs/cli-linux-x64-musl': 2.5.4 + '@biomejs/cli-win32-arm64': 2.5.4 + '@biomejs/cli-win32-x64': 2.5.4 + + '@biomejs/cli-darwin-arm64@2.5.4': optional: true - '@biomejs/cli-darwin-x64@2.4.15': + '@biomejs/cli-darwin-x64@2.5.4': optional: true - '@biomejs/cli-linux-arm64-musl@2.4.15': + '@biomejs/cli-linux-arm64-musl@2.5.4': optional: true - '@biomejs/cli-linux-arm64@2.4.15': + '@biomejs/cli-linux-arm64@2.5.4': optional: true - '@biomejs/cli-linux-x64-musl@2.4.15': + '@biomejs/cli-linux-x64-musl@2.5.4': optional: true - '@biomejs/cli-linux-x64@2.4.15': + '@biomejs/cli-linux-x64@2.5.4': optional: true - '@biomejs/cli-win32-arm64@2.4.15': + '@biomejs/cli-win32-arm64@2.5.4': optional: true - '@biomejs/cli-win32-x64@2.4.15': + '@biomejs/cli-win32-x64@2.5.4': optional: true + '@cfworker/json-schema@4.1.1': {} + + '@cloudflare/workers-types@4.20260702.1': {} + '@esbuild/aix-ppc64@0.27.7': optional: true @@ -1381,9 +2748,37 @@ snapshots: '@esbuild/win32-x64@0.28.1': optional: true - '@hono/node-server@2.0.1(hono@4.12.30)': + '@google/genai@1.52.0': dependencies: - hono: 4.12.30 + google-auth-library: 10.9.0 + p-retry: 4.6.2 + protobufjs: 7.6.5 + ws: 8.21.0 + transitivePeerDependencies: + - bufferutil + - supports-color + - utf-8-validate + + '@hono/node-server@2.0.1(hono@4.12.30)': + dependencies: + hono: 4.12.30 + + '@jest/expect-utils@29.7.0': + dependencies: + jest-get-type: 29.6.3 + + '@jest/schemas@29.6.3': + dependencies: + '@sinclair/typebox': 0.27.12 + + '@jest/types@29.6.3': + dependencies: + '@jest/schemas': 29.6.3 + '@types/istanbul-lib-coverage': 2.0.6 + '@types/istanbul-reports': 3.0.4 + '@types/node': 25.9.5 + '@types/yargs': 17.0.35 + chalk: 4.1.2 '@jridgewell/gen-mapping@0.3.13': dependencies: @@ -1399,6 +2794,35 @@ snapshots: '@jridgewell/resolve-uri': 3.1.2 '@jridgewell/sourcemap-codec': 1.5.5 + '@langchain/core@1.2.3(@opentelemetry/api@1.9.1)(ws@8.21.0)': + dependencies: + '@cfworker/json-schema': 4.1.1 + '@standard-schema/spec': 1.1.0 + js-tiktoken: 1.0.21 + langsmith: 0.8.3(@opentelemetry/api@1.9.1)(ws@8.21.0) + mustache: 4.2.0 + p-queue: 6.6.2 + zod: 4.4.3 + transitivePeerDependencies: + - '@opentelemetry/api' + - '@opentelemetry/exporter-trace-otlp-proto' + - '@opentelemetry/sdk-trace-base' + - openai + - ws + + '@mistralai/mistralai@1.15.1': + dependencies: + ws: 8.21.0 + zod: 4.4.3 + zod-to-json-schema: 3.25.2(zod@4.4.3) + transitivePeerDependencies: + - bufferutil + - utf-8-validate + + '@mongodb-js/saslprep@1.4.12': + dependencies: + sparse-bitfield: 3.0.3 + '@neo4j-labs/agent-memory@0.4.0': {} '@noble/ciphers@1.3.0': {} @@ -1417,6 +2841,56 @@ snapshots: '@opentelemetry/api@1.9.1': {} + '@protobufjs/aspromise@1.1.2': {} + + '@protobufjs/base64@1.1.2': {} + + '@protobufjs/codegen@2.0.5': {} + + '@protobufjs/eventemitter@1.1.1': {} + + '@protobufjs/fetch@1.1.1': + dependencies: + '@protobufjs/aspromise': 1.1.2 + + '@protobufjs/float@1.0.2': {} + + '@protobufjs/path@1.1.2': {} + + '@protobufjs/pool@1.1.0': {} + + '@protobufjs/utf8@1.1.2': {} + + '@qdrant/js-client-rest@1.18.0(typescript@5.9.3)': + dependencies: + '@qdrant/openapi-typescript-fetch': 1.2.6 + typescript: 5.9.3 + undici: 6.27.0 + + '@qdrant/openapi-typescript-fetch@1.2.6': {} + + '@redis/bloom@5.12.1(@redis/client@5.12.1(@opentelemetry/api@1.9.1))': + dependencies: + '@redis/client': 5.12.1(@opentelemetry/api@1.9.1) + + '@redis/client@5.12.1(@opentelemetry/api@1.9.1)': + dependencies: + cluster-key-slot: 1.1.2 + optionalDependencies: + '@opentelemetry/api': 1.9.1 + + '@redis/json@5.12.1(@redis/client@5.12.1(@opentelemetry/api@1.9.1))': + dependencies: + '@redis/client': 5.12.1(@opentelemetry/api@1.9.1) + + '@redis/search@5.12.1(@redis/client@5.12.1(@opentelemetry/api@1.9.1))': + dependencies: + '@redis/client': 5.12.1(@opentelemetry/api@1.9.1) + + '@redis/time-series@5.12.1(@redis/client@5.12.1(@opentelemetry/api@1.9.1))': + dependencies: + '@redis/client': 5.12.1(@opentelemetry/api@1.9.1) + '@rollup/rollup-android-arm-eabi@4.60.2': optional: true @@ -1518,6 +2992,42 @@ snapshots: '@noble/hashes': 2.2.0 '@scure/base': 2.2.0 + '@sinclair/typebox@0.27.12': {} + + '@standard-schema/spec@1.1.0': {} + + '@supabase/auth-js@2.110.7': + dependencies: + tslib: 2.8.1 + + '@supabase/functions-js@2.110.7': + dependencies: + tslib: 2.8.1 + + '@supabase/phoenix@0.4.5': {} + + '@supabase/postgrest-js@2.110.7': + dependencies: + tslib: 2.8.1 + + '@supabase/realtime-js@2.110.7': + dependencies: + '@supabase/phoenix': 0.4.5 + tslib: 2.8.1 + + '@supabase/storage-js@2.110.7': + dependencies: + iceberg-js: 0.8.1 + tslib: 2.8.1 + + '@supabase/supabase-js@2.110.7': + dependencies: + '@supabase/auth-js': 2.110.7 + '@supabase/functions-js': 2.110.7 + '@supabase/postgrest-js': 2.110.7 + '@supabase/realtime-js': 2.110.7 + '@supabase/storage-js': 2.110.7 + '@tangle-network/agent-core@0.3.8': dependencies: '@tangle-network/agent-interface': 0.17.1 @@ -1528,7 +3038,7 @@ snapshots: '@tangle-network/agent-interface': 0.26.0 zod: 4.4.3 - '@tangle-network/agent-eval@0.122.7(typescript@5.9.3)': + '@tangle-network/agent-eval@0.122.8(typescript@5.9.3)': dependencies: '@asteasolutions/zod-to-openapi': 8.5.0(zod@4.4.3) '@ax-llm/ax': 23.0.0(zod@4.4.3) @@ -1565,7 +3075,7 @@ snapshots: '@noble/hashes': 1.8.0 zod: 4.4.3 - '@tangle-network/sandbox@0.9.7(viem@2.48.8(typescript@5.9.3)(zod@4.4.2))': + '@tangle-network/sandbox@0.9.7(viem@2.48.8(typescript@5.9.3)(zod@4.4.3))': dependencies: '@tangle-network/agent-core': 0.3.8 '@tangle-network/agent-interface': 0.13.0 @@ -1578,7 +3088,7 @@ snapshots: dependencies: '@scure/bip32': 2.2.0 '@scure/bip39': 2.2.0 - '@tangle-network/sandbox': 0.9.7(viem@2.48.8(typescript@5.9.3)(zod@4.4.2)) + '@tangle-network/sandbox': 0.9.7(viem@2.48.8(typescript@5.9.3)(zod@4.4.3)) '@tangle-network/tcloud-attestation': 0.1.1 commander: 14.0.3 viem: 2.48.8(typescript@5.9.3)(zod@4.4.3) @@ -1601,16 +3111,74 @@ snapshots: '@types/estree@1.0.8': {} + '@types/istanbul-lib-coverage@2.0.6': {} + + '@types/istanbul-lib-report@3.0.3': + dependencies: + '@types/istanbul-lib-coverage': 2.0.6 + + '@types/istanbul-reports@3.0.4': + dependencies: + '@types/istanbul-lib-report': 3.0.3 + + '@types/jest@29.5.14': + dependencies: + expect: 29.7.0 + pretty-format: 29.7.0 + + '@types/node-fetch@2.6.13': + dependencies: + '@types/node': 25.6.0 + form-data: 4.0.6 + + '@types/node@18.19.130': + dependencies: + undici-types: 5.26.5 + '@types/node@25.6.0': dependencies: undici-types: 7.19.2 + '@types/node@25.9.5': + dependencies: + undici-types: 7.24.6 + + '@types/pg@8.11.0': + dependencies: + '@types/node': 25.9.5 + pg-protocol: 1.15.0 + pg-types: 4.1.0 + '@types/proper-lockfile@4.1.4': dependencies: '@types/retry': 0.12.5 + '@types/retry@0.12.0': {} + '@types/retry@0.12.5': {} + '@types/stack-utils@2.0.3': {} + + '@types/webidl-conversions@7.0.3': {} + + '@types/whatwg-url@13.0.0': + dependencies: + '@types/webidl-conversions': 7.0.3 + + '@types/yargs-parser@21.0.3': {} + + '@types/yargs@17.0.35': + dependencies: + '@types/yargs-parser': 21.0.3 + + '@typespec/ts-http-runtime@0.3.7': + dependencies: + http-proxy-agent: 7.0.2 + https-proxy-agent: 7.0.6 + tslib: 2.8.1 + transitivePeerDependencies: + - supports-color + '@vitest/expect@3.2.4': dependencies: '@types/chai': 5.2.3 @@ -1619,13 +3187,13 @@ snapshots: chai: 5.3.3 tinyrainbow: 2.0.0 - '@vitest/mocker@3.2.4(vite@7.3.2(@types/node@25.6.0)(tsx@4.22.4)(yaml@2.8.4))': + '@vitest/mocker@3.2.4(vite@7.3.2(@types/node@25.6.0)(tsx@4.23.1)(yaml@2.8.4))': dependencies: '@vitest/spy': 3.2.4 estree-walker: 3.0.3 magic-string: 0.30.21 optionalDependencies: - vite: 7.3.2(@types/node@25.6.0)(tsx@4.22.4)(yaml@2.8.4) + vite: 7.3.2(@types/node@25.6.0)(tsx@4.23.1)(yaml@2.8.4) '@vitest/pretty-format@3.2.4': dependencies: @@ -1658,12 +3226,92 @@ snapshots: typescript: 5.9.3 zod: 4.4.3 + abort-controller@3.0.0: + dependencies: + event-target-shim: 5.0.1 + acorn@8.16.0: {} + afinn-165-financialmarketnews@3.0.0: {} + + afinn-165@2.0.2: {} + + agent-base@6.0.2: + dependencies: + debug: 4.4.3 + transitivePeerDependencies: + - supports-color + + agent-base@7.1.4: {} + + agentkeepalive@4.6.0: + dependencies: + humanize-ms: 1.2.1 + + ansi-styles@4.3.0: + dependencies: + color-convert: 2.0.1 + + ansi-styles@5.2.0: {} + any-promise@1.3.0: {} + apparatus@0.0.10: + dependencies: + sylvester: 0.0.21 + assertion-error@2.0.1: {} + asynckit@0.4.0: {} + + axios@1.18.1: + dependencies: + follow-redirects: 1.16.0 + form-data: 4.0.6 + https-proxy-agent: 5.0.1 + proxy-from-env: 2.1.0 + transitivePeerDependencies: + - debug + - supports-color + + base-64@0.1.0: {} + + base64-js@1.5.1: {} + + better-sqlite3@12.11.1: + dependencies: + bindings: 1.5.0 + prebuild-install: 7.1.3 + + bignumber.js@9.3.1: {} + + bindings@1.5.0: + dependencies: + file-uri-to-path: 1.0.0 + + bl@4.1.0: + dependencies: + buffer: 5.7.1 + inherits: 2.0.4 + readable-stream: 3.6.2 + + braces@3.0.3: + dependencies: + fill-range: 7.1.1 + + bson@7.3.1: {} + + buffer-equal-constant-time@1.0.1: {} + + buffer@5.7.1: + dependencies: + base64-js: 1.5.1 + ieee754: 1.2.1 + + bundle-name@4.1.0: + dependencies: + run-applescript: 7.1.0 + bundle-require@5.1.0(esbuild@0.27.7): dependencies: esbuild: 0.27.7 @@ -1671,6 +3319,11 @@ snapshots: cac@6.7.14: {} + call-bind-apply-helpers@1.0.2: + dependencies: + es-errors: 1.3.0 + function-bind: 1.1.2 + chai@5.3.3: dependencies: assertion-error: 2.0.1 @@ -1679,28 +3332,132 @@ snapshots: loupe: 3.2.1 pathval: 2.0.1 + chalk@4.1.2: + dependencies: + ansi-styles: 4.3.0 + supports-color: 7.2.0 + + charenc@0.0.2: {} + check-error@2.1.3: {} chokidar@4.0.3: dependencies: readdirp: 4.1.2 + chownr@1.1.4: {} + + ci-info@3.9.0: {} + + cloudflare@4.5.0: + dependencies: + '@types/node': 18.19.130 + '@types/node-fetch': 2.6.13 + abort-controller: 3.0.0 + agentkeepalive: 4.6.0 + form-data-encoder: 1.7.2 + formdata-node: 4.4.1 + node-fetch: 2.7.0 + transitivePeerDependencies: + - encoding + + cluster-key-slot@1.1.2: {} + + color-convert@2.0.1: + dependencies: + color-name: 1.1.4 + + color-name@1.1.4: {} + + combined-stream@1.0.8: + dependencies: + delayed-stream: 1.0.0 + commander@14.0.3: {} commander@4.1.1: {} + compromise@14.16.0: + dependencies: + efrt: 2.7.0 + grad-school: 0.0.5 + suffix-thumb: 5.0.2 + confbox@0.1.8: {} consola@3.4.2: {} + crypt@0.0.2: {} + + data-uri-to-buffer@4.0.1: {} + debug@4.4.3: dependencies: ms: 2.1.3 + decompress-response@6.0.0: + dependencies: + mimic-response: 3.1.0 + deep-eql@5.0.2: {} + deep-extend@0.6.0: {} + + default-browser-id@5.0.1: {} + + default-browser@5.5.0: + dependencies: + bundle-name: 4.1.0 + default-browser-id: 5.0.1 + + define-lazy-prop@3.0.0: {} + + delayed-stream@1.0.0: {} + + detect-libc@2.1.2: {} + + diff-sequences@29.6.3: {} + + digest-fetch@1.3.0: + dependencies: + base-64: 0.1.0 + md5: 2.3.0 + + dotenv@17.4.2: {} + + dunder-proto@1.0.1: + dependencies: + call-bind-apply-helpers: 1.0.2 + es-errors: 1.3.0 + gopd: 1.2.0 + + ecdsa-sig-formatter@1.0.11: + dependencies: + safe-buffer: 5.2.1 + + efrt@2.7.0: {} + + end-of-stream@1.4.5: + dependencies: + once: 1.4.0 + + es-define-property@1.0.1: {} + + es-errors@1.3.0: {} + es-module-lexer@1.7.0: {} + es-object-atoms@1.1.2: + dependencies: + es-errors: 1.3.0 + + es-set-tostringtag@2.1.0: + dependencies: + es-errors: 1.3.0 + get-intrinsic: 1.3.0 + has-tostringtag: 1.0.2 + hasown: 2.0.4 + esbuild@0.27.7: optionalDependencies: '@esbuild/aix-ppc64': 0.27.7 @@ -1759,51 +3516,391 @@ snapshots: '@esbuild/win32-ia32': 0.28.1 '@esbuild/win32-x64': 0.28.1 + escape-string-regexp@2.0.0: {} + estree-walker@3.0.3: dependencies: '@types/estree': 1.0.8 + event-target-shim@5.0.1: {} + + eventemitter3@4.0.7: {} + eventemitter3@5.0.1: {} + events@3.3.0: {} + + expand-template@2.0.3: {} + expect-type@1.3.0: {} + expect@29.7.0: + dependencies: + '@jest/expect-utils': 29.7.0 + jest-get-type: 29.6.3 + jest-matcher-utils: 29.7.0 + jest-message-util: 29.7.0 + jest-util: 29.7.0 + + extend@3.0.2: {} + fdir@6.5.0(picomatch@4.0.4): optionalDependencies: picomatch: 4.0.4 + fetch-blob@3.2.0: + dependencies: + node-domexception: 1.0.0 + web-streams-polyfill: 3.3.3 + + file-uri-to-path@1.0.0: {} + + fill-range@7.1.1: + dependencies: + to-regex-range: 5.0.1 + fix-dts-default-cjs-exports@1.0.1: dependencies: magic-string: 0.30.21 mlly: 1.8.2 rollup: 4.60.2 + follow-redirects@1.16.0: {} + + form-data-encoder@1.7.2: {} + + form-data@4.0.6: + dependencies: + asynckit: 0.4.0 + combined-stream: 1.0.8 + es-set-tostringtag: 2.1.0 + hasown: 2.0.4 + mime-types: 2.1.35 + + formdata-node@4.4.1: + dependencies: + node-domexception: 1.0.0 + web-streams-polyfill: 4.0.0-beta.3 + + formdata-polyfill@4.0.10: + dependencies: + fetch-blob: 3.2.0 + + fs-constants@1.0.0: {} + fsevents@2.3.3: optional: true + function-bind@1.1.2: {} + + gaxios@7.2.0: + dependencies: + extend: 3.0.2 + https-proxy-agent: 7.0.6 + node-fetch: 3.3.2 + transitivePeerDependencies: + - supports-color + + gcp-metadata@8.1.2: + dependencies: + gaxios: 7.2.0 + google-logging-utils: 1.1.3 + json-bigint: 1.0.0 + transitivePeerDependencies: + - supports-color + + get-intrinsic@1.3.0: + dependencies: + call-bind-apply-helpers: 1.0.2 + es-define-property: 1.0.1 + es-errors: 1.3.0 + es-object-atoms: 1.1.2 + function-bind: 1.1.2 + get-proto: 1.0.1 + gopd: 1.2.0 + has-symbols: 1.1.0 + hasown: 2.0.4 + math-intrinsics: 1.1.0 + + get-proto@1.0.1: + dependencies: + dunder-proto: 1.0.1 + es-object-atoms: 1.1.2 + + github-from-package@0.0.0: {} + + google-auth-library@10.9.0: + dependencies: + base64-js: 1.5.1 + ecdsa-sig-formatter: 1.0.11 + gaxios: 7.2.0 + gcp-metadata: 8.1.2 + google-logging-utils: 1.1.3 + jws: 4.0.1 + transitivePeerDependencies: + - supports-color + + google-logging-utils@1.1.3: {} + + gopd@1.2.0: {} + graceful-fs@4.2.11: {} + grad-school@0.0.5: {} + + groq-sdk@0.3.0: + dependencies: + '@types/node': 18.19.130 + '@types/node-fetch': 2.6.13 + abort-controller: 3.0.0 + agentkeepalive: 4.6.0 + digest-fetch: 1.3.0 + form-data-encoder: 1.7.2 + formdata-node: 4.4.1 + node-fetch: 2.7.0 + web-streams-polyfill: 3.3.3 + transitivePeerDependencies: + - encoding + + has-flag@4.0.0: {} + + has-symbols@1.1.0: {} + + has-tostringtag@1.0.2: + dependencies: + has-symbols: 1.1.0 + + hasown@2.0.4: + dependencies: + function-bind: 1.1.2 + hono@4.12.30: {} + http-proxy-agent@7.0.2: + dependencies: + agent-base: 7.1.4 + debug: 4.4.3 + transitivePeerDependencies: + - supports-color + + https-proxy-agent@5.0.1: + dependencies: + agent-base: 6.0.2 + debug: 4.4.3 + transitivePeerDependencies: + - supports-color + + https-proxy-agent@7.0.6: + dependencies: + agent-base: 7.1.4 + debug: 4.4.3 + transitivePeerDependencies: + - supports-color + + humanize-ms@1.2.1: + dependencies: + ms: 2.1.3 + + iceberg-js@0.8.1: {} + + ieee754@1.2.1: {} + + inherits@2.0.4: {} + + ini@1.3.8: {} + + is-buffer@1.1.6: {} + + is-docker@3.0.0: {} + + is-inside-container@1.0.0: + dependencies: + is-docker: 3.0.0 + + is-number@7.0.0: {} + + is-wsl@3.1.1: + dependencies: + is-inside-container: 1.0.0 + isows@1.0.7(ws@8.21.0): dependencies: ws: 8.21.0 + jest-diff@29.7.0: + dependencies: + chalk: 4.1.2 + diff-sequences: 29.6.3 + jest-get-type: 29.6.3 + pretty-format: 29.7.0 + + jest-get-type@29.6.3: {} + + jest-matcher-utils@29.7.0: + dependencies: + chalk: 4.1.2 + jest-diff: 29.7.0 + jest-get-type: 29.6.3 + pretty-format: 29.7.0 + + jest-message-util@29.7.0: + dependencies: + '@babel/code-frame': 7.29.7 + '@jest/types': 29.6.3 + '@types/stack-utils': 2.0.3 + chalk: 4.1.2 + graceful-fs: 4.2.11 + micromatch: 4.0.8 + pretty-format: 29.7.0 + slash: 3.0.0 + stack-utils: 2.0.6 + + jest-util@29.7.0: + dependencies: + '@jest/types': 29.6.3 + '@types/node': 25.9.5 + chalk: 4.1.2 + ci-info: 3.9.0 + graceful-fs: 4.2.11 + picomatch: 2.3.2 + joycon@3.1.1: {} + js-tiktoken@1.0.21: + dependencies: + base64-js: 1.5.1 + + js-tokens@4.0.0: {} + js-tokens@9.0.1: {} + json-bigint@1.0.0: + dependencies: + bignumber.js: 9.3.1 + + jsonwebtoken@9.0.3: + dependencies: + jws: 4.0.1 + lodash.includes: 4.3.0 + lodash.isboolean: 3.0.3 + lodash.isinteger: 4.0.4 + lodash.isnumber: 3.0.3 + lodash.isplainobject: 4.0.6 + lodash.isstring: 4.0.1 + lodash.once: 4.1.1 + ms: 2.1.3 + semver: 7.8.5 + + jwa@2.0.1: + dependencies: + buffer-equal-constant-time: 1.0.1 + ecdsa-sig-formatter: 1.0.11 + safe-buffer: 5.2.1 + + jws@4.0.1: + dependencies: + jwa: 2.0.1 + safe-buffer: 5.2.1 + + kareem@3.3.0: {} + + langsmith@0.8.3(@opentelemetry/api@1.9.1)(ws@8.21.0): + dependencies: + p-queue: 6.6.2 + optionalDependencies: + '@opentelemetry/api': 1.9.1 + ws: 8.21.0 + lilconfig@3.1.3: {} lines-and-columns@1.2.4: {} load-tsconfig@0.2.5: {} + lodash.includes@4.3.0: {} + + lodash.isboolean@3.0.3: {} + + lodash.isinteger@4.0.4: {} + + lodash.isnumber@3.0.3: {} + + lodash.isplainobject@4.0.6: {} + + lodash.isstring@4.0.1: {} + + lodash.once@4.1.1: {} + + long@5.3.2: {} + loupe@3.2.1: {} magic-string@0.30.21: dependencies: '@jridgewell/sourcemap-codec': 1.5.5 + math-intrinsics@1.1.0: {} + + md5@2.3.0: + dependencies: + charenc: 0.0.2 + crypt: 0.0.2 + is-buffer: 1.1.6 + + mem0ai@3.1.0(@anthropic-ai/sdk@0.40.1)(@azure/identity@4.13.1)(@azure/search-documents@12.2.0)(@cloudflare/workers-types@4.20260702.1)(@google/genai@1.52.0)(@langchain/core@1.2.3(@opentelemetry/api@1.9.1)(ws@8.21.0))(@mistralai/mistralai@1.15.1)(@qdrant/js-client-rest@1.18.0(typescript@5.9.3))(@supabase/supabase-js@2.110.7)(@types/jest@29.5.14)(@types/pg@8.11.0)(better-sqlite3@12.11.1)(cloudflare@4.5.0)(compromise@14.16.0)(groq-sdk@0.3.0)(mongodb@7.2.0)(natural@8.1.1(@opentelemetry/api@1.9.1))(ollama@0.5.18)(pg@8.22.0)(redis@5.12.1(@opentelemetry/api@1.9.1))(ws@8.21.0): + dependencies: + '@anthropic-ai/sdk': 0.40.1 + '@azure/identity': 4.13.1 + '@azure/search-documents': 12.2.0 + '@cloudflare/workers-types': 4.20260702.1 + '@google/genai': 1.52.0 + '@langchain/core': 1.2.3(@opentelemetry/api@1.9.1)(ws@8.21.0) + '@mistralai/mistralai': 1.15.1 + '@qdrant/js-client-rest': 1.18.0(typescript@5.9.3) + '@supabase/supabase-js': 2.110.7 + '@types/jest': 29.5.14 + '@types/pg': 8.11.0 + axios: 1.18.1 + better-sqlite3: 12.11.1 + cloudflare: 4.5.0 + compromise: 14.16.0 + groq-sdk: 0.3.0 + natural: 8.1.1(@opentelemetry/api@1.9.1) + ollama: 0.5.18 + openai: 4.104.0(ws@8.21.0)(zod@3.25.76) + pg: 8.22.0 + redis: 5.12.1(@opentelemetry/api@1.9.1) + uuid: 11.1.1 + zod: 3.25.76 + optionalDependencies: + mongodb: 7.2.0 + transitivePeerDependencies: + - debug + - encoding + - supports-color + - ws + + memjs@1.3.2: {} + + memory-pager@1.5.0: {} + + micromatch@4.0.8: + dependencies: + braces: 3.0.3 + picomatch: 2.3.2 + + mime-db@1.52.0: {} + + mime-types@2.1.35: + dependencies: + mime-db: 1.52.0 + + mimic-response@3.1.0: {} + + minimist@1.2.8: {} + + mkdirp-classic@0.5.3: {} + mlly@1.8.2: dependencies: acorn: 8.16.0 @@ -1811,8 +3908,43 @@ snapshots: pkg-types: 1.3.1 ufo: 1.6.4 + mongodb-connection-string-url@7.0.2: + dependencies: + '@types/whatwg-url': 13.0.0 + whatwg-url: 14.2.0 + + mongodb@7.2.0: + dependencies: + '@mongodb-js/saslprep': 1.4.12 + bson: 7.3.1 + mongodb-connection-string-url: 7.0.2 + + mongoose@9.7.4: + dependencies: + '@standard-schema/spec': 1.1.0 + kareem: 3.3.0 + mongodb: 7.2.0 + mpath: 0.9.0 + mquery: 6.0.0 + ms: 2.1.3 + sift: 17.1.3 + transitivePeerDependencies: + - '@aws-sdk/credential-providers' + - '@mongodb-js/zstd' + - gcp-metadata + - kerberos + - mongodb-client-encryption + - snappy + - socks + + mpath@0.9.0: {} + + mquery@6.0.0: {} + ms@2.1.3: {} + mustache@4.2.0: {} + mz@2.7.0: dependencies: any-promise: 1.3.0 @@ -1821,8 +3953,86 @@ snapshots: nanoid@3.3.12: {} + napi-build-utils@2.0.0: {} + + natural@8.1.1(@opentelemetry/api@1.9.1): + dependencies: + afinn-165: 2.0.2 + afinn-165-financialmarketnews: 3.0.0 + apparatus: 0.0.10 + dotenv: 17.4.2 + memjs: 1.3.2 + mongoose: 9.7.4 + pg: 8.22.0 + redis: 5.12.1(@opentelemetry/api@1.9.1) + safe-stable-stringify: 2.5.0 + stopwords-iso: 1.1.0 + sylvester: 0.0.21 + underscore: 1.13.8 + uuid: 13.0.2 + wordnet-db: 3.1.14 + transitivePeerDependencies: + - '@aws-sdk/credential-providers' + - '@mongodb-js/zstd' + - '@node-rs/xxhash' + - '@opentelemetry/api' + - gcp-metadata + - kerberos + - mongodb-client-encryption + - pg-native + - snappy + - socks + + node-abi@3.94.0: + dependencies: + semver: 7.8.5 + + node-domexception@1.0.0: {} + + node-fetch@2.7.0: + dependencies: + whatwg-url: 5.0.0 + + node-fetch@3.3.2: + dependencies: + data-uri-to-buffer: 4.0.1 + fetch-blob: 3.2.0 + formdata-polyfill: 4.0.10 + object-assign@4.1.1: {} + obuf@1.1.2: {} + + ollama@0.5.18: + dependencies: + whatwg-fetch: 3.6.20 + + once@1.4.0: + dependencies: + wrappy: 1.0.2 + + open@10.2.0: + dependencies: + default-browser: 5.5.0 + define-lazy-prop: 3.0.0 + is-inside-container: 1.0.0 + wsl-utils: 0.1.0 + + openai@4.104.0(ws@8.21.0)(zod@3.25.76): + dependencies: + '@types/node': 18.19.130 + '@types/node-fetch': 2.6.13 + abort-controller: 3.0.0 + agentkeepalive: 4.6.0 + form-data-encoder: 1.7.2 + formdata-node: 4.4.1 + node-fetch: 2.7.0 + optionalDependencies: + ws: 8.21.0 + zod: 3.25.76 + transitivePeerDependencies: + - encoding + openapi3-ts@4.5.0: dependencies: yaml: 2.8.4 @@ -1842,12 +4052,77 @@ snapshots: transitivePeerDependencies: - zod + p-finally@1.0.0: {} + + p-queue@6.6.2: + dependencies: + eventemitter3: 4.0.7 + p-timeout: 3.2.0 + + p-retry@4.6.2: + dependencies: + '@types/retry': 0.12.0 + retry: 0.13.1 + + p-timeout@3.2.0: + dependencies: + p-finally: 1.0.0 + pathe@2.0.3: {} pathval@2.0.1: {} + pg-cloudflare@1.4.0: + optional: true + + pg-connection-string@2.14.0: {} + + pg-int8@1.0.1: {} + + pg-numeric@1.0.2: {} + + pg-pool@3.14.0(pg@8.22.0): + dependencies: + pg: 8.22.0 + + pg-protocol@1.15.0: {} + + pg-types@2.2.0: + dependencies: + pg-int8: 1.0.1 + postgres-array: 2.0.0 + postgres-bytea: 1.0.1 + postgres-date: 1.0.7 + postgres-interval: 1.2.0 + + pg-types@4.1.0: + dependencies: + pg-int8: 1.0.1 + pg-numeric: 1.0.2 + postgres-array: 3.0.4 + postgres-bytea: 3.0.0 + postgres-date: 2.1.0 + postgres-interval: 3.0.0 + postgres-range: 1.1.4 + + pg@8.22.0: + dependencies: + pg-connection-string: 2.14.0 + pg-pool: 3.14.0(pg@8.22.0) + pg-protocol: 1.15.0 + pg-types: 2.2.0 + pgpass: 1.0.5 + optionalDependencies: + pg-cloudflare: 1.4.0 + + pgpass@1.0.5: + dependencies: + split2: 4.2.0 + picocolors@1.1.1: {} + picomatch@2.3.2: {} + picomatch@4.0.4: {} pirates@4.0.7: {} @@ -1858,12 +4133,12 @@ snapshots: mlly: 1.8.2 pathe: 2.0.3 - postcss-load-config@6.0.1(postcss@8.5.13)(tsx@4.22.4)(yaml@2.8.4): + postcss-load-config@6.0.1(postcss@8.5.13)(tsx@4.23.1)(yaml@2.8.4): dependencies: lilconfig: 3.1.3 optionalDependencies: postcss: 8.5.13 - tsx: 4.22.4 + tsx: 4.23.1 yaml: 2.8.4 postcss@8.5.13: @@ -1872,18 +4147,112 @@ snapshots: picocolors: 1.1.1 source-map-js: 1.2.1 + postgres-array@2.0.0: {} + + postgres-array@3.0.4: {} + + postgres-bytea@1.0.1: {} + + postgres-bytea@3.0.0: + dependencies: + obuf: 1.1.2 + + postgres-date@1.0.7: {} + + postgres-date@2.1.0: {} + + postgres-interval@1.2.0: + dependencies: + xtend: 4.0.2 + + postgres-interval@3.0.0: {} + + postgres-range@1.1.4: {} + + prebuild-install@7.1.3: + dependencies: + detect-libc: 2.1.2 + expand-template: 2.0.3 + github-from-package: 0.0.0 + minimist: 1.2.8 + mkdirp-classic: 0.5.3 + napi-build-utils: 2.0.0 + node-abi: 3.94.0 + pump: 3.0.4 + rc: 1.2.8 + simple-get: 4.0.1 + tar-fs: 2.1.5 + tunnel-agent: 0.6.0 + + pretty-format@29.7.0: + dependencies: + '@jest/schemas': 29.6.3 + ansi-styles: 5.2.0 + react-is: 18.3.1 + proper-lockfile@4.1.2: dependencies: graceful-fs: 4.2.11 retry: 0.12.0 signal-exit: 3.0.7 + protobufjs@7.6.5: + dependencies: + '@protobufjs/aspromise': 1.1.2 + '@protobufjs/base64': 1.1.2 + '@protobufjs/codegen': 2.0.5 + '@protobufjs/eventemitter': 1.1.1 + '@protobufjs/fetch': 1.1.1 + '@protobufjs/float': 1.0.2 + '@protobufjs/path': 1.1.2 + '@protobufjs/pool': 1.1.0 + '@protobufjs/utf8': 1.1.2 + '@types/node': 25.9.5 + long: 5.3.2 + + proxy-from-env@2.1.0: {} + + pump@3.0.4: + dependencies: + end-of-stream: 1.4.5 + once: 1.4.0 + + punycode@2.3.1: {} + + rc@1.2.8: + dependencies: + deep-extend: 0.6.0 + ini: 1.3.8 + minimist: 1.2.8 + strip-json-comments: 2.0.1 + + react-is@18.3.1: {} + + readable-stream@3.6.2: + dependencies: + inherits: 2.0.4 + string_decoder: 1.3.0 + util-deprecate: 1.0.2 + readdirp@4.1.2: {} + redis@5.12.1(@opentelemetry/api@1.9.1): + dependencies: + '@redis/bloom': 5.12.1(@redis/client@5.12.1(@opentelemetry/api@1.9.1)) + '@redis/client': 5.12.1(@opentelemetry/api@1.9.1) + '@redis/json': 5.12.1(@redis/client@5.12.1(@opentelemetry/api@1.9.1)) + '@redis/search': 5.12.1(@redis/client@5.12.1(@opentelemetry/api@1.9.1)) + '@redis/time-series': 5.12.1(@redis/client@5.12.1(@opentelemetry/api@1.9.1)) + transitivePeerDependencies: + - '@node-rs/xxhash' + - '@opentelemetry/api' + resolve-from@5.0.0: {} retry@0.12.0: {} + retry@0.13.1: {} + rollup@4.60.2: dependencies: '@types/estree': 1.0.8 @@ -1915,18 +4284,56 @@ snapshots: '@rollup/rollup-win32-x64-msvc': 4.60.2 fsevents: 2.3.3 + run-applescript@7.1.0: {} + + safe-buffer@5.2.1: {} + + safe-stable-stringify@2.5.0: {} + + semver@7.8.5: {} + + sift@17.1.3: {} + siginfo@2.0.0: {} signal-exit@3.0.7: {} + simple-concat@1.0.1: {} + + simple-get@4.0.1: + dependencies: + decompress-response: 6.0.0 + once: 1.4.0 + simple-concat: 1.0.1 + + slash@3.0.0: {} + source-map-js@1.2.1: {} source-map@0.7.6: {} + sparse-bitfield@3.0.3: + dependencies: + memory-pager: 1.5.0 + + split2@4.2.0: {} + + stack-utils@2.0.6: + dependencies: + escape-string-regexp: 2.0.0 + stackback@0.0.2: {} std-env@3.10.0: {} + stopwords-iso@1.1.0: {} + + string_decoder@1.3.0: + dependencies: + safe-buffer: 5.2.1 + + strip-json-comments@2.0.1: {} + strip-literal@3.1.0: dependencies: js-tokens: 9.0.1 @@ -1941,6 +4348,29 @@ snapshots: tinyglobby: 0.2.16 ts-interface-checker: 0.1.13 + suffix-thumb@5.0.2: {} + + supports-color@7.2.0: + dependencies: + has-flag: 4.0.0 + + sylvester@0.0.21: {} + + tar-fs@2.1.5: + dependencies: + chownr: 1.1.4 + mkdirp-classic: 0.5.3 + pump: 3.0.4 + tar-stream: 2.2.0 + + tar-stream@2.2.0: + dependencies: + bl: 4.1.0 + end-of-stream: 1.4.5 + fs-constants: 1.0.0 + inherits: 2.0.4 + readable-stream: 3.6.2 + thenify-all@1.6.0: dependencies: thenify: 3.3.1 @@ -1964,11 +4394,23 @@ snapshots: tinyspy@4.0.4: {} + to-regex-range@5.0.1: + dependencies: + is-number: 7.0.0 + + tr46@0.0.3: {} + + tr46@5.1.1: + dependencies: + punycode: 2.3.1 + tree-kill@1.2.2: {} ts-interface-checker@0.1.13: {} - tsup@8.5.1(postcss@8.5.13)(tsx@4.22.4)(typescript@5.9.3)(yaml@2.8.4): + tslib@2.8.1: {} + + tsup@8.5.1(postcss@8.5.13)(tsx@4.23.1)(typescript@5.9.3)(yaml@2.8.4): dependencies: bundle-require: 5.1.0(esbuild@0.27.7) cac: 6.7.14 @@ -1979,7 +4421,7 @@ snapshots: fix-dts-default-cjs-exports: 1.0.1 joycon: 3.1.1 picocolors: 1.1.1 - postcss-load-config: 6.0.1(postcss@8.5.13)(tsx@4.22.4)(yaml@2.8.4) + postcss-load-config: 6.0.1(postcss@8.5.13)(tsx@4.23.1)(yaml@2.8.4) resolve-from: 5.0.0 rollup: 4.60.2 source-map: 0.7.6 @@ -1996,18 +4438,36 @@ snapshots: - tsx - yaml - tsx@4.22.4: + tsx@4.23.1: dependencies: esbuild: 0.28.1 optionalDependencies: fsevents: 2.3.3 + tunnel-agent@0.6.0: + dependencies: + safe-buffer: 5.2.1 + typescript@5.9.3: {} ufo@1.6.4: {} + underscore@1.13.8: {} + + undici-types@5.26.5: {} + undici-types@7.19.2: {} + undici-types@7.24.6: {} + + undici@6.27.0: {} + + util-deprecate@1.0.2: {} + + uuid@11.1.1: {} + + uuid@13.0.2: {} + viem@2.48.8(typescript@5.9.3)(zod@4.4.3): dependencies: '@noble/curves': 1.9.1 @@ -2025,13 +4485,13 @@ snapshots: - utf-8-validate - zod - vite-node@3.2.4(@types/node@25.6.0)(tsx@4.22.4)(yaml@2.8.4): + vite-node@3.2.4(@types/node@25.6.0)(tsx@4.23.1)(yaml@2.8.4): dependencies: cac: 6.7.14 debug: 4.4.3 es-module-lexer: 1.7.0 pathe: 2.0.3 - vite: 7.3.2(@types/node@25.6.0)(tsx@4.22.4)(yaml@2.8.4) + vite: 7.3.2(@types/node@25.6.0)(tsx@4.23.1)(yaml@2.8.4) transitivePeerDependencies: - '@types/node' - jiti @@ -2046,7 +4506,7 @@ snapshots: - tsx - yaml - vite@7.3.2(@types/node@25.6.0)(tsx@4.22.4)(yaml@2.8.4): + vite@7.3.2(@types/node@25.6.0)(tsx@4.23.1)(yaml@2.8.4): dependencies: esbuild: 0.27.7 fdir: 6.5.0(picomatch@4.0.4) @@ -2057,14 +4517,14 @@ snapshots: optionalDependencies: '@types/node': 25.6.0 fsevents: 2.3.3 - tsx: 4.22.4 + tsx: 4.23.1 yaml: 2.8.4 - vitest@3.2.4(@types/node@25.6.0)(tsx@4.22.4)(yaml@2.8.4): + vitest@3.2.4(@types/node@25.6.0)(tsx@4.23.1)(yaml@2.8.4): dependencies: '@types/chai': 5.2.3 '@vitest/expect': 3.2.4 - '@vitest/mocker': 3.2.4(vite@7.3.2(@types/node@25.6.0)(tsx@4.22.4)(yaml@2.8.4)) + '@vitest/mocker': 3.2.4(vite@7.3.2(@types/node@25.6.0)(tsx@4.23.1)(yaml@2.8.4)) '@vitest/pretty-format': 3.2.4 '@vitest/runner': 3.2.4 '@vitest/snapshot': 3.2.4 @@ -2082,8 +4542,8 @@ snapshots: tinyglobby: 0.2.16 tinypool: 1.1.1 tinyrainbow: 2.0.0 - vite: 7.3.2(@types/node@25.6.0)(tsx@4.22.4)(yaml@2.8.4) - vite-node: 3.2.4(@types/node@25.6.0)(tsx@4.22.4)(yaml@2.8.4) + vite: 7.3.2(@types/node@25.6.0)(tsx@4.23.1)(yaml@2.8.4) + vite-node: 3.2.4(@types/node@25.6.0)(tsx@4.23.1)(yaml@2.8.4) why-is-node-running: 2.3.0 optionalDependencies: '@types/node': 25.6.0 @@ -2101,15 +4561,49 @@ snapshots: - tsx - yaml + web-streams-polyfill@3.3.3: {} + + web-streams-polyfill@4.0.0-beta.3: {} + + webidl-conversions@3.0.1: {} + + webidl-conversions@7.0.0: {} + + whatwg-fetch@3.6.20: {} + + whatwg-url@14.2.0: + dependencies: + tr46: 5.1.1 + webidl-conversions: 7.0.0 + + whatwg-url@5.0.0: + dependencies: + tr46: 0.0.3 + webidl-conversions: 3.0.1 + why-is-node-running@2.3.0: dependencies: siginfo: 2.0.0 stackback: 0.0.2 + wordnet-db@3.1.14: {} + + wrappy@1.0.2: {} + ws@8.21.0: {} + wsl-utils@0.1.0: + dependencies: + is-wsl: 3.1.1 + + xtend@4.0.2: {} + yaml@2.8.4: {} - zod@4.4.2: {} + zod-to-json-schema@3.25.2(zod@4.4.3): + dependencies: + zod: 4.4.3 + + zod@3.25.76: {} zod@4.4.3: {} diff --git a/src/benchmarks/index.ts b/src/benchmarks/index.ts index c64832e..9e86aec 100644 --- a/src/benchmarks/index.ts +++ b/src/benchmarks/index.ts @@ -1,14 +1,43 @@ +import { randomUUID } from 'node:crypto' import { join } from 'node:path' +import { canonicalJson } from '@tangle-network/agent-eval' import { type CampaignResult, type CampaignStorage, + type CostLedgerHandle, + createRunCostLedger, type DispatchContext, fsCampaignStorage, type JudgeConfig, type RunCampaignOptions, + resolveRunDir, runCampaign, type Scenario, } from '@tangle-network/agent-eval/campaign' +import { stableId } from '../ids' +import { + appendAttemptJournalEvent, + assertNoInterruptedPaidCalls, + DEFAULT_MEMORY_RECOVERY_RETRIES_PER_ATTEMPT, + hasSettledPaidCall, + readActiveAttemptJournal, + reconcileInterruptedMemoryPaidCalls, + reserveRecoveryAttempts, +} from '../memory/attempt-log' +import { + createMemoryExecutionPool, + memoryRecoveryDelayMs, + releaseMemoryAdapterCreatedAfterAbort, + resolveMemoryCleanupTimeoutMs, + runBoundedMemoryLifecycle, + sleepForMemoryRecovery, +} from '../memory/lifecycle' +import { + type AgentMemoryAcquireRunLease, + type AgentMemoryControllerMode, + acquireAgentMemoryRunLease, + type OwnedAgentMemoryRunLease, +} from '../memory/run-control' import { memoryHitToSourceRecord, memoryWriteResultToSourceRecord } from '../memory/source-record' import type { AgentMemoryAdapter, @@ -24,6 +53,9 @@ import { scoreRetrievalArtifact, } from '../retrieval-eval' +const KNOWLEDGE_BENCHMARK_IMPLEMENTATION_REF = 'agent-knowledge:benchmark-suite:v2' +const MEMORY_ADAPTER_BENCHMARK_IMPLEMENTATION_REF = 'agent-knowledge:memory-adapter-benchmark:v7' + export type KnowledgeBenchmarkTaskKind = | 'retrieval' | 'rag-answer' @@ -170,6 +202,8 @@ export interface KnowledgeBenchmarkEvaluation { score: number passed: boolean dimensions: Record + /** Dimensions for which this case declared an actual target. */ + applicableDimensions?: readonly string[] notes: string raw: Record } @@ -191,6 +225,8 @@ export type KnowledgeBenchmarkResponder export interface RunKnowledgeBenchmarkSuiteOptions { cases: readonly KnowledgeBenchmarkCase[] respond: KnowledgeBenchmarkResponder + /** Versioned identity for the model, prompt, retrieval, and runtime behavior. */ + respondRef?: string runDir: string splits?: readonly KnowledgeBenchmarkSplit[] repo?: string @@ -198,6 +234,9 @@ export interface RunKnowledgeBenchmarkSuiteOptions AgentMemoryAdapter | Promise + /** Local construction is free; call markExternalCall before billable provisioning or reconnects. */ + createAdapter: (input: { + purpose: 'execute' | 'recovery' + signal: AbortSignal + markExternalCall(): void + }) => AgentMemoryAdapter | Promise + /** Conservative charge for one billable adapter provisioning or reconnect call. */ + adapterCreationCostUsd?: number searchLimit?: number costUsdPerCase?: number + /** Conservative extra provider charge for recovering one interrupted case. */ + recoveryCostUsdPerAttempt?: number scope?: AgentMemoryScope } export interface RunMemoryAdapterBenchmarkOptions { cases: readonly KnowledgeMemoryBenchmarkCase[] candidates: readonly MemoryAdapterBenchmarkCandidate[] + /** Retired candidates retained only so interrupted scopes can be cleaned on resume. */ + recoveryCandidates?: readonly MemoryAdapterBenchmarkCandidate[] runDir: string storage?: CampaignStorage repo?: string @@ -261,10 +315,22 @@ export interface RunMemoryAdapterBenchmarkOptions { reps?: number resumable?: boolean costCeiling?: number + /** Shared with nested benchmark suites so the dollar limit applies to the whole comparison. */ + costLedger?: CostLedgerHandle + costPhase?: string maxConcurrency?: number dispatchTimeoutMs?: number + cleanupTimeoutMs?: number + /** Refuse a damaged run with more unfinished attempts than this. Default 1000. */ + maxRecoveryAttempts?: number + /** Bound repeated provider cleanup after process crashes. Default 3 per attempt. */ + maxRecoveryRetriesPerAttempt?: number expectUsage?: 'assert' | 'warn' | 'off' now?: () => Date + /** Required with custom storage when all controllers are confined to one process. */ + controllerMode?: AgentMemoryControllerMode + /** Required for distributed controllers that share custom storage. */ + acquireRunLease?: AgentMemoryAcquireRunLease } export interface MemoryAdapterBenchmarkRankingRow { @@ -285,8 +351,32 @@ export interface MemoryAdapterBenchmarkRankingRow { export interface RunMemoryAdapterBenchmarkResult { rows: readonly MemoryAdapterBenchmarkRankingRow[] + totalCostUsd: number + /** Recovery spend for retired candidates, excluded from ranking rows but included in totalCostUsd. */ + unrankedRecoveryCostUsd: number rankingJsonPath: string rankingMarkdownPath: string + attemptLogPath: string + recoveryLogPath: string +} + +class MemoryAdapterBenchmarkCleanupError extends AggregateError {} + +interface MemoryAdapterBenchmarkAttemptEvent { + schema: 3 + status: 'started' | 'cleaned' + attemptId: string + candidateId: string + candidateRef: string + adapterId: string + caseId: string + cellId: string + scope: AgentMemoryScope + adapterCreationCostUsd: number + costUsdPerCase: number + recoveryCostUsdPerAttempt: number + recordedAt: string + recovery: boolean } export interface KnowledgeRetrievalBenchmarkQuery { @@ -837,14 +927,22 @@ export function buildFirstPartyMemoryLifecycleBenchmarkCases(): KnowledgeMemoryB ] } -export function createMemoryAdapterBenchmarkResponder(options: { +function createMemoryAdapterBenchmarkResponder(options: { adapter: AgentMemoryAdapter candidateId: string + candidateRef: string + storage: CampaignStorage + attemptLogPath: string + lease: OwnedAgentMemoryRunLease + cleanupTimeoutMs: number searchLimit?: number scope?: AgentMemoryScope costUsdPerCase?: number + adapterCreationCostUsd?: number + recoveryCostUsdPerAttempt?: number now?: () => Date }): KnowledgeBenchmarkResponder { + assertScopedMemoryBenchmarkAdapter(options.adapter) return async ({ case: testCase, context: dispatchContext }) => { if (!isKnowledgeMemoryBenchmarkCase(testCase)) { return { answer: '', metadata: { candidateId: options.candidateId, skipped: true } } @@ -854,68 +952,198 @@ export function createMemoryAdapterBenchmarkResponder(options: { throw new Error(`memory adapter costUsdPerCase must be non-negative finite, got ${costUsd}`) } + dispatchContext.signal.throwIfAborted() + await options.lease.assertOwned() + const startedAt = Date.now() + const attemptId = randomUUID() + const scope = benchmarkMemoryScope( + options.candidateId, + testCase, + dispatchContext.cellId, + attemptId, + options.scope, + ) + const attempt: MemoryAdapterBenchmarkAttemptEvent = { + schema: 3, + status: 'started', + attemptId, + candidateId: options.candidateId, + candidateRef: options.candidateRef, + adapterId: options.adapter.id, + caseId: testCase.id, + cellId: dispatchContext.cellId, + scope, + adapterCreationCostUsd: options.adapterCreationCostUsd ?? 0, + costUsdPerCase: costUsd, + recoveryCostUsdPerAttempt: options.recoveryCostUsdPerAttempt ?? 0, + recordedAt: (options.now ?? (() => new Date()))().toISOString(), + recovery: false, + } + appendMemoryBenchmarkAttemptEvent(options.storage, options.attemptLogPath, attempt) + + let externalCallAttempted = false + const appendCleanedAttempt = (priorError?: unknown): void => { + try { + appendMemoryBenchmarkAttemptEvent(options.storage, options.attemptLogPath, { + ...attempt, + status: 'cleaned', + recordedAt: (options.now ?? (() => new Date()))().toISOString(), + }) + } catch (error) { + throw new MemoryAdapterBenchmarkCleanupError( + [...(priorError ? [priorError] : []), error], + `${options.candidateId}: memory benchmark cleanup could not be recorded`, + ) + } + } const execute = async (): Promise => { - const startedAt = Date.now() - const scope = benchmarkMemoryScope(options.candidateId, testCase, options.scope) - for (const event of testCase.events) { - await options.adapter.write({ - id: event.id, - kind: 'message', - text: event.text, - role: event.actorId === 'user' ? 'user' : 'assistant', - title: `${testCase.id}:${event.id}`, + dispatchContext.signal.throwIfAborted() + await options.lease.assertOwned() + let artifact: KnowledgeBenchmarkArtifact | undefined + let primaryError: unknown + try { + for (const event of testCase.events) { + dispatchContext.signal.throwIfAborted() + await options.lease.assertOwned() + externalCallAttempted = true + await options.adapter.write({ + id: event.id, + kind: 'message', + text: event.text, + role: event.actorId === 'user' ? 'user' : 'assistant', + title: `${testCase.id}:${event.id}`, + scope, + metadata: compactObject({ + benchmarkCaseId: testCase.id, + benchmarkCellId: dispatchContext.cellId, + benchmarkAttemptId: attemptId, + eventId: event.id, + actorId: event.actorId, + sessionId: event.sessionId, + timestamp: event.timestamp, + ...event.metadata, + }) as Record, + }) + dispatchContext.signal.throwIfAborted() + await options.lease.assertOwned() + } + externalCallAttempted = true + await options.adapter.flush?.() + dispatchContext.signal.throwIfAborted() + await options.lease.assertOwned() + + externalCallAttempted = true + const adapterContext = await options.adapter.getContext(testCase.prompt, { scope, - metadata: compactObject({ + limit: options.searchLimit ?? 1, + metadata: { benchmarkCaseId: testCase.id, - eventId: event.id, - actorId: event.actorId, - sessionId: event.sessionId, - timestamp: event.timestamp, - ...event.metadata, - }) as Record, + benchmarkCellId: dispatchContext.cellId, + benchmarkAttemptId: attemptId, + candidateId: options.candidateId, + }, }) + dispatchContext.signal.throwIfAborted() + await options.lease.assertOwned() + const hits = adapterContext.hits + artifact = { + answer: adapterContext.text, + rememberedFacts: hits.map((hit) => hit.text), + citedEventIds: unique(hits.map(memoryEventId).filter((id): id is string => Boolean(id))), + usedMemoryIds: hits.map((hit) => hit.id), + actorIds: unique(hits.map(memoryActorId).filter((id): id is string => Boolean(id))), + costUsd, + durationMs: Math.max(0, Date.now() - startedAt), + metadata: { + candidateId: options.candidateId, + adapterId: options.adapter.id, + hitCount: hits.length, + }, + } + } catch (error) { + primaryError = error } - const adapterContext = await options.adapter.getContext(testCase.prompt, { - scope, - limit: options.searchLimit ?? 1, - metadata: { - benchmarkCaseId: testCase.id, - candidateId: options.candidateId, - }, - }) - const hits = adapterContext.hits - return { - answer: adapterContext.text, - rememberedFacts: hits.map((hit) => hit.text), - citedEventIds: unique(hits.map(memoryEventId).filter((id): id is string => Boolean(id))), - usedMemoryIds: hits.map((hit) => hit.id), - actorIds: unique(hits.map(memoryActorId).filter((id): id is string => Boolean(id))), - costUsd, - durationMs: Math.max(0, Date.now() - startedAt), - metadata: { - candidateId: options.candidateId, - adapterId: options.adapter.id, - hitCount: hits.length, - }, + const cleanupErrors: unknown[] = [] + let cleanupOwned = true + try { + await options.lease.assertOwned() + } catch (error) { + cleanupOwned = false + cleanupErrors.push(error) } + if (cleanupOwned) { + try { + await runBoundedMemoryLifecycle({ + operation: `${options.candidateId}: benchmark attempt flush`, + timeoutMs: options.cleanupTimeoutMs, + resource: options.adapter, + run: () => { + externalCallAttempted = true + return options.adapter.flush?.() + }, + }) + } catch (error) { + cleanupErrors.push(error) + } + try { + await runBoundedMemoryLifecycle({ + operation: `${options.candidateId}: benchmark attempt cleanup`, + timeoutMs: options.cleanupTimeoutMs, + resource: options.adapter, + run: () => { + externalCallAttempted = true + return options.adapter.clear!(scope) + }, + }) + } catch (error) { + cleanupErrors.push(error) + } + } + if (cleanupOwned && cleanupErrors.length === 0) appendCleanedAttempt(primaryError) + if (cleanupErrors.length > 0) { + const errors = [...(primaryError ? [primaryError] : []), ...cleanupErrors] + throw new MemoryAdapterBenchmarkCleanupError( + errors, + `${options.candidateId}: memory benchmark attempt cleanup failed`, + ) + } + if (primaryError) throw primaryError + if (!artifact) + throw new Error(`${options.candidateId}: memory benchmark produced no artifact`) + return artifact } - if (costUsd === 0) return execute() + if (costUsd === 0) { + let artifact: KnowledgeBenchmarkArtifact | undefined + let error: unknown + try { + artifact = await execute() + } catch (caught) { + error = caught + } + if (error) throw error + if (!artifact) + throw new Error(`${options.candidateId}: memory benchmark produced no artifact`) + return artifact + } const receipt = { model: options.adapter.id, inputTokens: 0, outputTokens: 0, - usageUnknown: true, actualCostUsd: costUsd, } as const const paid = await dispatchContext.cost.runPaidCall({ + callId: memoryBenchmarkCostCallId(attempt, 'execute', 0), actor: `agent-knowledge:memory-adapter:${options.adapter.id}`, model: options.adapter.id, maximumCharge: { externallyEnforcedMaximumUsd: costUsd }, execute, receipt: () => receipt, - receiptFromError: () => receipt, + receiptFromError: () => ({ + ...receipt, + actualCostUsd: externalCallAttempted ? costUsd : 0, + }), }) if (!paid.succeeded) throw paid.error return paid.value @@ -925,67 +1153,961 @@ export function createMemoryAdapterBenchmarkResponder(options: { export async function runMemoryAdapterBenchmark( options: RunMemoryAdapterBenchmarkOptions, ): Promise { + if (options.candidates.length === 0) + throw new Error('memory adapter benchmark requires candidates') + const allCandidates = [...options.candidates, ...(options.recoveryCandidates ?? [])] + assertUniqueNonEmptyStrings( + allCandidates.map((candidate) => candidate.id), + 'memory adapter candidate id', + ) + for (const candidate of allCandidates) { + assertNonEmptyBenchmarkString(candidate.ref, `memory adapter candidate ${candidate.id} ref`) + if (candidate.adapterId !== undefined) { + assertNonEmptyBenchmarkString( + candidate.adapterId, + `memory adapter candidate ${candidate.id} adapterId`, + ) + } + if (!/^[A-Za-z0-9][A-Za-z0-9._-]*$/.test(candidate.id)) { + throw new Error( + `memory adapter candidate id '${candidate.id}' must be a safe directory segment`, + ) + } + if ( + candidate.adapterCreationCostUsd !== undefined && + (!Number.isFinite(candidate.adapterCreationCostUsd) || candidate.adapterCreationCostUsd < 0) + ) { + throw new Error( + `${candidate.id}: adapterCreationCostUsd must be a non-negative finite number`, + ) + } + if ( + candidate.costUsdPerCase !== undefined && + (!Number.isFinite(candidate.costUsdPerCase) || candidate.costUsdPerCase < 0) + ) { + throw new Error(`${candidate.id}: costUsdPerCase must be a non-negative finite number`) + } + if ( + candidate.recoveryCostUsdPerAttempt !== undefined && + (!Number.isFinite(candidate.recoveryCostUsdPerAttempt) || + candidate.recoveryCostUsdPerAttempt < 0) + ) { + throw new Error( + `${candidate.id}: recoveryCostUsdPerAttempt must be a non-negative finite number`, + ) + } + } const storage = options.storage ?? fsCampaignStorage() + if (!storage.append) { + throw new Error('memory adapter benchmark requires CampaignStorage.append') + } + const runDir = resolveRunDir(options.runDir, options.repo) + const cleanupTimeoutMs = resolveMemoryCleanupTimeoutMs( + options.cleanupTimeoutMs, + 'memory adapter benchmark', + ) + const maxRecoveryAttempts = options.maxRecoveryAttempts ?? 1_000 + if (!Number.isSafeInteger(maxRecoveryAttempts) || maxRecoveryAttempts <= 0) { + throw new Error('memory adapter benchmark maxRecoveryAttempts must be a positive safe integer') + } + const maxRecoveryRetriesPerAttempt = + options.maxRecoveryRetriesPerAttempt ?? DEFAULT_MEMORY_RECOVERY_RETRIES_PER_ATTEMPT + if (!Number.isSafeInteger(maxRecoveryRetriesPerAttempt) || maxRecoveryRetriesPerAttempt <= 0) { + throw new Error( + 'memory adapter benchmark maxRecoveryRetriesPerAttempt must be a positive safe integer', + ) + } + storage.ensureDir(runDir) + const lease = await acquireAgentMemoryRunLease({ + experimentId: `memory-adapter-benchmark:${runDir}`, + runDir, + storage, + customStorage: options.storage !== undefined, + lockFileName: 'memory-adapter-benchmark.lock', + label: 'memory adapter benchmark', + controllerMode: options.controllerMode, + acquireRunLease: options.acquireRunLease, + }) + let result: RunMemoryAdapterBenchmarkResult | undefined + let primaryError: unknown + try { + result = await runOwnedMemoryAdapterBenchmark( + options, + storage, + runDir, + lease, + cleanupTimeoutMs, + maxRecoveryAttempts, + maxRecoveryRetriesPerAttempt, + ) + } catch (error) { + primaryError = error + } + let releaseError: unknown + try { + await lease.release() + } catch (error) { + releaseError = error + } + if (primaryError && releaseError) { + throw new AggregateError( + [primaryError, releaseError], + 'memory adapter benchmark failed and its controller lease could not be released', + ) + } + if (primaryError) throw primaryError + if (releaseError) throw releaseError + if (!result) throw new Error('memory adapter benchmark produced no result') + return result +} + +async function runOwnedMemoryAdapterBenchmark( + options: RunMemoryAdapterBenchmarkOptions, + storage: CampaignStorage, + runDir: string, + lease: OwnedAgentMemoryRunLease, + cleanupTimeoutMs: number, + maxRecoveryAttempts: number, + maxRecoveryRetriesPerAttempt: number, +): Promise { + const maxConcurrency = options.maxConcurrency ?? 2 + if (!Number.isSafeInteger(maxConcurrency) || maxConcurrency <= 0) { + throw new Error('memory adapter benchmark maxConcurrency must be a positive safe integer') + } + const costCeiling = options.costCeiling ?? options.costLedger?.costCeilingUsd ?? 0 + const costLedger = + options.costLedger ?? + createRunCostLedger({ + storage, + runDir, + costCeilingUsd: costCeiling, + }) + if (costLedger.costCeilingUsd !== costCeiling) { + throw new Error( + 'memory adapter benchmark costCeiling must match the shared cost ledger ceiling', + ) + } + const attemptLogPath = join(runDir, 'memory-adapter-attempts.jsonl') + const recoveryLogPath = join(runDir, 'memory-adapter-recovery-attempts.jsonl') + await recoverMemoryAdapterBenchmarkAttempts({ + candidates: [...options.candidates, ...(options.recoveryCandidates ?? [])], + storage, + attemptLogPath, + lease, + cleanupTimeoutMs, + maxConcurrency, + now: options.now, + runDir, + costLedger, + costPhase: options.costPhase ?? 'memory.adapter-benchmark', + maxRecoveryAttempts, + recoveryLogPath, + maxRecoveryRetriesPerAttempt, + }) + await lease.assertOwned() const rows: MemoryAdapterBenchmarkRankingRow[] = [] for (const candidate of options.candidates) { - const adapter = await candidate.createAdapter() - const run = await runKnowledgeBenchmarkSuite({ - cases: options.cases, - respond: createMemoryAdapterBenchmarkResponder({ - adapter, - candidateId: candidate.id, - searchLimit: candidate.searchLimit, - scope: candidate.scope, - costUsdPerCase: candidate.costUsdPerCase, + await lease.assertOwned() + const expectedAdapterId = memoryAdapterBenchmarkExpectedId(candidate) + let adapter: AgentMemoryAdapter | undefined + let adapterPromise: Promise | undefined + let adapterCreationError: unknown + const getAdapter = (): Promise => { + if (!adapterPromise) { + const abortController = new AbortController() + const creation = createMemoryAdapterBenchmarkAdapter({ + candidate, + purpose: 'execute', + signal: abortController.signal, + costLedger, + runDir, + costPhase: options.costPhase ?? 'memory.adapter-benchmark', + }) + releaseMemoryAdapterCreatedAfterAbort({ creation, signal: abortController.signal }) + adapterPromise = runBoundedMemoryLifecycle({ + operation: `${candidate.id}: benchmark execute adapter creation`, + timeoutMs: Math.min(cleanupTimeoutMs, options.dispatchTimeoutMs ?? cleanupTimeoutMs), + abortController, + run: () => creation, + }) + .then((created) => { + adapter = created + if (created.id !== expectedAdapterId) { + throw new Error( + `${candidate.id}: createAdapter returned id '${created.id}', expected '${expectedAdapterId}'`, + ) + } + assertScopedMemoryBenchmarkAdapter(created) + return created + }) + .catch((error) => { + adapterCreationError = error + throw error + }) + } + return adapterPromise + } + const dispatchedExecutions: Promise[] = [] + let run: RunKnowledgeBenchmarkSuiteResult | undefined + let primaryError: unknown + try { + await lease.assertOwned() + let respond: KnowledgeBenchmarkResponder | undefined + run = await runKnowledgeBenchmarkSuite({ + cases: options.cases, + respond(input) { + const operation = getAdapter().then((activeAdapter) => { + respond ??= createMemoryAdapterBenchmarkResponder({ + adapter: activeAdapter, + candidateId: candidate.id, + candidateRef: candidate.ref, + storage, + attemptLogPath, + lease, + cleanupTimeoutMs, + searchLimit: candidate.searchLimit, + scope: candidate.scope, + adapterCreationCostUsd: candidate.adapterCreationCostUsd, + costUsdPerCase: candidate.costUsdPerCase, + recoveryCostUsdPerAttempt: candidate.recoveryCostUsdPerAttempt, + now: options.now, + }) + return respond(input) + }) + dispatchedExecutions.push(operation) + return operation + }, + respondRef: stableId( + 'memory_adapter_benchmark', + canonicalJson({ + implementationRef: MEMORY_ADAPTER_BENCHMARK_IMPLEMENTATION_REF, + candidateRef: candidate.ref, + adapterId: expectedAdapterId, + searchLimit: candidate.searchLimit ?? null, + adapterCreationCostUsd: candidate.adapterCreationCostUsd ?? 0, + costUsdPerCase: candidate.costUsdPerCase ?? 0, + recoveryCostUsdPerAttempt: candidate.recoveryCostUsdPerAttempt ?? 0, + scope: candidate.scope ?? null, + }), + ), + runDir: join(runDir, candidate.id), + storage, + seed: options.seed, + reps: options.reps, + resumable: options.resumable, + costCeiling, + costLedger, + costPhase: `${options.costPhase ?? 'memory.adapter-benchmark'}.${candidate.id}`, + maxConcurrency: options.maxConcurrency, + dispatchTimeoutMs: options.dispatchTimeoutMs, + expectUsage: options.expectUsage ?? 'off', now: options.now, - }), - runDir: join(options.runDir, candidate.id), - storage, - repo: options.repo, - seed: options.seed, - reps: options.reps, - resumable: options.resumable, - costCeiling: options.costCeiling, - maxConcurrency: options.maxConcurrency, - dispatchTimeoutMs: options.dispatchTimeoutMs, - expectUsage: options.expectUsage ?? 'off', - now: options.now, - }) - rows.push({ - rank: 0, - candidateId: candidate.id, - label: candidate.label ?? candidate.id, - adapterId: adapter.id, - scoreMean: run.report.score.mean, - passRate: run.report.dimensions.passed?.mean ?? 0, - totalCases: run.report.totalCases, - totalCells: run.report.totalCells, - cellsFailed: run.report.cellsFailed, - totalCostUsd: run.report.totalCostUsd, - reportJsonPath: run.reportJsonPath, - reportMarkdownPath: run.reportMarkdownPath, - report: run.report, - }) - await adapter.flush?.() + }) + } catch (error) { + primaryError = error + } + + const settledExecutions = await Promise.allSettled(dispatchedExecutions) + const dispatchCleanupErrors = settledExecutions.flatMap((settled) => + settled.status === 'rejected' && settled.reason instanceof MemoryAdapterBenchmarkCleanupError + ? [settled.reason] + : [], + ) + if (!primaryError && adapterCreationError) primaryError = adapterCreationError + if (run) { + rows.push({ + rank: 0, + candidateId: candidate.id, + label: candidate.label ?? candidate.id, + adapterId: expectedAdapterId, + scoreMean: run.report.score.mean, + passRate: run.report.dimensions.passed?.mean ?? 0, + totalCases: run.report.totalCases, + totalCells: run.report.totalCells, + cellsFailed: run.report.cellsFailed, + totalCostUsd: run.report.totalCostUsd, + reportJsonPath: run.reportJsonPath, + reportMarkdownPath: run.reportMarkdownPath, + report: run.report, + }) + } + const cleanupErrors: unknown[] = [] + let cleanupOwned = true + try { + await lease.assertOwned() + } catch (error) { + cleanupOwned = false + cleanupErrors.push(error) + } + if (cleanupOwned && adapter && !adapterCreationError) { + const activeAdapter = adapter + try { + await runBoundedMemoryLifecycle({ + operation: `${candidate.id}: benchmark adapter flush`, + timeoutMs: cleanupTimeoutMs, + resource: activeAdapter, + run: () => activeAdapter.flush?.(), + }) + } catch (error) { + cleanupErrors.push(error) + } + } + if (adapter) { + try { + await runBoundedMemoryLifecycle({ + operation: `${candidate.id}: benchmark adapter close`, + timeoutMs: cleanupTimeoutMs, + resource: adapter, + run: () => adapter!.close?.(), + }) + } catch (error) { + cleanupErrors.push(error) + } + } + if (primaryError || dispatchCleanupErrors.length > 0 || cleanupErrors.length > 0) { + const errors = [ + ...(primaryError ? [primaryError] : []), + ...dispatchCleanupErrors, + ...cleanupErrors, + ] + if (errors.length === 1) throw errors[0] + throw new AggregateError(errors, `${candidate.id}: memory adapter benchmark cleanup failed`) + } + await lease.assertOwned() } + const costByCandidate = memoryAdapterBenchmarkCostByCandidate(costLedger, runDir, [ + ...options.candidates, + ...(options.recoveryCandidates ?? []), + ]) const ranked = rows - .sort((a, b) => b.scoreMean - a.scoreMean || a.totalCostUsd - b.totalCostUsd) + .map((row) => { + const totalCostUsd = normalizeUsd(costByCandidate.get(row.candidateId) ?? 0) + return { + ...row, + totalCostUsd, + } + }) + .sort( + (a, b) => + Number(a.cellsFailed > 0) - Number(b.cellsFailed > 0) || + b.scoreMean - a.scoreMean || + b.passRate - a.passRate || + a.totalCostUsd - b.totalCostUsd || + a.candidateId.localeCompare(b.candidateId), + ) .map((row, index) => ({ ...row, rank: index + 1 })) - const rankingJsonPath = join(options.runDir, 'memory-adapter-ranking.json') - const rankingMarkdownPath = join(options.runDir, 'memory-adapter-ranking.md') - storage.write(rankingJsonPath, `${JSON.stringify({ rows: ranked }, null, 2)}\n`) - storage.write(rankingMarkdownPath, renderMemoryAdapterRankingMarkdown(ranked)) + const rankingJsonPath = join(runDir, 'memory-adapter-ranking.json') + const rankingMarkdownPath = join(runDir, 'memory-adapter-ranking.md') + const unrankedRecoveryCostUsd = normalizeUsd( + (options.recoveryCandidates ?? []).reduce( + (sum, candidate) => sum + (costByCandidate.get(candidate.id) ?? 0), + 0, + ), + ) + const totalCostUsd = normalizeUsd( + [...costByCandidate.values()].reduce((sum, cost) => sum + cost, 0), + ) + storage.write( + rankingJsonPath, + `${JSON.stringify({ totalCostUsd, unrankedRecoveryCostUsd, rows: ranked }, null, 2)}\n`, + ) + storage.write( + rankingMarkdownPath, + renderMemoryAdapterRankingMarkdown(ranked, totalCostUsd, unrankedRecoveryCostUsd), + ) return { rows: ranked, + totalCostUsd, + unrankedRecoveryCostUsd, rankingJsonPath, rankingMarkdownPath, + attemptLogPath, + recoveryLogPath, + } +} + +async function createMemoryAdapterBenchmarkAdapter(input: { + candidate: MemoryAdapterBenchmarkCandidate + purpose: 'execute' | 'recovery' + signal: AbortSignal + costLedger: CostLedgerHandle + runDir: string + costPhase: string +}): Promise { + const { candidate, purpose, signal, costLedger, runDir, costPhase } = input + const costUsd = candidate.adapterCreationCostUsd ?? 0 + let externalCallAttempted = false + const create = async (): Promise => { + const adapter = await candidate.createAdapter({ + purpose, + signal, + markExternalCall: () => { + externalCallAttempted = true + }, + }) + if (!adapter || typeof adapter !== 'object') { + throw new Error(`${candidate.id}: createAdapter returned no ${purpose} adapter`) + } + return adapter + } + if (costUsd === 0) return create() + + const tags = memoryAdapterCreationCostTags(runDir, candidate.id, purpose) + const generation = costLedger.list({ tags }).length + const receipt = { + model: candidate.id, + inputTokens: 0, + outputTokens: 0, + actualCostUsd: costUsd, + } as const + const paid = await costLedger.runPaidCall({ + callId: memoryAdapterCreationCostCallId(candidate, purpose, generation), + channel: 'driver', + phase: `${costPhase}.${candidate.id}.adapter-${purpose}`, + actor: `agent-knowledge:memory-adapter:${candidate.id}`, + model: candidate.id, + tags, + maximumCharge: { externallyEnforcedMaximumUsd: costUsd }, + execute: create, + receipt: () => ({ + ...receipt, + actualCostUsd: externalCallAttempted ? costUsd : 0, + }), + receiptFromError: () => ({ + ...receipt, + actualCostUsd: externalCallAttempted ? costUsd : 0, + }), + }) + if (!paid.succeeded) throw paid.error + return paid.value +} + +function memoryAdapterCreationCostTags( + runDir: string, + candidateId: string, + purpose: 'execute' | 'recovery', +): Record { + return { + runDir: join(runDir, candidateId), + candidateId, + memoryAdapterCreation: purpose, + } +} + +function memoryAdapterCreationCostCallId( + candidate: MemoryAdapterBenchmarkCandidate, + purpose: 'execute' | 'recovery', + generation: number, +): string { + return stableId( + 'memory_adapter_creation_cost_call', + canonicalJson({ + purpose, + generation, + candidateId: candidate.id, + candidateRef: candidate.ref, + adapterCreationCostUsd: candidate.adapterCreationCostUsd ?? 0, + }), + ) +} + +function memoryAdapterBenchmarkExpectedId(candidate: MemoryAdapterBenchmarkCandidate): string { + return candidate.adapterId ?? candidate.id +} + +function memoryAdapterBenchmarkCostByCandidate( + costLedger: CostLedgerHandle, + runDir: string, + candidates: readonly MemoryAdapterBenchmarkCandidate[], +): ReadonlyMap { + const candidateByRunDir = new Map( + candidates.map((candidate) => [join(runDir, candidate.id), candidate.id]), + ) + const totals = new Map() + for (const receipt of costLedger.list()) { + const candidateId = receipt.tags?.runDir + ? candidateByRunDir.get(receipt.tags.runDir) + : undefined + if (!candidateId) continue + totals.set(candidateId, (totals.get(candidateId) ?? 0) + receipt.costUsd) + } + return totals +} + +async function recoverMemoryAdapterBenchmarkAttempts(input: { + candidates: readonly MemoryAdapterBenchmarkCandidate[] + storage: CampaignStorage + attemptLogPath: string + lease: OwnedAgentMemoryRunLease + cleanupTimeoutMs: number + maxConcurrency: number + now?: () => Date + runDir: string + costLedger: CostLedgerHandle + costPhase: string + maxRecoveryAttempts: number + recoveryLogPath: string + maxRecoveryRetriesPerAttempt: number +}): Promise { + let attempts = readActiveMemoryBenchmarkAttempts(input.storage, input.attemptLogPath) + if (attempts.length > input.maxRecoveryAttempts) { + throw new Error( + `memory adapter benchmark has ${attempts.length} unfinished attempts; maxRecoveryAttempts is ${input.maxRecoveryAttempts}`, + ) + } + const candidateById = new Map(input.candidates.map((candidate) => [candidate.id, candidate])) + for (const attempt of attempts) { + const candidate = candidateById.get(attempt.candidateId) + if (!candidate) { + throw new Error( + `cannot recover memory benchmark attempts: candidate '${attempt.candidateId}' is missing; pass it in recoveryCandidates`, + ) + } + assertMemoryBenchmarkAttemptCandidateMatches(attempt, candidate) + } + + reconcileInterruptedMemoryPaidCalls(input.costLedger) + assertNoInterruptedPaidCalls(input.costLedger, 'memory adapter benchmark recovery') + + for (const attempt of attempts) { + const candidate = candidateById.get(attempt.candidateId)! + const costUsd = candidate.costUsdPerCase ?? 0 + if ( + costUsd > 0 && + !hasSettledPaidCall(input.costLedger, memoryBenchmarkCostCallId(attempt, 'execute', 0)) + ) { + appendMemoryBenchmarkAttemptEvent(input.storage, input.attemptLogPath, { + ...attempt, + status: 'cleaned', + recovery: true, + recordedAt: (input.now ?? (() => new Date()))().toISOString(), + }) + } + } + attempts = readActiveMemoryBenchmarkAttempts(input.storage, input.attemptLogPath) + const recoveryGenerations = reserveRecoveryAttempts({ + storage: input.storage, + path: input.recoveryLogPath, + attemptIds: attempts.map((attempt) => attempt.attemptId), + maxRetriesPerAttempt: input.maxRecoveryRetriesPerAttempt, + label: 'memory benchmark recovery attempt log', + now: input.now, + }) + const grouped = groupMemoryBenchmarkAttempts(attempts) + const pool = createMemoryExecutionPool(input.maxConcurrency) + const settled = await Promise.allSettled( + [...grouped] + .sort(([left], [right]) => left.localeCompare(right)) + .map(([candidateId, candidateAttempts]) => + pool.run(async () => { + await input.lease.assertOwned() + const candidate = candidateById.get(candidateId) + if (!candidate) { + throw new Error( + `cannot recover memory benchmark attempts: candidate '${candidateId}' is missing; pass it in recoveryCandidates`, + ) + } + for (const attempt of candidateAttempts) { + assertMemoryBenchmarkAttemptCandidateMatches(attempt, candidate) + if (recoveryGenerations.get(attempt.attemptId) === undefined) { + throw new Error( + `missing recovery generation for memory benchmark attempt '${attempt.attemptId}'`, + ) + } + } + + let recoveryAttemptsStarted = 0 + let adapterCreationExternalCallAttempted = false + const cleared: MemoryAdapterBenchmarkAttemptEvent[] = [] + const recover = async (): Promise => { + let adapter: AgentMemoryAdapter | undefined + let primaryError: unknown + try { + const abortController = new AbortController() + const creation = Promise.resolve().then(() => + candidate.createAdapter({ + purpose: 'recovery', + signal: abortController.signal, + markExternalCall: () => { + adapterCreationExternalCallAttempted = true + }, + }), + ) + releaseMemoryAdapterCreatedAfterAbort({ + creation, + signal: abortController.signal, + }) + adapter = await runBoundedMemoryLifecycle({ + operation: `${candidate.id}: benchmark recovery adapter creation`, + timeoutMs: input.cleanupTimeoutMs, + abortController, + run: () => creation, + }) + assertScopedMemoryBenchmarkAdapter(adapter) + for (const attempt of candidateAttempts) { + if (adapter.id !== attempt.adapterId) { + throw new Error( + `cannot recover memory benchmark attempt '${attempt.attemptId}': adapter changed from '${attempt.adapterId}' to '${adapter.id}'`, + ) + } + } + await sleepForMemoryRecovery( + memoryRecoveryDelayMs(adapter), + () => input.lease.assertOwned(), + input.cleanupTimeoutMs, + `${candidate.id}: benchmark recovery visibility wait`, + ) + for (const attempt of candidateAttempts.sort((left, right) => + left.attemptId.localeCompare(right.attemptId), + )) { + await input.lease.assertOwned() + try { + recoveryAttemptsStarted += 1 + await runBoundedMemoryLifecycle({ + operation: `${candidate.id}: abandoned benchmark attempt cleanup`, + timeoutMs: input.cleanupTimeoutMs, + resource: adapter, + run: () => adapter!.clear!(attempt.scope), + }) + await input.lease.assertOwned() + cleared.push(attempt) + } catch (error) { + primaryError = primaryError + ? new AggregateError( + [primaryError, error], + `${candidate.id}: multiple benchmark attempts failed recovery`, + ) + : error + } + } + } catch (error) { + primaryError = primaryError + ? new AggregateError( + [primaryError, error], + `${candidate.id}: benchmark recovery failed in multiple operations`, + ) + : error + } + + let closeError: unknown + if (adapter) { + try { + await runBoundedMemoryLifecycle({ + operation: `${candidate.id}: benchmark recovery adapter close`, + timeoutMs: input.cleanupTimeoutMs, + resource: adapter, + run: () => adapter!.close?.(), + }) + } catch (error) { + closeError = error + } + } + let journalError: unknown + if (!closeError) { + try { + for (const attempt of cleared) { + await input.lease.assertOwned() + appendMemoryBenchmarkAttemptEvent(input.storage, input.attemptLogPath, { + ...attempt, + status: 'cleaned', + recovery: true, + recordedAt: (input.now ?? (() => new Date()))().toISOString(), + }) + } + } catch (error) { + journalError = error + } + } + const failures = [primaryError, closeError, journalError].filter( + (error) => error !== undefined, + ) + if (failures.length > 1) { + throw new AggregateError( + failures, + `${candidate.id}: benchmark attempt recovery, adapter close, or cleanup journal failed`, + ) + } + if (primaryError) throw primaryError + if (closeError) throw closeError + if (journalError) throw journalError + } + + const recoveryCostUsd = candidate.recoveryCostUsdPerAttempt ?? 0 + const adapterCreationCostUsd = candidate.adapterCreationCostUsd ?? 0 + const maximumCostUsd = adapterCreationCostUsd + recoveryCostUsd * candidateAttempts.length + const actualCostUsd = (): number => + (adapterCreationExternalCallAttempted ? adapterCreationCostUsd : 0) + + recoveryCostUsd * recoveryAttemptsStarted + let recoveryError: unknown + if (maximumCostUsd === 0) { + try { + await recover() + } catch (error) { + recoveryError = error + } + } else { + const receipt = { + model: candidate.id, + inputTokens: 0, + outputTokens: 0, + } as const + const tags = memoryBenchmarkRecoveryCostTags(input.runDir, candidate.id) + const paid = await input.costLedger.runPaidCall({ + callId: memoryBenchmarkRecoveryCostCallId( + candidate, + candidateAttempts, + recoveryGenerations, + ), + channel: 'driver', + phase: `${input.costPhase}.${candidate.id}.recovery`, + actor: `agent-knowledge:memory-adapter-recovery:${candidate.id}`, + model: candidate.id, + tags, + maximumCharge: { externallyEnforcedMaximumUsd: maximumCostUsd }, + execute: recover, + receipt: () => ({ ...receipt, actualCostUsd: actualCostUsd() }), + receiptFromError: () => ({ + ...receipt, + actualCostUsd: actualCostUsd(), + }), + }) + if (!paid.succeeded) recoveryError = paid.error + } + if (recoveryError) throw recoveryError + }), + ), + ) + const failures = settled.flatMap((result) => + result.status === 'rejected' ? [result.reason] : [], + ) + if (failures.length === 1) throw failures[0] + if (failures.length > 1) { + throw new AggregateError(failures, 'multiple memory benchmark candidates failed recovery') + } +} + +function groupMemoryBenchmarkAttempts( + attempts: readonly MemoryAdapterBenchmarkAttemptEvent[], +): Map { + const grouped = new Map() + for (const attempt of attempts) { + const group = grouped.get(attempt.candidateId) ?? [] + group.push(attempt) + grouped.set(attempt.candidateId, group) + } + return grouped +} + +function memoryBenchmarkCostCallId( + attempt: MemoryAdapterBenchmarkAttemptEvent, + purpose: 'execute' | 'recovery', + generation: number, +): string { + return stableId( + 'memory_benchmark_cost_call', + canonicalJson({ + purpose, + generation, + attemptId: attempt.attemptId, + candidateId: attempt.candidateId, + candidateRef: attempt.candidateRef, + adapterId: attempt.adapterId, + adapterCreationCostUsd: attempt.adapterCreationCostUsd, + costUsdPerCase: attempt.costUsdPerCase, + recoveryCostUsdPerAttempt: attempt.recoveryCostUsdPerAttempt, + caseId: attempt.caseId, + cellId: attempt.cellId, + }), + ) +} + +function memoryBenchmarkRecoveryCostCallId( + candidate: MemoryAdapterBenchmarkCandidate, + attempts: readonly MemoryAdapterBenchmarkAttemptEvent[], + generations: ReadonlyMap, +): string { + return stableId( + 'memory_benchmark_recovery_cost_call', + canonicalJson({ + candidateId: candidate.id, + candidateRef: candidate.ref, + adapterCreationCostUsd: candidate.adapterCreationCostUsd ?? 0, + recoveryCostUsdPerAttempt: candidate.recoveryCostUsdPerAttempt ?? 0, + attempts: attempts + .map((attempt) => ({ + attemptId: attempt.attemptId, + generation: generations.get(attempt.attemptId), + })) + .sort((left, right) => left.attemptId.localeCompare(right.attemptId)), + }), + ) +} + +function memoryBenchmarkRecoveryCostTags( + runDir: string, + candidateId: string, +): Record { + return { + runDir: join(runDir, candidateId), + candidateId, + memoryRecovery: 'benchmark', + } +} + +function appendMemoryBenchmarkAttemptEvent( + storage: CampaignStorage, + path: string, + event: MemoryAdapterBenchmarkAttemptEvent, +): void { + appendAttemptJournalEvent({ + storage, + path, + event, + label: 'memory benchmark attempt log', + }) +} + +function readActiveMemoryBenchmarkAttempts( + storage: CampaignStorage, + path: string, +): MemoryAdapterBenchmarkAttemptEvent[] { + return readActiveAttemptJournal({ + storage, + path, + label: 'memory benchmark attempt log', + parse: parseMemoryBenchmarkAttemptEvent, + id: (event) => event.attemptId, + sameAttempt: sameMemoryBenchmarkAttempt, + }) +} + +function parseMemoryBenchmarkAttemptEvent( + value: unknown, + path: string, + line: number, +): MemoryAdapterBenchmarkAttemptEvent { + const event = value as Partial | null + const valid = + typeof event === 'object' && + event !== null && + event.schema === 3 && + (event.status === 'started' || event.status === 'cleaned') && + isNonEmptyString(event.attemptId) && + isNonEmptyString(event.candidateId) && + isNonEmptyString(event.candidateRef) && + isNonEmptyString(event.adapterId) && + isNonEmptyString(event.caseId) && + isNonEmptyString(event.cellId) && + isAgentMemoryScope(event.scope) && + typeof event.adapterCreationCostUsd === 'number' && + Number.isFinite(event.adapterCreationCostUsd) && + event.adapterCreationCostUsd >= 0 && + typeof event.costUsdPerCase === 'number' && + Number.isFinite(event.costUsdPerCase) && + event.costUsdPerCase >= 0 && + typeof event.recoveryCostUsdPerAttempt === 'number' && + Number.isFinite(event.recoveryCostUsdPerAttempt) && + event.recoveryCostUsdPerAttempt >= 0 && + typeof event.recordedAt === 'string' && + !Number.isNaN(Date.parse(event.recordedAt)) && + typeof event.recovery === 'boolean' + if (!valid) { + throw new Error(`invalid memory benchmark attempt event in '${path}' line ${line}`) } + return value as MemoryAdapterBenchmarkAttemptEvent +} + +function sameMemoryBenchmarkAttempt( + left: MemoryAdapterBenchmarkAttemptEvent, + right: MemoryAdapterBenchmarkAttemptEvent, +): boolean { + return ( + left.attemptId === right.attemptId && + left.candidateId === right.candidateId && + left.candidateRef === right.candidateRef && + left.adapterId === right.adapterId && + left.caseId === right.caseId && + left.cellId === right.cellId && + canonicalJson(left.scope) === canonicalJson(right.scope) && + left.adapterCreationCostUsd === right.adapterCreationCostUsd && + left.costUsdPerCase === right.costUsdPerCase && + left.recoveryCostUsdPerAttempt === right.recoveryCostUsdPerAttempt + ) +} + +function assertMemoryBenchmarkAttemptCandidateMatches( + attempt: MemoryAdapterBenchmarkAttemptEvent, + candidate: MemoryAdapterBenchmarkCandidate, +): void { + if (attempt.candidateRef !== candidate.ref) { + throw new Error( + `cannot recover memory benchmark attempt '${attempt.attemptId}': candidate ref changed from '${attempt.candidateRef}' to '${candidate.ref}'`, + ) + } + const expectedAdapterId = memoryAdapterBenchmarkExpectedId(candidate) + if (attempt.adapterId !== expectedAdapterId) { + throw new Error( + `cannot recover memory benchmark attempt '${attempt.attemptId}': adapter id changed from '${attempt.adapterId}' to '${expectedAdapterId}'`, + ) + } + const executionCost = candidate.costUsdPerCase ?? 0 + const adapterCreationCost = candidate.adapterCreationCostUsd ?? 0 + const recoveryCost = candidate.recoveryCostUsdPerAttempt ?? 0 + if ( + adapterCreationCost !== attempt.adapterCreationCostUsd || + executionCost !== attempt.costUsdPerCase || + recoveryCost !== attempt.recoveryCostUsdPerAttempt + ) { + throw new Error( + `cannot recover memory benchmark attempt '${attempt.attemptId}': candidate cost settings changed; start a new run or restore the recorded costs`, + ) + } +} + +function isAgentMemoryScope(value: unknown): value is AgentMemoryScope { + if (!isRecordValue(value)) return false + const allowed = new Set([ + 'tenantId', + 'userId', + 'agentId', + 'teamId', + 'runId', + 'sessionId', + 'namespace', + 'tags', + ]) + if (Object.keys(value).some((key) => !allowed.has(key))) return false + for (const key of [ + 'tenantId', + 'userId', + 'agentId', + 'teamId', + 'runId', + 'sessionId', + 'namespace', + ]) { + if (value[key] !== undefined && typeof value[key] !== 'string') return false + } + if (value.tags === undefined) return true + return ( + isRecordValue(value.tags) && + Object.values(value.tags).every((entry) => typeof entry === 'string') + ) +} + +function isNonEmptyString(value: unknown): value is string { + return typeof value === 'string' && value.trim().length > 0 +} + +function isRecordValue(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value) } export function createNoopMemoryBenchmarkAdapter(id = 'no-memory'): AgentMemoryAdapter { return { id, + branchIsolation: { mode: 'scoped' }, async search() { return [] }, @@ -1000,6 +2122,7 @@ export function createNoopMemoryBenchmarkAdapter(id = 'no-memory'): AgentMemoryA kind: input.kind, } }, + async clear() {}, async flush() {}, } } @@ -1014,6 +2137,7 @@ export function createInMemoryBenchmarkAdapter(options: { id?: string } = {}): A let seq = 0 const adapter: AgentMemoryAdapter = { id, + branchIsolation: { mode: 'scoped' }, async search(query, searchOptions = {}) { const scored = rows .filter((row) => memoryScopeMatches(row.input.scope, searchOptions.scope)) @@ -1081,10 +2205,12 @@ export function createInMemoryBenchmarkAdapter(options: { id?: string } = {}): A metadata: hit.metadata, } }, - async flush() { - rows.length = 0 - seq = 0 + async clear(scope) { + for (let index = rows.length - 1; index >= 0; index -= 1) { + if (memoryScopeMatches(rows[index]!.input.scope, scope)) rows.splice(index, 1) + } }, + async flush() {}, } return adapter } @@ -1164,7 +2290,17 @@ export function buildRetrievalBenchmarkCasesFromQrels( export async function runKnowledgeBenchmarkSuite( options: RunKnowledgeBenchmarkSuiteOptions, ): Promise> { + assertKnowledgeBenchmarkCases(options.cases) + if (options.respondRef !== undefined) { + assertNonEmptyBenchmarkString(options.respondRef, 'knowledge benchmark respondRef') + } else if (options.resumable !== false) { + throw new Error('knowledge benchmark respondRef is required when resumable is enabled') + } const storage = options.storage ?? fsCampaignStorage() + const costCeiling = options.costCeiling ?? options.costLedger?.costCeilingUsd ?? 0 + if (options.costLedger && options.costLedger.costCeilingUsd !== costCeiling) { + throw new Error('knowledge benchmark costCeiling must match the shared cost ledger ceiling') + } const scenarios = buildKnowledgeBenchmarkScenarios(options.cases, options.splits) const dispatch: RunCampaignOptions['dispatch'] = async ( scenario, @@ -1176,14 +2312,22 @@ export async function runKnowledgeBenchmarkSuite({ scenarios, dispatch, - dispatchRef: 'agent-knowledge:benchmark-suite', + dispatchRef: stableId( + 'knowledge_benchmark', + canonicalJson({ + implementationRef: KNOWLEDGE_BENCHMARK_IMPLEMENTATION_REF, + respondRef: options.respondRef ?? 'non-resumable', + }), + ), judges: [knowledgeBenchmarkJudge()], runDir: options.runDir, repo: options.repo, seed: options.seed, reps: options.reps, resumable: options.resumable, - costCeiling: options.costCeiling, + costCeiling, + costLedger: options.costLedger, + costPhase: options.costPhase, maxConcurrency: options.maxConcurrency, dispatchTimeoutMs: options.dispatchTimeoutMs, expectUsage: options.expectUsage ?? 'off', @@ -1263,6 +2407,7 @@ export function knowledgeBenchmarkJudge( > { return { name: 'knowledge-benchmark', + judgeVersion: 'agent-knowledge:knowledge-benchmark:v2', dimensions: [ { key: 'score', description: 'primary knowledge benchmark score' }, { key: 'passed', description: '1 when the benchmark case passes' }, @@ -1409,23 +2554,32 @@ export function scoreMemoryBenchmarkArtifact( required.totalWeight > 0 ? required.recall : undefined, testCase.expectedEventIds && testCase.expectedEventIds.length > 0 ? eventRecall : undefined, testCase.expectedActorIds && testCase.expectedActorIds.length > 0 ? actorRecall : undefined, - forbidden.safe, + testCase.forbiddenFacts && testCase.forbiddenFacts.length > 0 ? forbidden.safe : undefined, ].filter((value): value is number => value !== undefined) const score = mean(components) + const dimensions: Record = {} + if (required.totalWeight > 0) { + dimensions.memory_fact_recall = required.recall + dimensions.memory_required_fact_count = required.total + dimensions.memory_matched_fact_count = required.matched + } + if (testCase.expectedEventIds && testCase.expectedEventIds.length > 0) { + dimensions.memory_event_recall = eventRecall + } + if (testCase.expectedActorIds && testCase.expectedActorIds.length > 0) { + dimensions.memory_actor_recall = actorRecall + } + if (testCase.forbiddenFacts && testCase.forbiddenFacts.length > 0) { + dimensions.memory_stale_safe = forbidden.safe + dimensions.memory_stale_rate = forbidden.rate + dimensions.memory_forbidden_fact_count = forbidden.total + dimensions.memory_matched_forbidden_fact_count = forbidden.matched + } return { score, passed: score >= 1, - dimensions: { - memory_fact_recall: required.recall, - memory_event_recall: eventRecall, - memory_actor_recall: actorRecall, - memory_stale_safe: forbidden.safe, - memory_stale_rate: forbidden.rate, - memory_required_fact_count: required.total, - memory_matched_fact_count: required.matched, - memory_forbidden_fact_count: forbidden.total, - memory_matched_forbidden_fact_count: forbidden.matched, - }, + dimensions, + applicableDimensions: Object.keys(dimensions), notes: `memory required=${required.matched}/${required.total}; stale=${forbidden.matched}/${forbidden.total}; event_recall=${eventRecall.toFixed(3)}; actor_recall=${actorRecall.toFixed(3)}`, raw: { matchedRequiredFactIds: required.matchedIds, @@ -1610,10 +2764,15 @@ function memoryLifecycleCase(input: { function renderMemoryAdapterRankingMarkdown( rows: readonly MemoryAdapterBenchmarkRankingRow[], + totalCostUsd: number, + unrankedRecoveryCostUsd: number, ): string { return [ '# Memory Adapter Ranking', '', + `- total cost: $${formatNumber(totalCostUsd)}`, + `- retired-candidate recovery cost: $${formatNumber(unrankedRecoveryCostUsd)}`, + '', '| rank | candidate | adapter | cases | cells | failed | mean score | pass rate | cost |', '| ---: | --- | --- | ---: | ---: | ---: | ---: | ---: | ---: |', ...rows.map( @@ -1627,19 +2786,34 @@ function renderMemoryAdapterRankingMarkdown( function benchmarkMemoryScope( candidateId: string, testCase: KnowledgeMemoryBenchmarkCase, + cellId: string, + attemptId: string, scope: AgentMemoryScope = {}, ): AgentMemoryScope { return { ...scope, - namespace: scope.namespace ?? 'agent-knowledge-memory-benchmark', + namespace: `${scope.namespace ?? 'agent-knowledge-memory-benchmark'}:${attemptId}`, tags: { ...(scope.tags ?? {}), benchmarkCandidateId: candidateId, benchmarkCaseId: testCase.id, + benchmarkCellId: cellId, + benchmarkAttemptId: attemptId, }, } } +function assertScopedMemoryBenchmarkAdapter(adapter: AgentMemoryAdapter): void { + if (adapter.branchIsolation?.mode !== 'scoped') { + throw new Error( + `${adapter.id}: direct memory benchmark requires branchIsolation mode scoped; use runAgentMemoryExperiment for dedicated instances`, + ) + } + if (!adapter.clear) { + throw new Error(`${adapter.id}: direct memory benchmark requires exact scoped clear`) + } +} + function memoryEventId(hit: AgentMemoryHit): string | undefined { const eventId = hit.metadata?.eventId return typeof eventId === 'string' ? eventId : undefined @@ -1654,6 +2828,9 @@ function memoryScopeMatches(stored?: AgentMemoryScope, requested?: AgentMemorySc if (!requested) return true if (requested.tenantId !== undefined && stored?.tenantId !== requested.tenantId) return false if (requested.userId !== undefined && stored?.userId !== requested.userId) return false + if (requested.agentId !== undefined && stored?.agentId !== requested.agentId) return false + if (requested.teamId !== undefined && stored?.teamId !== requested.teamId) return false + if (requested.runId !== undefined && stored?.runId !== requested.runId) return false if (requested.sessionId !== undefined && stored?.sessionId !== requested.sessionId) return false if (requested.namespace !== undefined && stored?.namespace !== requested.namespace) return false for (const [key, value] of Object.entries(requested.tags ?? {})) { @@ -1718,6 +2895,74 @@ export function isKnowledgeMemoryBenchmarkCase( return testCase.taskKind.startsWith('memory-') } +function assertKnowledgeBenchmarkCases(cases: readonly KnowledgeBenchmarkCase[]): void { + if (cases.length === 0) throw new Error('knowledge benchmark requires cases') + assertUniqueNonEmptyStrings( + cases.map((testCase) => testCase.id), + 'knowledge benchmark case id', + ) + for (const testCase of cases) { + if (typeof testCase.family !== 'string' || !testCase.family.trim()) { + throw new Error(`knowledge benchmark case ${testCase.id} requires a family`) + } + if (testCase.taskKind === 'retrieval') continue + if (isKnowledgeMemoryBenchmarkCase(testCase)) { + assertUniqueNonEmptyStrings( + testCase.events.map((event) => event.id), + `${testCase.id} memory event id`, + ) + for (const event of testCase.events) { + assertNonEmptyBenchmarkString(event.text, `${testCase.id} memory event ${event.id} text`) + } + assertClaimMatchers(testCase.requiredFacts ?? [], `${testCase.id} requiredFacts`) + assertClaimMatchers(testCase.forbiddenFacts ?? [], `${testCase.id} forbiddenFacts`) + assertUniqueNonEmptyStrings( + testCase.expectedEventIds ?? [], + `${testCase.id} expected event id`, + ) + assertUniqueNonEmptyStrings( + testCase.expectedActorIds ?? [], + `${testCase.id} expected actor id`, + ) + } else { + assertClaimMatchers(testCase.requiredClaims ?? [], `${testCase.id} requiredClaims`) + assertClaimMatchers(testCase.forbiddenClaims ?? [], `${testCase.id} forbiddenClaims`) + assertUniqueNonEmptyStrings(testCase.expectedSourceIds ?? [], `${testCase.id} source id`) + } + } +} + +function assertClaimMatchers(claims: readonly KnowledgeClaimMatcher[], label: string): void { + assertUniqueNonEmptyStrings( + claims.map((claim) => claim.id), + `${label} matcher id`, + ) + for (const claim of claims) { + if (claim.anyOf.length === 0) throw new Error(`${label} matcher ${claim.id} requires anyOf`) + assertUniqueNonEmptyStrings(claim.anyOf, `${label} matcher ${claim.id} anyOf`) + if (claim.weight !== undefined && (!Number.isFinite(claim.weight) || claim.weight <= 0)) { + throw new Error(`${label} matcher ${claim.id} weight must be a positive finite number`) + } + const sourceEventIds = (claim as Partial).sourceEventIds + if (sourceEventIds !== undefined) { + assertUniqueNonEmptyStrings(sourceEventIds, `${label} matcher ${claim.id} source event id`) + } + } +} + +function assertUniqueNonEmptyStrings(values: readonly string[], label: string): void { + const seen = new Set() + for (const value of values) { + assertNonEmptyBenchmarkString(value, label) + if (seen.has(value)) throw new Error(`duplicate ${label}: ${value}`) + seen.add(value) + } +} + +function assertNonEmptyBenchmarkString(value: unknown, label: string): asserts value is string { + if (typeof value !== 'string' || !value.trim()) throw new Error(`${label} must be non-empty`) +} + function scoreClaims(text: string, claims: readonly KnowledgeClaimMatcher[]) { let matched = 0 let matchedWeight = 0 @@ -1725,7 +2970,19 @@ function scoreClaims(text: string, claims: readonly KnowledgeClaimMatcher[]) { const matchedIds: string[] = [] const haystack = text.toLowerCase() for (const claim of claims) { + if ( + !claim.id.trim() || + claim.anyOf.length === 0 || + claim.anyOf.some((value) => !value.trim()) + ) { + throw new Error( + 'claim matchers require a non-empty id and at least one non-empty alternative', + ) + } const weight = claim.weight ?? 1 + if (!Number.isFinite(weight) || weight <= 0) { + throw new Error(`claim matcher ${claim.id} weight must be a positive finite number`) + } totalWeight += weight if (claim.anyOf.some((fragment) => haystack.includes(fragment.toLowerCase()))) { matched += 1 @@ -1854,6 +3111,10 @@ function formatNumber(value: number): string { return value.toFixed(value === 0 || Math.abs(value) >= 10 ? 0 : 3) } +function normalizeUsd(value: number): number { + return Number(value.toFixed(12)) +} + function compactObject(value: unknown): unknown { if (Array.isArray(value)) return value.map(compactObject) if (!value || typeof value !== 'object') return value diff --git a/src/memory/attempt-log.ts b/src/memory/attempt-log.ts new file mode 100644 index 0000000..56f1f02 --- /dev/null +++ b/src/memory/attempt-log.ts @@ -0,0 +1,292 @@ +import type { CampaignStorage, CostLedgerHandle } from '@tangle-network/agent-eval/campaign' + +export interface AttemptJournalEvent { + status: 'started' | 'cleaned' + recordedAt: string +} + +interface RecoveryAttemptEvent { + schema: 1 + attemptId: string + generation: number + recordedAt: string +} + +const journalByteLengths = new WeakMap>() + +export const DEFAULT_MEMORY_RECOVERY_RETRIES_PER_ATTEMPT = 3 + +const MEMORY_PROVIDER_ACTOR_PREFIXES = [ + 'agent-knowledge:memory-adapter:', + 'agent-knowledge:memory-adapter-recovery:', + 'agent-knowledge:memory-experiment:', + 'agent-knowledge:memory-recovery:', +] as const + +export function reconcileInterruptedMemoryPaidCalls(costLedger: CostLedgerHandle): void { + reconcileInterruptedPaidCallsAtMaximum( + costLedger, + (actor) => isMemoryProviderActor(actor), + 'external memory provider', + ) +} + +export function reconcileInterruptedRunPaidCalls( + costLedger: CostLedgerHandle, + label: string, +): void { + reconcileInterruptedPaidCallsAtMaximum(costLedger, () => true, label) +} + +function reconcileInterruptedPaidCallsAtMaximum( + costLedger: CostLedgerHandle, + ownsActor: (actor: string) => boolean, + label: string, +): void { + if (costLedger.summary().unresolvedCalls === 0) return + const pending = requirePendingCostCallInspection(costLedger) + for (const call of pending) { + if (call.state !== 'interrupted' || !ownsActor(call.actor)) continue + if (call.maximumCostUsd === undefined) { + throw new Error( + `cannot recover unbounded interrupted ${label} call '${call.callId}' for '${call.actor}'`, + ) + } + try { + costLedger.reconcile( + call.callId, + { + model: call.model, + inputTokens: 0, + outputTokens: 0, + actualCostUsd: call.maximumCostUsd, + }, + { + error: `process exited before the ${label} receipt for '${call.actor}' was recorded; charged the reserved maximum`, + }, + ) + } catch (error) { + if (isMissingPendingCostCall(error, call.callId)) continue + throw error + } + } +} + +export function assertNoInterruptedPaidCalls(costLedger: CostLedgerHandle, label: string): void { + const unresolved = costLedger.summary().unresolvedCalls + if (unresolved === 0) return + const pending = requirePendingCostCallInspection(costLedger) + const blocked = pending.filter((call) => call.state !== 'active') + throw new Error( + `${label} has ${unresolved} unresolved paid call(s) that cannot be resumed: ${blocked + .map((call) => `${call.callId} (${call.actor}, ${call.state})`) + .join(', ')}`, + ) +} + +export function hasSettledPaidCall(costLedger: CostLedgerHandle, callId: string): boolean { + return costLedger.list().some((receipt) => receipt.callId === callId) +} + +export function appendAttemptJournalEvent(input: { + storage: CampaignStorage + path: string + event: TEvent + label: string +}): void { + appendDurableJournalEvent(input) +} + +export function appendDurableJournalEvent(input: { + storage: CampaignStorage + path: string + event: TEvent + label: string +}): void { + const { storage, path, event, label } = input + if (!storage.append) throw new Error(`${label} requires CampaignStorage.append`) + const line = `${JSON.stringify(event)}\n` + const byteLengths = journalByteLengths.get(storage) ?? new Map() + journalByteLengths.set(storage, byteLengths) + for (let retry = 0; retry < 100; retry += 1) { + let expectedBytes = byteLengths.get(path) + if (expectedBytes === undefined) { + const current = storage.read(path) + if (current === undefined && storage.exists(path)) { + throw new Error(`cannot read ${label} '${path}'`) + } + expectedBytes = new TextEncoder().encode(current ?? '').byteLength + byteLengths.set(path, expectedBytes) + } + const appendedBytes = storage.append(path, line, expectedBytes) + if (appendedBytes !== undefined) { + byteLengths.set(path, appendedBytes) + return + } + byteLengths.delete(path) + } + throw new Error(`${label} '${path}' remained contended after 100 retries`) +} + +/** Reserves one durable recovery generation per attempt before provider work starts. */ +export function reserveRecoveryAttempts(input: { + storage: CampaignStorage + path: string + attemptIds: readonly string[] + maxRetriesPerAttempt: number + label: string + now?: () => Date +}): ReadonlyMap { + const { storage, path, attemptIds, maxRetriesPerAttempt, label } = input + if (!Number.isSafeInteger(maxRetriesPerAttempt) || maxRetriesPerAttempt <= 0) { + throw new Error(`${label} maxRetriesPerAttempt must be a positive safe integer`) + } + const uniqueAttemptIds = new Set(attemptIds) + if (uniqueAttemptIds.size !== attemptIds.length || attemptIds.some((id) => !id.trim())) { + throw new Error(`${label} attempt ids must be unique non-empty strings`) + } + const generations = readRecoveryAttemptGenerations(storage, path, label) + for (const attemptId of attemptIds) { + const prior = generations.get(attemptId) ?? 0 + if (prior >= maxRetriesPerAttempt) { + throw new Error( + `${label} '${attemptId}' exhausted ${maxRetriesPerAttempt} recovery attempts; increase maxRecoveryRetriesPerAttempt only after inspecting provider state`, + ) + } + } + const reserved = new Map() + for (const attemptId of attemptIds) { + const generation = (generations.get(attemptId) ?? 0) + 1 + appendDurableJournalEvent({ + storage, + path, + label, + event: { + schema: 1, + attemptId, + generation, + recordedAt: (input.now ?? (() => new Date()))().toISOString(), + } satisfies RecoveryAttemptEvent, + }) + generations.set(attemptId, generation) + reserved.set(attemptId, generation) + } + return reserved +} + +export function readActiveAttemptJournal(input: { + storage: CampaignStorage + path: string + label: string + parse(value: unknown, path: string, line: number): TEvent + id(event: TEvent): string + sameAttempt(left: TEvent, right: TEvent): boolean +}): TEvent[] { + const { storage, path, label, parse, id, sameAttempt } = input + const stored = storage.read(path) + if (stored === undefined) { + if (storage.exists(path)) throw new Error(`cannot read ${label} '${path}'`) + return [] + } + const active = new Map() + const completed = new Map() + for (const [index, line] of stored.split('\n').entries()) { + if (!line) continue + let raw: unknown + try { + raw = JSON.parse(line) + } catch (error) { + throw new Error(`invalid ${label} '${path}' line ${index + 1}`, { cause: error }) + } + const event = parse(raw, path, index + 1) + const attemptId = id(event) + if (event.status === 'started') { + if (active.has(attemptId) || completed.has(attemptId)) { + throw new Error(`${label} '${path}' repeats attempt '${attemptId}'`) + } + active.set(attemptId, event) + continue + } + + const started = active.get(attemptId) + if (started && sameAttempt(started, event)) { + active.delete(attemptId) + completed.set(attemptId, started) + continue + } + + const prior = completed.get(attemptId) + if (prior && sameAttempt(prior, event)) continue + throw new Error(`${label} '${path}' has an unmatched cleanup for '${attemptId}'`) + } + return [...active.values()] +} + +function readRecoveryAttemptGenerations( + storage: CampaignStorage, + path: string, + label: string, +): Map { + const stored = storage.read(path) + if (stored === undefined) { + if (storage.exists(path)) throw new Error(`cannot read ${label} '${path}'`) + return new Map() + } + const generations = new Map() + for (const [index, line] of stored.split('\n').entries()) { + if (!line) continue + let raw: unknown + try { + raw = JSON.parse(line) + } catch (error) { + throw new Error(`invalid ${label} '${path}' line ${index + 1}`, { cause: error }) + } + const event = parseRecoveryAttemptEvent(raw, path, index + 1, label) + const expected = (generations.get(event.attemptId) ?? 0) + 1 + if (event.generation !== expected) { + throw new Error( + `${label} '${path}' has recovery generation ${event.generation} for '${event.attemptId}', expected ${expected}`, + ) + } + generations.set(event.attemptId, event.generation) + } + return generations +} + +function parseRecoveryAttemptEvent( + value: unknown, + path: string, + line: number, + label: string, +): RecoveryAttemptEvent { + const event = value as Partial | null + const valid = + typeof event === 'object' && + event !== null && + event.schema === 1 && + typeof event.attemptId === 'string' && + event.attemptId.length > 0 && + Number.isSafeInteger(event.generation) && + event.generation! > 0 && + typeof event.recordedAt === 'string' && + !Number.isNaN(Date.parse(event.recordedAt)) + if (!valid) throw new Error(`invalid ${label} '${path}' line ${line}`) + return value as RecoveryAttemptEvent +} + +function isMissingPendingCostCall(error: unknown, callId: string): boolean { + return error instanceof Error && error.message === `CostLedger: no pending call '${callId}'` +} + +function requirePendingCostCallInspection(costLedger: CostLedgerHandle) { + if (!costLedger.listPending) { + throw new Error( + 'interrupted memory recovery requires CostLedger.listPending() from @tangle-network/agent-eval 0.122.8 or newer', + ) + } + return costLedger.listPending() +} + +function isMemoryProviderActor(actor: string): boolean { + return MEMORY_PROVIDER_ACTOR_PREFIXES.some((prefix) => actor.startsWith(prefix)) +} diff --git a/src/memory/branch.ts b/src/memory/branch.ts new file mode 100644 index 0000000..f75ac76 --- /dev/null +++ b/src/memory/branch.ts @@ -0,0 +1,771 @@ +import { canonicalJson } from '@tangle-network/agent-eval' +import { sha256, stableId } from '../ids' +import { defaultGetMemoryContext } from './adapter' +import { mergeRankedMemoryHits } from './rank' +import type { + AgentMemoryAdapter, + AgentMemoryScope, + AgentMemoryWriteInput, + AgentMemoryWriteResult, +} from './types' + +export type AgentMemoryVisibility = 'private' | 'team' | 'shared' +export type AgentMemoryBranchLifetime = 'resumable' | 'attempt' + +export interface AgentMemorySharingPolicy { + read: readonly AgentMemoryVisibility[] + write: AgentMemoryVisibility +} + +export interface AgentMemoryJournalEntry { + sequence: number + input: AgentMemoryWriteInput + result: AgentMemoryWriteResult +} + +export interface AgentMemoryBranchSnapshot { + branchId: string + lifetime: AgentMemoryBranchLifetime + parentBranchId?: string + adapterId: string + policy: AgentMemorySharingPolicy + baseScope: AgentMemoryScope + journal: readonly AgentMemoryJournalEntry[] + digest: `sha256:${string}` +} + +export interface AgentMemoryBranch extends AgentMemoryAdapter { + readonly branchId: string + readonly lifetime: AgentMemoryBranchLifetime + readonly parentBranchId?: string + readonly policy: AgentMemorySharingPolicy + snapshot(): Promise + fork(options: { + branchId: string + adapter?: AgentMemoryAdapter + policy?: AgentMemorySharingPolicy + baseScope?: AgentMemoryScope + lifetime?: AgentMemoryBranchLifetime + }): Promise +} + +export interface CreateAgentMemoryBranchOptions { + adapter: AgentMemoryAdapter + branchId: string + parentBranchId?: string + policy?: AgentMemorySharingPolicy + baseScope?: AgentMemoryScope + /** Attempt branches use a new ID after a process restart and cannot bind persisted state. */ + lifetime?: AgentMemoryBranchLifetime + /** Rebind a persisted branch whose external memory is still present. */ + snapshot?: AgentMemoryBranchSnapshot + /** Optional exact logical scopes allowed to write, used by crash-safe experiments. */ + allowedWriteScopes?: readonly AgentMemoryScope[] +} + +export interface ForkAgentMemoryBranchSnapshotOptions { + snapshot: AgentMemoryBranchSnapshot + adapter: AgentMemoryAdapter + branchId: string + policy?: AgentMemorySharingPolicy + baseScope?: AgentMemoryScope + lifetime?: AgentMemoryBranchLifetime +} + +const DEFAULT_POLICY: AgentMemorySharingPolicy = { + read: ['private'], + write: 'private', +} + +interface AdapterLeaseState { + references: number + closed: boolean + activeBranchIds: Set + closePromise?: Promise +} + +const adapterLeases = new WeakMap() + +/** + * Isolates one candidate branch and records accepted writes for durable resume + * or replay into a child branch, including a child backed by another provider. + */ +export function createAgentMemoryBranch( + options: CreateAgentMemoryBranchOptions, +): AgentMemoryBranch { + if (typeof options.branchId !== 'string' || !options.branchId.trim()) { + throw new Error('memory branchId must be a non-empty string') + } + const snapshot = options.snapshot + if (snapshot && snapshot.branchId !== options.branchId) { + throw new Error( + `memory branch snapshot belongs to ${snapshot.branchId}, not ${options.branchId}`, + ) + } + if (snapshot && snapshot.adapterId !== options.adapter.id) { + throw new Error( + `memory branch snapshot uses ${snapshot.adapterId}, not ${options.adapter.id}; fork it to replay into another adapter`, + ) + } + if (snapshot?.lifetime === 'attempt') { + throw new Error( + 'memory branch attempt snapshots cannot be resumed; replay them into a fresh branch instead', + ) + } + if (options.lifetime !== undefined && !['resumable', 'attempt'].includes(options.lifetime)) { + throw new Error('memory branch lifetime must be resumable or attempt') + } + if (snapshot) { + assertDurableValue(snapshot, `${options.branchId}: branch snapshot`) + assertSnapshotDigest(snapshot) + } + if ( + snapshot && + options.policy && + canonicalJson(stripUndefined(options.policy)) !== canonicalJson(stripUndefined(snapshot.policy)) + ) { + throw new Error('memory branch snapshot policy cannot change during resume; fork it instead') + } + if ( + snapshot && + options.baseScope && + canonicalJson(stripUndefined(options.baseScope)) !== + canonicalJson(stripUndefined(snapshot.baseScope)) + ) { + throw new Error('memory branch snapshot scope cannot change during resume; fork it instead') + } + if (snapshot && options.lifetime && options.lifetime !== snapshot.lifetime) { + throw new Error('memory branch snapshot lifetime cannot change during resume; fork it instead') + } + + const branchId = options.branchId + const parentBranchId = options.parentBranchId ?? snapshot?.parentBranchId + const lifetime = options.lifetime ?? snapshot?.lifetime ?? 'resumable' + const policy = normalizePolicy(options.policy ?? snapshot?.policy ?? DEFAULT_POLICY) + assertBranchIsolation(options.adapter, branchId, policy, lifetime) + const mergedBaseScope = mergeScopes(undefined, snapshot?.baseScope ?? options.baseScope) + assertDurableValue(mergedBaseScope, `${branchId}: base scope`) + const baseScope = cloneDurableValue(mergedBaseScope) + const allowedWriteScopeKeys = options.allowedWriteScopes + ? new Set( + options.allowedWriteScopes.map((scope) => + canonicalJson(stripUndefined(mergeScopes(baseScope, scope))), + ), + ) + : undefined + const journal = [...(snapshot?.journal ?? [])].map(cloneJournalEntry) + let nextSequence = journal.reduce((max, entry) => Math.max(max, entry.sequence + 1), 0) + const queues = new Map>() + const pending = new Set>() + const reads = new Set>() + const active = new Set>() + const touchedScopes = new Map() + for (const entry of journal) { + const scope = mergeScopes(baseScope, entry.input.scope) + touchedScopes.set(canonicalJson(stripUndefined(scope)), scope) + } + let mutationBarrier = Promise.resolve() + const releaseAdapter = retainAdapter(options.adapter, branchId) + let lifecycle: 'open' | 'closing' | 'closed' = 'open' + let closePromise: Promise | undefined + + const assertOpen = (operation: string): void => { + if (lifecycle !== 'open') throw new Error(`${branchId}: cannot ${operation} a closed branch`) + } + const track = (operation: Promise): Promise => { + active.add(operation) + void operation.then( + () => active.delete(operation), + () => active.delete(operation), + ) + return operation + } + const trackRead = (operation: Promise): Promise => { + reads.add(operation) + void operation.then( + () => reads.delete(operation), + () => reads.delete(operation), + ) + return track(operation) + } + const flushWrites = async (): Promise => { + const results = await Promise.allSettled([...pending]) + const failures = results.flatMap((result) => + result.status === 'rejected' ? [result.reason] : [], + ) + if (failures.length === 1) throw failures[0] + if (failures.length > 1) { + throw new AggregateError(failures, `${branchId}: pending memory writes failed`) + } + } + const flushAll = async (): Promise => { + await flushWrites() + await options.adapter.flush?.() + } + const runExclusive = async ( + operation: () => Promise, + exclusiveOptions: { failOnOperationError?: boolean; flushAdapter?: boolean } = {}, + ): Promise => { + const precedingBarrier = mutationBarrier + const precedingWrites = [...pending] + const precedingReads = [...reads] + let releaseBarrier!: () => void + const ownBarrier = new Promise((resolve) => { + releaseBarrier = resolve + }) + mutationBarrier = precedingBarrier.then(() => ownBarrier) + await precedingBarrier + const settled = await Promise.allSettled([...precedingWrites, ...precedingReads]) + const failures = settled.flatMap((result) => + result.status === 'rejected' ? [result.reason] : [], + ) + try { + if ((exclusiveOptions.failOnOperationError ?? true) && failures.length > 0) { + if (failures.length === 1) throw failures[0] + throw new AggregateError(failures, `${branchId}: pending memory operations failed`) + } + if (exclusiveOptions.flushAdapter ?? true) await options.adapter.flush?.() + return await operation() + } finally { + releaseBarrier() + } + } + + const branch: AgentMemoryBranch = { + id: `${options.adapter.id}:branch:${stableId('branch', branchId)}`, + branchId, + lifetime, + ...(parentBranchId !== undefined ? { parentBranchId } : {}), + policy, + async search(query, searchOptions = {}) { + assertOpen('search') + const precedingBarrier = mutationBarrier + return trackRead( + (async () => { + await precedingBarrier + const logicalScope = mergeScopes(baseScope, searchOptions.scope) + const scopes = policy.read.map((visibility) => + physicalScope(branchId, logicalScope, visibility), + ) + const groups = await Promise.all( + scopes.map(async (scope) => { + const hits = await options.adapter.search(query, { + ...searchOptions, + scope, + holdout: undefined, + }) + return hits.map((hit) => ({ + ...hit, + metadata: { + ...hit.metadata, + memoryBranchId: branchId, + memoryVisibility: scope.tags?.memoryVisibility, + }, + })) + }), + ) + return mergeRankedMemoryHits(groups, searchOptions.limit) + })(), + ) + }, + async getContext(query, searchOptions = {}) { + assertOpen('read context from') + return track(defaultGetMemoryContext(branch, query, searchOptions)) + }, + async write(input) { + assertOpen('write to') + const sequence = nextSequence + nextSequence += 1 + assertDurableValue(input, `${branch.id}: write input`) + const logicalInput = cloneWriteInput(input) + logicalInput.id ??= stableId('memory_write', `${branchId}:${sequence}`) + const logicalScope = mergeScopes(baseScope, logicalInput.scope) + if ( + allowedWriteScopeKeys && + !allowedWriteScopeKeys.has(canonicalJson(stripUndefined(logicalScope))) + ) { + throw new Error( + `${branch.id}: write scope was not declared in the experiment sequence cleanupScopes`, + ) + } + const scope = physicalScope(branchId, logicalScope, policy.write) + const queueKey = writeQueueKey(branchId, logicalScope, policy.write) + touchedScopes.set(canonicalJson(stripUndefined(logicalScope)), logicalScope) + const previous = queues.get(queueKey) ?? Promise.resolve() + const precedingBarrier = mutationBarrier + let resultPromise: Promise + const queued = Promise.all([previous.catch(() => undefined), precedingBarrier]).then( + async () => { + const result = await options.adapter.write({ ...logicalInput, scope }) + if (result.accepted) { + assertDurableValue(result, `${branch.id}: write result`) + journal.push({ + sequence, + input: logicalInput, + result: cloneWriteResult(result), + }) + } + return result + }, + ) + resultPromise = queued + const tail = queued.then( + () => undefined, + () => undefined, + ) + queues.set(queueKey, tail) + pending.add(resultPromise) + void resultPromise.then( + () => { + pending.delete(resultPromise) + if (queues.get(queueKey) === tail) queues.delete(queueKey) + }, + () => { + pending.delete(resultPromise) + if (queues.get(queueKey) === tail) queues.delete(queueKey) + }, + ) + return track(resultPromise) + }, + async clear(scope) { + assertOpen('clear') + return track( + runExclusive( + async () => { + if (!options.adapter.clear) { + throw new Error(`${options.adapter.id}: adapter does not support scoped clear`) + } + const flushErrors: unknown[] = [] + try { + await options.adapter.flush?.() + } catch (error) { + flushErrors.push(error) + } + const requestedScope = scope ? mergeScopes(baseScope, scope) : undefined + const logicalScopes = new Map() + if (requestedScope) { + logicalScopes.set(canonicalJson(stripUndefined(requestedScope)), requestedScope) + for (const touched of touchedScopes.values()) { + if (scopeMatches(touched, requestedScope)) { + logicalScopes.set(canonicalJson(stripUndefined(touched)), touched) + } + } + } else if (touchedScopes.size > 0) { + for (const [key, touched] of touchedScopes) logicalScopes.set(key, touched) + } else { + logicalScopes.set(canonicalJson(stripUndefined(baseScope)), baseScope) + } + const visibilities = new Set([...policy.read, policy.write]) + const physicalScopes = new Map() + for (const logicalScope of logicalScopes.values()) { + for (const visibility of visibilities) { + const physical = physicalScope(branchId, logicalScope, visibility) + physicalScopes.set(canonicalJson(stripUndefined(physical)), physical) + } + } + const clearErrors: unknown[] = [] + const clearedPhysicalScopes = new Set() + for (const [key, physical] of physicalScopes) { + try { + await options.adapter.clear(physical) + clearedPhysicalScopes.add(key) + } catch (error) { + clearErrors.push(error) + } + } + if (clearedPhysicalScopes.size > 0) { + for (let index = journal.length - 1; index >= 0; index -= 1) { + const logical = mergeScopes(baseScope, journal[index]!.input.scope) + const physical = physicalScope(branchId, logical, policy.write) + if (clearedPhysicalScopes.has(canonicalJson(stripUndefined(physical)))) { + journal.splice(index, 1) + } + } + for (const [key, touched] of touchedScopes) { + const physical = physicalScope(branchId, touched, policy.write) + if (clearedPhysicalScopes.has(canonicalJson(stripUndefined(physical)))) { + touchedScopes.delete(key) + } + } + } + const failures = [...flushErrors, ...clearErrors] + if (failures.length === 1) throw failures[0] + if (failures.length > 1) { + throw new AggregateError(failures, `${branchId}: memory clear failed`) + } + }, + { failOnOperationError: false, flushAdapter: false }, + ), + ) + }, + async flush() { + assertOpen('flush') + return track(runExclusive(async () => undefined)) + }, + close() { + if (closePromise) return closePromise + lifecycle = 'closing' + closePromise = (async () => { + const errors: unknown[] = [] + const operations = await Promise.allSettled([...active]) + for (const result of operations) { + if (result.status === 'rejected') errors.push(result.reason) + } + try { + await flushAll() + } catch (error) { + errors.push(error) + } + try { + await releaseAdapter() + } catch (error) { + errors.push(error) + } + lifecycle = 'closed' + if (errors.length === 1) throw errors[0] + if (errors.length > 1) throw new AggregateError(errors, `${branch.id}: close failed`) + })() + return closePromise + }, + async snapshot() { + assertOpen('snapshot') + return track( + runExclusive(async () => { + return createSnapshot({ + branchId, + lifetime, + adapterId: options.adapter.id, + policy, + baseScope, + journal, + ...(parentBranchId !== undefined ? { parentBranchId } : {}), + }) + }), + ) + }, + async fork(forkOptions) { + assertOpen('fork') + if (forkOptions.branchId === branchId) { + throw new Error(`memory branch ${branchId}: child branchId must differ from its parent`) + } + return track( + (async () => + forkAgentMemoryBranchSnapshot({ + snapshot: await branch.snapshot(), + adapter: forkOptions.adapter ?? options.adapter, + branchId: forkOptions.branchId, + policy: forkOptions.policy, + baseScope: forkOptions.baseScope, + lifetime: forkOptions.lifetime, + }))(), + ) + }, + } + return branch +} + +/** Replays a durable journal into a fresh provider branch. */ +export async function forkAgentMemoryBranchSnapshot( + options: ForkAgentMemoryBranchSnapshotOptions, +): Promise { + assertDurableValue(options.snapshot, `${options.snapshot.branchId}: branch snapshot`) + assertSnapshotDigest(options.snapshot) + if (options.branchId === options.snapshot.branchId) { + throw new Error(`memory branch ${options.branchId}: child branchId must differ from its parent`) + } + const child = createAgentMemoryBranch({ + adapter: options.adapter, + branchId: options.branchId, + parentBranchId: options.snapshot.branchId, + policy: options.policy ?? options.snapshot.policy, + baseScope: mergeScopes(options.snapshot.baseScope, options.baseScope), + lifetime: options.lifetime ?? options.snapshot.lifetime, + }) + try { + for (const entry of options.snapshot.journal) { + const result = await child.write(entry.input) + if (!result.accepted) { + throw new Error( + `memory branch ${options.branchId}: adapter rejected replay of journal entry ${entry.sequence}`, + ) + } + } + await child.flush?.() + return child + } catch (error) { + const cleanupErrors: unknown[] = [] + try { + await child.clear?.() + } catch (cleanupError) { + cleanupErrors.push(cleanupError) + } + try { + await child.close?.() + } catch (cleanupError) { + cleanupErrors.push(cleanupError) + } + if (cleanupErrors.length > 0) { + throw new AggregateError( + [error, ...cleanupErrors], + `memory branch ${options.branchId}: replay and cleanup failed`, + ) + } + throw error + } +} + +function assertBranchIsolation( + adapter: AgentMemoryAdapter, + branchId: string, + policy: AgentMemorySharingPolicy, + lifetime: AgentMemoryBranchLifetime, +): void { + const isolation = adapter.branchIsolation + if (!isolation) { + throw new Error( + `${adapter.id}: cannot isolate memory branch '${branchId}': adapter must declare branchIsolation`, + ) + } + if (isolation.mode === 'unsupported') { + throw new Error( + `${adapter.id}: cannot isolate memory branch '${branchId}': ${isolation.reason}`, + ) + } + if ( + isolation.mode === 'scoped' && + isolation.processExitSafe === false && + lifetime !== 'attempt' + ) { + throw new Error( + `${adapter.id}: memory branch '${branchId}' must use lifetime='attempt' because provider writes can outlive the worker process`, + ) + } + if ( + isolation.mode === 'scoped' && + isolation.processExitSafe === false && + (!Number.isSafeInteger(isolation.recoveryDelayMs) || isolation.recoveryDelayMs! <= 0) + ) { + throw new Error(`${adapter.id}: processExitSafe=false requires a positive recoveryDelayMs`) + } + if (isolation.mode === 'instance' && isolation.branchId !== branchId) { + throw new Error( + `${adapter.id}: adapter instance belongs to branch '${isolation.branchId}', not '${branchId}'`, + ) + } + if ( + isolation.mode === 'instance' && + isolation.supportsLogicalScopes !== true && + (policy.write !== 'shared' || policy.read.length !== 1 || policy.read[0] !== 'shared') + ) { + throw new Error( + `${adapter.id}: dedicated branch instances without logical scope isolation support only shared memory policy`, + ) + } +} + +function retainAdapter(adapter: AgentMemoryAdapter, branchId: string): () => Promise { + const existing = adapterLeases.get(adapter) + if (existing?.closed) throw new Error(`${adapter.id}: adapter is already closed`) + const state = existing ?? { references: 0, closed: false, activeBranchIds: new Set() } + if (state.activeBranchIds.has(branchId)) { + throw new Error(`${adapter.id}: memory branch '${branchId}' already has an open handle`) + } + state.activeBranchIds.add(branchId) + state.references += 1 + adapterLeases.set(adapter, state) + let released = false + return async () => { + if (released) return + released = true + state.activeBranchIds.delete(branchId) + state.references -= 1 + if (state.references > 0) return + if (!adapter.close) { + adapterLeases.delete(adapter) + return + } + state.closed = true + state.closePromise ??= adapter.close() + await state.closePromise + } +} + +function physicalScope( + branchId: string, + logical: AgentMemoryScope, + visibility: AgentMemoryVisibility, +): AgentMemoryScope { + if (visibility === 'private' && !logical.agentId) { + throw new Error('private memory requires scope.agentId') + } + if (visibility === 'team' && !logical.teamId) { + throw new Error('team memory requires scope.teamId') + } + const principal = + visibility === 'private' ? logical.agentId! : visibility === 'team' ? logical.teamId! : 'all' + const namespace = [ + logical.namespace ?? 'default', + stableId('branch', branchId), + visibility, + stableId('principal', principal), + ].join('/') + const physical: AgentMemoryScope = { + ...logical, + namespace, + tags: { + ...(logical.tags ?? {}), + memoryBranchId: branchId, + memoryVisibility: visibility, + logicalNamespace: logical.namespace ?? 'default', + }, + } + if (visibility === 'shared') { + delete physical.agentId + delete physical.teamId + } else if (visibility === 'team') { + delete physical.agentId + } + return physical +} + +function writeQueueKey( + branchId: string, + logical: AgentMemoryScope, + visibility: AgentMemoryVisibility, +): string { + return canonicalJson( + stripUndefined({ + branchId, + visibility, + tenantId: logical.tenantId, + userId: logical.userId, + agentId: logical.agentId, + fallbackTeamId: logical.agentId === undefined ? logical.teamId : undefined, + fallbackPrincipal: + logical.agentId === undefined && logical.teamId === undefined ? 'unattributed' : undefined, + }), + ) +} + +function normalizePolicy(policy: AgentMemorySharingPolicy): AgentMemorySharingPolicy { + const allowed = new Set(['private', 'team', 'shared']) + if (!allowed.has(policy.write) || policy.read.some((visibility) => !allowed.has(visibility))) { + throw new Error('memory sharing policy contains an unsupported visibility') + } + const read = [...new Set(policy.read)] + if (read.length === 0) throw new Error('memory sharing policy must read at least one visibility') + return { read, write: policy.write } +} + +function createSnapshot( + input: Omit, +): AgentMemoryBranchSnapshot { + const payload = cloneDurableValue({ + ...input, + journal: [...input.journal].sort((a, b) => a.sequence - b.sequence), + }) + assertDurableValue(payload, `${input.branchId}: branch snapshot`) + return { + ...payload, + digest: `sha256:${sha256(canonicalJson(stripUndefined(payload)))}`, + } +} + +function assertSnapshotDigest(snapshot: AgentMemoryBranchSnapshot): void { + const { digest, ...payload } = snapshot + const actual = `sha256:${sha256(canonicalJson(stripUndefined(payload)))}` + if (actual !== digest) + throw new Error(`memory branch snapshot digest mismatch: ${digest} != ${actual}`) +} + +function mergeScopes(base?: AgentMemoryScope, extra?: AgentMemoryScope): AgentMemoryScope { + return { + ...(base ?? {}), + ...(extra ?? {}), + tags: { ...(base?.tags ?? {}), ...(extra?.tags ?? {}) }, + } +} + +function scopeMatches(actual: AgentMemoryScope, requested: AgentMemoryScope): boolean { + for (const key of [ + 'tenantId', + 'userId', + 'agentId', + 'teamId', + 'runId', + 'sessionId', + 'namespace', + ] as const) { + if (requested[key] !== undefined && actual[key] !== requested[key]) return false + } + for (const [key, value] of Object.entries(requested.tags ?? {})) { + if (actual.tags?.[key] !== value) return false + } + return true +} + +function stripUndefined(value: unknown): unknown { + if (Array.isArray(value)) return value.map(stripUndefined) + if (typeof value !== 'object' || value === null) return value + return Object.fromEntries( + Object.entries(value) + .filter(([, child]) => child !== undefined) + .map(([key, child]) => [key, stripUndefined(child)]), + ) +} + +function cloneWriteInput(input: AgentMemoryWriteInput): AgentMemoryWriteInput { + return cloneDurableValue(input) +} + +function cloneWriteResult(result: AgentMemoryWriteResult): AgentMemoryWriteResult { + return cloneDurableValue(result) +} + +function cloneJournalEntry(entry: AgentMemoryJournalEntry): AgentMemoryJournalEntry { + return { + sequence: entry.sequence, + input: cloneWriteInput(entry.input), + result: cloneWriteResult(entry.result), + } +} + +function assertDurableValue(value: unknown, label: string, seen = new WeakSet()): void { + if ( + value === null || + typeof value === 'string' || + typeof value === 'boolean' || + (typeof value === 'number' && Number.isFinite(value)) + ) { + return + } + if (value === undefined) return + if (typeof value !== 'object') { + throw new Error(`${label} must contain only JSON-compatible values`) + } + if (seen.has(value)) throw new Error(`${label} must not contain cycles`) + seen.add(value) + if (Array.isArray(value)) { + for (const child of value) { + if (child === undefined) { + throw new Error(`${label} arrays must not contain undefined values`) + } + assertDurableValue(child, label, seen) + } + } else { + const prototype = Object.getPrototypeOf(value) + if (prototype !== Object.prototype && prototype !== null) { + throw new Error(`${label} must contain only plain objects`) + } + for (const child of Object.values(value)) assertDurableValue(child, label, seen) + } + seen.delete(value) +} + +function cloneDurableValue(value: T): T { + if (Array.isArray(value)) return value.map((child) => cloneDurableValue(child)) as T + if (typeof value !== 'object' || value === null) return value + return Object.fromEntries( + Object.entries(value).map(([key, child]) => [key, cloneDurableValue(child)]), + ) as T +} diff --git a/src/memory/experiment.ts b/src/memory/experiment.ts new file mode 100644 index 0000000..a2a9db0 --- /dev/null +++ b/src/memory/experiment.ts @@ -0,0 +1,1852 @@ +import { randomUUID } from 'node:crypto' +import { join } from 'node:path' +import { canonicalJson } from '@tangle-network/agent-eval' +import { + type CampaignResult, + type CampaignStorage, + type CostLedgerHandle, + createRunCostLedger, + type DispatchContext, + fsCampaignStorage, + type JudgeConfig, + resolveRunDir, + runCampaign, + type Scenario, +} from '@tangle-network/agent-eval/campaign' +import { + type KnowledgeBenchmarkFamily, + type KnowledgeBenchmarkSplit, + type KnowledgeMemoryBenchmarkCase, + type KnowledgeMemoryBenchmarkTaskKind, + type KnowledgeMemoryEvent, + type KnowledgeMemoryFactMatcher, + scoreMemoryBenchmarkArtifact, +} from '../benchmarks/index' +import { stableId } from '../ids' +import { + appendAttemptJournalEvent, + assertNoInterruptedPaidCalls, + DEFAULT_MEMORY_RECOVERY_RETRIES_PER_ATTEMPT, + hasSettledPaidCall, + readActiveAttemptJournal, + reconcileInterruptedMemoryPaidCalls, + reserveRecoveryAttempts, +} from './attempt-log' +import { + type AgentMemoryBranch, + type AgentMemoryBranchSnapshot, + type AgentMemorySharingPolicy, + createAgentMemoryBranch, +} from './branch' +import { + createMemoryExecutionPool, + memoryRecoveryDelayMs, + releaseMemoryAdapterCreatedAfterAbort, + resolveMemoryCleanupTimeoutMs, + runBoundedMemoryLifecycle, + sleepForMemoryRecovery, +} from './lifecycle' +import { + type AgentMemoryAcquireRunLease, + type AgentMemoryControllerMode, + type AgentMemoryRunLease, + acquireAgentMemoryRunLease, + type OwnedAgentMemoryRunLease, +} from './run-control' +import type { AgentMemoryAdapter, AgentMemoryScope, AgentMemoryWriteInput } from './types' + +const MEMORY_EXPERIMENT_IMPLEMENTATION_REF = 'agent-knowledge:memory-experiment:v6' + +export interface AgentMemorySequenceProbe { + id: string + query: string + scope?: AgentMemoryScope + limit?: number + taskKind?: KnowledgeMemoryBenchmarkTaskKind + requiredFacts?: readonly KnowledgeMemoryFactMatcher[] + forbiddenFacts?: readonly KnowledgeMemoryFactMatcher[] + expectedEventIds?: readonly string[] + expectedActorIds?: readonly string[] + referenceAnswer?: string +} + +export interface AgentMemorySequenceStep { + id: string + instruction?: string + scope?: AgentMemoryScope + writes?: readonly AgentMemoryWriteInput[] + parallelWrites?: boolean + probes?: readonly AgentMemorySequenceProbe[] + parallelProbes?: boolean + metadata?: Record +} + +export interface AgentMemorySequence { + id: string + family: KnowledgeBenchmarkFamily | string + split?: KnowledgeBenchmarkSplit + steps: readonly AgentMemorySequenceStep[] + /** Exact scopes a runtime callback may write beyond scopes declared by steps. */ + cleanupScopes?: readonly AgentMemoryScope[] + tags?: readonly string[] + metadata?: Record +} + +export interface BuildAgentMemorySequencesFromBenchmarkCasesOptions { + memoryAgentId?: string + eventScope?: (input: { + event: KnowledgeMemoryEvent + case: KnowledgeMemoryBenchmarkCase + eventIndex: number + }) => AgentMemoryScope + probeScope?: (testCase: KnowledgeMemoryBenchmarkCase) => AgentMemoryScope +} + +export interface AgentMemoryExperimentCandidate { + id: string + label?: string + /** Change when provider configuration changes so cached cells cannot be reused. */ + ref: string + /** Local construction is free; call markExternalCall before billable provisioning or reconnects. */ + createAdapter(input: { + branchId: string + sequence: AgentMemorySequence + rep: number + seed: number + purpose: 'execute' | 'recovery' + signal: AbortSignal + markExternalCall(): void + }): AgentMemoryAdapter | null | Promise + policy?: AgentMemorySharingPolicy + baseScope?: AgentMemoryScope + /** Conservative external provider charge for one complete history. */ + externalCostUsdPerSequence?: number + /** Conservative extra provider charge when recovering one interrupted history. */ + externalRecoveryCostUsdPerAttempt?: number + /** Release resources and, when cleanupBranches is false, delete the isolated state. */ + disposeAdapter?(adapter: AgentMemoryAdapter): Promise +} + +export interface AgentMemorySequenceProbeResult { + id: string + stepId: string + query: string + score: number + passed: boolean + dimensions: Record + applicableDimensions: readonly string[] + notes: string + hitIds: readonly string[] +} + +export interface AgentMemorySequenceArtifact { + candidateId: string + sequenceId: string + score: number + passed: boolean + dimensions: Record + dimensionSampleCounts: Record + probes: readonly AgentMemorySequenceProbeResult[] + branchDigest: string + journalEntries: number + durationMs: number +} + +export interface AgentMemorySequenceScenario extends Scenario { + kind: 'agent-memory-sequence' + candidateId: string + sequenceId: string + sequence: AgentMemorySequence + seedGroup: string +} + +export interface AgentMemoryExperimentRankingRow { + rank: number + candidateId: string + label: string + scoreMean: number + passRate: number + totalSequences: number + totalCells: number + totalProbes: number + cellsFailed: number + totalCostUsd: number + durationMs: number + dimensions: Record +} + +export interface RunAgentMemoryExperimentOptions { + experimentId: string + /** Stable external branch namespace; distributed workers must use the same value. */ + experimentRunId?: string + sequences: readonly AgentMemorySequence[] + candidates: readonly AgentMemoryExperimentCandidate[] + /** Retired candidates retained only so interrupted branches can be cleaned on resume. */ + recoveryCandidates?: readonly AgentMemoryExperimentCandidate[] + runDir: string + executeStep?: (input: { + memory: AgentMemoryBranch + candidateId: string + sequence: AgentMemorySequence + step: AgentMemorySequenceStep + context: DispatchContext + }) => Promise + /** Required with executeStep; identify the runtime/profile behavior in cache keys. */ + executeStepRef?: string + onBranchSnapshot?: (input: { + candidateId: string + sequenceId: string + cellId: string + snapshot: AgentMemoryBranchSnapshot + }) => Promise | void + cleanupBranches?: boolean + storage?: CampaignStorage + repo?: string + seed?: number + reps?: number + resumable?: boolean + costCeiling?: number + /** Shared across nested experiments when an outer improvement run owns spend. */ + costLedger?: CostLedgerHandle + costPhase?: string + maxConcurrency?: number + dispatchTimeoutMs?: number + /** Total deadline for each provider cleanup, close, or recovery operation. */ + cleanupTimeoutMs?: number + /** Refuse a damaged run with more unfinished attempts than this. Default 1000. */ + maxRecoveryAttempts?: number + /** Bound repeated provider cleanup after process crashes. Default 3 per attempt. */ + maxRecoveryRetriesPerAttempt?: number + now?: () => Date + /** Required with custom storage when all controllers are confined to one process. */ + controllerMode?: AgentMemoryControllerMode + /** Required for distributed controllers that share custom storage. */ + acquireRunLease?: AgentMemoryAcquireRunLease +} + +export type AgentMemoryExperimentRunLease = AgentMemoryRunLease + +export interface AgentMemoryAttemptEvent { + schema: 2 + status: 'started' | 'cleaned' + branchId: string + candidateId: string + candidateRef: string + sequenceId: string + rep: number + seed: number + cleanupBranches: boolean + externalCostUsdPerSequence: number + externalRecoveryCostUsdPerAttempt: number + recordedAt: string + recovery: boolean +} + +export interface RunAgentMemoryExperimentResult { + campaign: CampaignResult + rows: readonly AgentMemoryExperimentRankingRow[] + totalCostUsd: number + /** Recovery spend for retired candidates, excluded from ranking rows but included in totalCostUsd. */ + unrankedRecoveryCostUsd: number + leaderCandidateId?: string + rankingJsonPath: string + rankingMarkdownPath: string + attemptLogPath: string + recoveryLogPath: string +} + +class AgentMemoryCleanupError extends AggregateError { + constructor(errors: Iterable, message: string) { + super(errors, message) + this.name = 'AgentMemoryCleanupError' + } +} + +type OwnedMemoryExperimentRunLease = OwnedAgentMemoryRunLease + +/** Converts existing ordered memory benchmark cases into executable histories. */ +export function buildAgentMemorySequencesFromBenchmarkCases( + cases: readonly KnowledgeMemoryBenchmarkCase[], + options: BuildAgentMemorySequencesFromBenchmarkCasesOptions = {}, +): AgentMemorySequence[] { + const memoryAgentId = options.memoryAgentId ?? 'benchmark-agent' + return cases.map((testCase) => ({ + id: testCase.id, + family: testCase.family, + ...(testCase.split !== undefined ? { split: testCase.split } : {}), + ...(testCase.tags !== undefined ? { tags: testCase.tags } : {}), + metadata: compactRecord({ + ...(testCase.metadata ?? {}), + taskKind: testCase.taskKind, + source: testCase.source, + }), + steps: [ + ...testCase.events.map((event, eventIndex) => ({ + id: `event:${event.id}`, + scope: compactScope( + options.eventScope?.({ event, case: testCase, eventIndex }) ?? { + agentId: memoryAgentId, + sessionId: testCase.id, + }, + ), + writes: [ + { + id: event.id, + kind: 'observation' as const, + text: event.text, + metadata: compactRecord({ + ...(event.metadata ?? {}), + eventId: event.id, + actorId: event.actorId, + sessionId: event.sessionId, + timestamp: event.timestamp, + }), + }, + ], + metadata: { eventIndex }, + })), + { + id: 'probe', + scope: compactScope( + options.probeScope?.(testCase) ?? { + agentId: memoryAgentId, + sessionId: testCase.id, + }, + ), + probes: [ + { + id: 'answer', + query: testCase.prompt, + taskKind: testCase.taskKind, + ...(testCase.requiredFacts !== undefined + ? { requiredFacts: testCase.requiredFacts } + : {}), + ...(testCase.forbiddenFacts !== undefined + ? { forbiddenFacts: testCase.forbiddenFacts } + : {}), + ...(testCase.expectedEventIds !== undefined + ? { expectedEventIds: testCase.expectedEventIds } + : {}), + ...(testCase.expectedActorIds !== undefined + ? { expectedActorIds: testCase.expectedActorIds } + : {}), + ...(testCase.referenceAnswer !== undefined + ? { referenceAnswer: testCase.referenceAnswer } + : {}), + }, + ], + }, + ], + })) +} + +/** Runs ordered, branch-isolated memory histories across candidate systems. */ +export async function runAgentMemoryExperiment( + options: RunAgentMemoryExperimentOptions, +): Promise { + assertNonEmptyString(options.experimentId, 'memory experiment experimentId') + assertNonEmptyString(options.runDir, 'memory experiment runDir') + if (options.experimentRunId !== undefined) { + assertNonEmptyString(options.experimentRunId, 'memory experiment experimentRunId') + } + if (options.sequences.length === 0) throw new Error('memory experiment requires sequences') + if (options.candidates.length === 0) throw new Error('memory experiment requires candidates') + if (options.executeStep && !options.executeStepRef) { + throw new Error('memory experiment executeStepRef is required when executeStep is configured') + } + assertUnique( + options.sequences.map((sequence) => sequence.id), + 'sequence', + ) + assertUnique( + [...options.candidates, ...(options.recoveryCandidates ?? [])].map((candidate) => candidate.id), + 'candidate', + ) + assertMemorySequences(options.sequences) + for (const candidate of [...options.candidates, ...(options.recoveryCandidates ?? [])]) { + if (options.cleanupBranches === false && !candidate.disposeAdapter) { + throw new Error( + `${candidate.id}: cleanupBranches=false requires disposeAdapter to delete isolated external state`, + ) + } + if ( + candidate.externalCostUsdPerSequence !== undefined && + (!Number.isFinite(candidate.externalCostUsdPerSequence) || + candidate.externalCostUsdPerSequence < 0) + ) { + throw new Error( + `${candidate.id}: externalCostUsdPerSequence must be a non-negative finite number`, + ) + } + if ( + candidate.externalRecoveryCostUsdPerAttempt !== undefined && + (!Number.isFinite(candidate.externalRecoveryCostUsdPerAttempt) || + candidate.externalRecoveryCostUsdPerAttempt < 0) + ) { + throw new Error( + `${candidate.id}: externalRecoveryCostUsdPerAttempt must be a non-negative finite number`, + ) + } + assertNonEmptyString(candidate.ref, `${candidate.id} ref`) + } + + const storage = options.storage ?? fsCampaignStorage() + const runDir = resolveRunDir(options.runDir, options.repo) + if (!storage.append) { + throw new Error('memory experiment requires CampaignStorage.append for durable attempt state') + } + resolveMemoryCleanupTimeoutMs(options.cleanupTimeoutMs, 'memory experiment') + const maxRecoveryAttempts = options.maxRecoveryAttempts ?? 1_000 + if (!Number.isSafeInteger(maxRecoveryAttempts) || maxRecoveryAttempts <= 0) { + throw new Error('memory experiment maxRecoveryAttempts must be a positive safe integer') + } + const maxRecoveryRetriesPerAttempt = + options.maxRecoveryRetriesPerAttempt ?? DEFAULT_MEMORY_RECOVERY_RETRIES_PER_ATTEMPT + if (!Number.isSafeInteger(maxRecoveryRetriesPerAttempt) || maxRecoveryRetriesPerAttempt <= 0) { + throw new Error( + 'memory experiment maxRecoveryRetriesPerAttempt must be a positive safe integer', + ) + } + storage.ensureDir(runDir) + const lease = await acquireAgentMemoryRunLease({ + experimentId: options.experimentId, + runDir, + storage, + customStorage: options.storage !== undefined, + lockFileName: 'memory-experiment.lock', + label: 'memory experiment', + controllerMode: options.controllerMode, + acquireRunLease: options.acquireRunLease, + }) + let result: RunAgentMemoryExperimentResult | undefined + let primaryError: unknown + try { + await lease.assertOwned() + result = await runOwnedAgentMemoryExperiment(options, storage, runDir, lease) + } catch (error) { + primaryError = error + } + let releaseError: unknown + try { + await lease.release() + } catch (error) { + releaseError = error + } + if (primaryError && releaseError) { + throw new AggregateError( + [primaryError, releaseError], + 'memory experiment failed and its controller lease could not be released', + ) + } + if (primaryError) throw primaryError + if (releaseError) throw releaseError + if (!result) throw new Error('memory experiment produced no result') + return result +} + +async function runOwnedAgentMemoryExperiment( + options: RunAgentMemoryExperimentOptions, + storage: CampaignStorage, + runDir: string, + lease: OwnedMemoryExperimentRunLease, +): Promise { + const runIdentity = stableId( + 'memory_run', + canonicalJson({ + experimentId: options.experimentId, + experimentRunId: options.experimentRunId ?? runDir, + }), + ) + const maxConcurrency = options.maxConcurrency ?? 2 + if (!Number.isSafeInteger(maxConcurrency) || maxConcurrency <= 0) { + throw new Error('memory experiment maxConcurrency must be a positive safe integer') + } + const executionPool = createMemoryExecutionPool(maxConcurrency) + const dispatchedExecutions: Promise[] = [] + const candidateById = new Map(options.candidates.map((candidate) => [candidate.id, candidate])) + const recoveryCandidateById = new Map( + [...options.candidates, ...(options.recoveryCandidates ?? [])].map((candidate) => [ + candidate.id, + candidate, + ]), + ) + const sequenceById = new Map(options.sequences.map((sequence) => [sequence.id, sequence])) + const scenarios = buildAgentMemorySequenceScenarios(options.sequences, options.candidates) + const attemptLogPath = join(runDir, 'memory-attempts.jsonl') + const recoveryLogPath = join(runDir, 'memory-recovery-attempts.jsonl') + const costCeiling = options.costCeiling ?? options.costLedger?.costCeilingUsd ?? 0 + const costLedger = + options.costLedger ?? + createRunCostLedger({ + storage, + runDir, + costCeilingUsd: costCeiling, + }) + if (costLedger.costCeilingUsd !== costCeiling) { + throw new Error('memory experiment costCeiling must match the shared cost ledger ceiling') + } + storage.ensureDir(runDir) + await recoverAbandonedMemoryAttempts({ + options, + storage, + runDir, + attemptLogPath, + candidateById: recoveryCandidateById, + sequenceById, + lease, + maxConcurrency, + costLedger, + maxRecoveryAttempts: options.maxRecoveryAttempts ?? 1_000, + recoveryLogPath, + maxRecoveryRetriesPerAttempt: + options.maxRecoveryRetriesPerAttempt ?? DEFAULT_MEMORY_RECOVERY_RETRIES_PER_ATTEMPT, + }) + await lease.assertOwned() + let campaign: CampaignResult | undefined + let campaignError: unknown + let settledExecutions: PromiseSettledResult[] = [] + try { + campaign = await runCampaign({ + scenarios, + dispatch: (scenario, context) => { + const candidate = candidateById.get(scenario.candidateId) + if (!candidate) throw new Error(`unknown memory candidate ${scenario.candidateId}`) + const operation = executionPool.run(() => + runSequenceCell({ + options, + candidate, + scenario, + context, + runIdentity, + storage, + attemptLogPath, + lease, + }), + ) + dispatchedExecutions.push(operation) + return operation + }, + dispatchRef: memoryExperimentDispatchRef(options), + judges: [agentMemorySequenceJudge()], + runDir, + storage, + seed: options.seed, + reps: options.reps, + resumable: options.resumable, + costCeiling, + costLedger, + costPhase: options.costPhase, + maxConcurrency, + dispatchTimeoutMs: options.dispatchTimeoutMs, + expectUsage: 'off', + now: options.now, + }) + } catch (error) { + campaignError = error + } finally { + settledExecutions = await Promise.allSettled(dispatchedExecutions) + } + const cleanupFailures = settledExecutions.flatMap((settled) => + settled.status === 'rejected' && settled.reason instanceof AgentMemoryCleanupError + ? [settled.reason] + : [], + ) + if (campaignError && cleanupFailures.length > 0) { + throw new AggregateError( + [campaignError, ...cleanupFailures], + 'memory experiment failed and provider cleanup also failed', + ) + } + if (campaignError) throw campaignError + if (cleanupFailures.length > 0) { + throw new AggregateError(cleanupFailures, 'memory experiment cleanup failed after dispatch') + } + if (!campaign) throw new Error('memory experiment produced no campaign result') + await lease.assertOwned() + const costByCandidate = memoryExperimentCostByCandidate( + costLedger, + campaign.runDir, + scenarios, + [...options.candidates, ...(options.recoveryCandidates ?? [])].map((candidate) => candidate.id), + ) + const unrankedRecoveryCostUsd = normalizeUsd( + (options.recoveryCandidates ?? []).reduce( + (sum, candidate) => sum + (costByCandidate.get(candidate.id) ?? 0), + 0, + ), + ) + const totalCostUsd = normalizeUsd( + [...costByCandidate.values()].reduce((sum, cost) => sum + cost, 0), + ) + const rows = rankAgentMemoryExperiment(options.candidates, scenarios, campaign, costByCandidate) + const rankingJsonPath = join(campaign.runDir, 'memory-experiment-ranking.json') + const rankingMarkdownPath = join(campaign.runDir, 'memory-experiment-ranking.md') + storage.write( + rankingJsonPath, + `${JSON.stringify( + { experimentId: options.experimentId, totalCostUsd, unrankedRecoveryCostUsd, rows }, + null, + 2, + )}\n`, + ) + storage.write( + rankingMarkdownPath, + renderAgentMemoryExperimentRanking(rows, totalCostUsd, unrankedRecoveryCostUsd), + ) + return { + campaign, + rows, + totalCostUsd, + unrankedRecoveryCostUsd, + leaderCandidateId: rows.find((row) => row.cellsFailed === 0)?.candidateId, + rankingJsonPath, + rankingMarkdownPath, + attemptLogPath, + recoveryLogPath, + } +} + +export function buildAgentMemorySequenceScenarios( + sequences: readonly AgentMemorySequence[], + candidates: readonly Pick[], +): AgentMemorySequenceScenario[] { + return candidates.flatMap((candidate) => + sequences.map((sequence) => ({ + id: `${stableId('candidate', candidate.id)}:${sequence.id}`, + kind: 'agent-memory-sequence' as const, + candidateId: candidate.id, + sequenceId: sequence.id, + sequence, + seedGroup: sequence.id, + tags: [...new Set([sequence.split ?? 'dev', ...(sequence.tags ?? []), candidate.id])], + })), + ) +} + +export function agentMemorySequenceJudge(): JudgeConfig< + AgentMemorySequenceArtifact, + AgentMemorySequenceScenario +> { + return { + name: 'agent-memory-sequence', + judgeVersion: 'agent-knowledge:memory-sequence:v2', + dimensions: [ + { key: 'score', description: 'mean memory probe score' }, + { key: 'passed', description: '1 when every memory probe passes' }, + { key: 'memory_fact_recall', description: 'current memory fact coverage' }, + { key: 'memory_event_recall', description: 'memory source event coverage' }, + { key: 'memory_actor_recall', description: 'memory actor attribution coverage' }, + { key: 'memory_stale_safe', description: '1 when obsolete memory is not reused' }, + ], + score({ artifact }) { + return { + composite: artifact.score, + dimensions: { + score: artifact.score, + passed: artifact.passed ? 1 : 0, + ...artifact.dimensions, + }, + notes: `${artifact.probes.filter((probe) => probe.passed).length}/${artifact.probes.length} probes passed`, + } + }, + } +} + +async function runSequenceCell(input: { + options: RunAgentMemoryExperimentOptions + candidate: AgentMemoryExperimentCandidate + scenario: AgentMemorySequenceScenario + context: DispatchContext + runIdentity: string + storage: CampaignStorage + attemptLogPath: string + lease: OwnedMemoryExperimentRunLease +}): Promise { + const { options, candidate, scenario, context, runIdentity, storage, attemptLogPath, lease } = + input + const cleanupBranches = options.cleanupBranches ?? true + const costUsd = candidate.externalCostUsdPerSequence ?? 0 + if (!Number.isFinite(costUsd) || costUsd < 0) { + throw new Error( + `${candidate.id}: externalCostUsdPerSequence must be a non-negative finite number`, + ) + } + if (!cleanupBranches && !candidate.disposeAdapter) { + throw new Error( + `${candidate.id}: cleanupBranches=false requires disposeAdapter to delete isolated external state`, + ) + } + const cleanupTimeoutMs = resolveMemoryCleanupTimeoutMs( + options.cleanupTimeoutMs, + `${candidate.id}: memory sequence`, + ) + + context.signal.throwIfAborted() + await lease.assertOwned() + const startedAt = Date.now() + const branchId = stableId( + 'memory_branch', + `${runIdentity}:${context.cellId}:${context.seed}:${randomUUID()}`, + ) + const attempt = memoryAttemptEvent({ + status: 'started', + branchId, + candidate, + sequence: scenario.sequence, + rep: context.rep, + seed: context.seed, + recovery: false, + cleanupBranches, + now: options.now, + }) + appendMemoryAttemptEvent(storage, attemptLogPath, attempt) + + let externalCallAttempted = false + const appendCleanedAttempt = (priorError?: unknown): void => { + try { + appendMemoryAttemptEvent(storage, attemptLogPath, { + ...attempt, + status: 'cleaned', + recordedAt: (options.now ?? (() => new Date()))().toISOString(), + }) + } catch (error) { + throw new AgentMemoryCleanupError( + [...(priorError ? [priorError] : []), error], + `${candidate.id}: memory branch cleanup could not be recorded`, + ) + } + } + const execute = async (): Promise => { + context.signal.throwIfAborted() + await lease.assertOwned() + let rawAdapter: AgentMemoryAdapter | undefined + let adapter: AgentMemoryAdapter | undefined + let memory: AgentMemoryBranch | undefined + let primaryError: unknown + let completedArtifact: AgentMemorySequenceArtifact | undefined + let finalClearStarted = false + let finalClearCompleted = false + try { + const created = await candidate.createAdapter({ + branchId, + sequence: scenario.sequence, + rep: context.rep, + seed: context.seed, + purpose: 'execute', + signal: context.signal, + markExternalCall: () => { + externalCallAttempted = true + }, + }) + if (!created) throw new Error(`${candidate.id}: createAdapter returned no execution adapter`) + rawAdapter = created + adapter = trackExternalMemoryCalls(created, () => { + externalCallAttempted = true + }) + await lease.assertOwned() + context.signal.throwIfAborted() + if (cleanupBranches && !adapter.clear) { + throw new Error( + `${candidate.id}: cleanupBranches requires an adapter with scoped clear support`, + ) + } + memory = createAgentMemoryBranch({ + adapter, + branchId, + lifetime: 'attempt', + policy: candidate.policy, + allowedWriteScopes: sequenceCleanupScopes(scenario.sequence), + baseScope: memoryExperimentBaseScope(options, candidate, scenario.sequenceId), + }) + const probes: AgentMemorySequenceProbeResult[] = [] + for (const step of scenario.sequence.steps) { + context.signal.throwIfAborted() + await lease.assertOwned() + await writeStep(memory, step) + await lease.assertOwned() + await options.executeStep?.({ + memory, + candidateId: candidate.id, + sequence: scenario.sequence, + step, + context, + }) + context.signal.throwIfAborted() + await lease.assertOwned() + const stepProbes = await probeStep(memory, scenario.sequence, step) + await lease.assertOwned() + probes.push(...stepProbes) + } + const snapshot = await memory.snapshot() + await lease.assertOwned() + await options.onBranchSnapshot?.({ + candidateId: candidate.id, + sequenceId: scenario.sequenceId, + cellId: context.cellId, + snapshot, + }) + await lease.assertOwned() + const dimensions = meanDimensions(probes.map((probe) => probe.dimensions)) + const dimensionSampleCounts = countDimensions( + probes.map((probe) => probe.applicableDimensions), + ) + const artifact: AgentMemorySequenceArtifact = { + candidateId: candidate.id, + sequenceId: scenario.sequenceId, + score: mean(probes.map((probe) => probe.score)), + passed: probes.length > 0 && probes.every((probe) => probe.passed), + dimensions, + dimensionSampleCounts, + probes, + branchDigest: snapshot.digest, + journalEntries: snapshot.journal.length, + durationMs: Math.max(0, Date.now() - startedAt), + } + if (cleanupBranches) { + finalClearStarted = true + await runBoundedMemoryLifecycle({ + operation: `${candidate.id}: final branch cleanup`, + timeoutMs: cleanupTimeoutMs, + resource: adapter, + run: () => clearSequenceScopes(memory!, scenario.sequence), + }) + finalClearCompleted = true + await lease.assertOwned() + } + completedArtifact = artifact + } catch (error) { + primaryError = error + } + const cleanupErrors: unknown[] = [] + let cleanupOwned = true + let ownershipError: unknown + try { + await lease.assertOwned() + } catch (error) { + cleanupOwned = false + ownershipError = error + } + if (finalClearStarted && !finalClearCompleted && primaryError) { + cleanupErrors.push(primaryError) + } + if ( + primaryError && + cleanupOwned && + !finalClearStarted && + memory && + cleanupBranches && + adapter?.clear + ) { + try { + await runBoundedMemoryLifecycle({ + operation: `${candidate.id}: failed branch cleanup`, + timeoutMs: cleanupTimeoutMs, + resource: adapter, + run: () => clearSequenceScopes(memory!, scenario.sequence), + }) + } catch (error) { + cleanupErrors.push(error) + } + } + if (adapter) { + try { + await runBoundedMemoryLifecycle({ + operation: `${candidate.id}: adapter close`, + timeoutMs: cleanupTimeoutMs, + resource: adapter, + run: async () => { + if (memory && cleanupOwned) await memory.close?.() + else { + if (cleanupOwned) await adapter!.flush?.() + await adapter!.close?.() + } + }, + }) + } catch (error) { + cleanupErrors.push(error) + } + if (cleanupOwned) { + try { + await runBoundedMemoryLifecycle({ + operation: `${candidate.id}: adapter disposal`, + timeoutMs: cleanupTimeoutMs, + resource: adapter, + run: () => { + if (candidate.disposeAdapter) externalCallAttempted = true + return candidate.disposeAdapter?.(rawAdapter!) + }, + }) + } catch (error) { + cleanupErrors.push(error) + } + } + } else { + cleanupErrors.push( + new Error(`${candidate.id}: adapter creation failed before cleanup could be confirmed`), + ) + } + if (cleanupOwned && cleanupErrors.length === 0) appendCleanedAttempt(primaryError) + if (!cleanupOwned && cleanupErrors.length === 0) { + if (primaryError) throw primaryError + throw ownershipError + } + if (cleanupErrors.length > 0) { + const primaryMessage = + primaryError instanceof Error ? primaryError.message : String(primaryError ?? '') + throw new AgentMemoryCleanupError( + [ + ...(primaryError && !cleanupErrors.includes(primaryError) ? [primaryError] : []), + ...(ownershipError ? [ownershipError] : []), + ...cleanupErrors, + ], + `${candidate.id}: memory branch cleanup failed${primaryMessage ? ` after: ${primaryMessage}` : ''}`, + ) + } + if (primaryError) throw primaryError + if (!completedArtifact) throw new Error(`${candidate.id}: memory sequence produced no result`) + return completedArtifact + } + + if (costUsd === 0) { + let artifact: AgentMemorySequenceArtifact | undefined + let error: unknown + try { + artifact = await execute() + } catch (caught) { + error = caught + } + if (error) throw error + if (!artifact) throw new Error(`${candidate.id}: memory sequence produced no result`) + return artifact + } + const receipt = { + model: candidate.id, + inputTokens: 0, + outputTokens: 0, + actualCostUsd: costUsd, + } as const + const paid = await context.cost.runPaidCall({ + callId: memoryAttemptCostCallId(attempt, 'execute', 0), + actor: `agent-knowledge:memory-experiment:${candidate.id}`, + model: candidate.id, + maximumCharge: { externallyEnforcedMaximumUsd: costUsd }, + execute, + receipt: () => receipt, + receiptFromError: () => ({ + ...receipt, + actualCostUsd: externalCallAttempted ? costUsd : 0, + }), + }) + if (!paid.succeeded) throw paid.error + return paid.value +} + +function trackExternalMemoryCalls( + adapter: AgentMemoryAdapter, + onExternalCall: () => void, +): AgentMemoryAdapter { + return { + id: adapter.id, + branchIsolation: adapter.branchIsolation, + search(query, options) { + onExternalCall() + return adapter.search.call(adapter, query, options) + }, + getContext(query, options) { + onExternalCall() + return adapter.getContext.call(adapter, query, options) + }, + write(input) { + onExternalCall() + return adapter.write.call(adapter, input) + }, + ...(adapter.clear + ? { + clear(scope?: AgentMemoryScope) { + onExternalCall() + return adapter.clear!.call(adapter, scope) + }, + } + : {}), + ...(adapter.flush + ? { + flush() { + onExternalCall() + return adapter.flush!.call(adapter) + }, + } + : {}), + ...(adapter.close + ? { + close() { + onExternalCall() + return adapter.close!.call(adapter) + }, + } + : {}), + } +} + +async function recoverAbandonedMemoryAttempts(input: { + options: RunAgentMemoryExperimentOptions + storage: CampaignStorage + runDir: string + attemptLogPath: string + candidateById: ReadonlyMap + sequenceById: ReadonlyMap + lease: OwnedMemoryExperimentRunLease + maxConcurrency: number + costLedger: CostLedgerHandle + maxRecoveryAttempts: number + recoveryLogPath: string + maxRecoveryRetriesPerAttempt: number +}): Promise { + let attempts = readActiveMemoryAttempts(input.storage, input.attemptLogPath) + if (attempts.length > input.maxRecoveryAttempts) { + throw new Error( + `memory experiment has ${attempts.length} unfinished attempts; maxRecoveryAttempts is ${input.maxRecoveryAttempts}`, + ) + } + for (const attempt of attempts) { + const candidate = input.candidateById.get(attempt.candidateId) + if (!candidate) { + throw new Error( + `cannot recover memory branch '${attempt.branchId}': candidate '${attempt.candidateId}' is missing; pass it in recoveryCandidates`, + ) + } + assertMemoryAttemptCandidateMatches(attempt, candidate) + if (!input.sequenceById.has(attempt.sequenceId)) { + throw new Error( + `cannot recover memory branch '${attempt.branchId}': sequence '${attempt.sequenceId}' is missing`, + ) + } + if ((input.options.cleanupBranches ?? true) !== attempt.cleanupBranches) { + throw new Error(`cannot recover memory branch '${attempt.branchId}': cleanupBranches changed`) + } + } + + reconcileInterruptedMemoryPaidCalls(input.costLedger) + assertNoInterruptedPaidCalls(input.costLedger, 'memory experiment recovery') + + for (const attempt of attempts) { + const candidate = input.candidateById.get(attempt.candidateId)! + const executionCostUsd = candidate.externalCostUsdPerSequence ?? 0 + if ( + executionCostUsd > 0 && + !hasSettledPaidCall(input.costLedger, memoryAttemptCostCallId(attempt, 'execute', 0)) + ) { + appendMemoryAttemptEvent(input.storage, input.attemptLogPath, { + ...attempt, + status: 'cleaned', + recovery: true, + recordedAt: (input.options.now ?? (() => new Date()))().toISOString(), + }) + } + } + attempts = readActiveMemoryAttempts(input.storage, input.attemptLogPath) + const recoveryGenerations = reserveRecoveryAttempts({ + storage: input.storage, + path: input.recoveryLogPath, + attemptIds: attempts.map((attempt) => attempt.branchId), + maxRetriesPerAttempt: input.maxRecoveryRetriesPerAttempt, + label: 'memory recovery attempt log', + now: input.options.now, + }) + const pool = createMemoryExecutionPool(input.maxConcurrency) + const settled = await Promise.allSettled( + attempts + .sort((left, right) => left.branchId.localeCompare(right.branchId)) + .map((attempt) => + pool.run(async () => { + await input.lease.assertOwned() + const candidate = input.candidateById.get(attempt.candidateId) + if (!candidate) { + throw new Error( + `cannot recover memory branch '${attempt.branchId}': candidate '${attempt.candidateId}' is missing; pass it in recoveryCandidates`, + ) + } + assertMemoryAttemptCandidateMatches(attempt, candidate) + const sequence = input.sequenceById.get(attempt.sequenceId) + if (!sequence) { + throw new Error( + `cannot recover memory branch '${attempt.branchId}': sequence '${attempt.sequenceId}' is missing`, + ) + } + if ((input.options.cleanupBranches ?? true) !== attempt.cleanupBranches) { + throw new Error( + `cannot recover memory branch '${attempt.branchId}': cleanupBranches changed`, + ) + } + const recoveryCostUsd = candidate.externalRecoveryCostUsdPerAttempt ?? 0 + const recoveryGeneration = recoveryGenerations.get(attempt.branchId) + if (recoveryGeneration === undefined) { + throw new Error(`missing recovery generation for memory branch '${attempt.branchId}'`) + } + let externalRecoveryAttempted = false + const recover = async (): Promise => { + await recoverMemoryAttempt({ + options: input.options, + candidate, + sequence, + attempt, + lease: input.lease, + onExternalCall: () => { + externalRecoveryAttempted = true + }, + }) + appendMemoryAttemptEvent(input.storage, input.attemptLogPath, { + ...attempt, + status: 'cleaned', + recovery: true, + recordedAt: (input.options.now ?? (() => new Date()))().toISOString(), + }) + } + if (recoveryCostUsd === 0) { + await recover() + } else { + const tags = memoryRecoveryCostTags(input.runDir, candidate.id, attempt.branchId) + const receipt = { + model: candidate.id, + inputTokens: 0, + outputTokens: 0, + actualCostUsd: recoveryCostUsd, + } as const + const paid = await input.costLedger.runPaidCall({ + callId: memoryAttemptCostCallId(attempt, 'recovery', recoveryGeneration), + channel: 'driver', + phase: `${input.options.costPhase ?? 'memory.experiment'}.recovery`, + actor: `agent-knowledge:memory-recovery:${candidate.id}`, + model: candidate.id, + tags, + maximumCharge: { externallyEnforcedMaximumUsd: recoveryCostUsd }, + execute: recover, + receipt: () => ({ + ...receipt, + actualCostUsd: externalRecoveryAttempted ? recoveryCostUsd : 0, + }), + receiptFromError: () => ({ + ...receipt, + actualCostUsd: externalRecoveryAttempted ? recoveryCostUsd : 0, + }), + }) + if (!paid.succeeded) throw paid.error + } + }), + ), + ) + const failures = settled.flatMap((result) => + result.status === 'rejected' ? [result.reason] : [], + ) + if (failures.length === 1) throw failures[0] + if (failures.length > 1) { + throw new AggregateError(failures, 'multiple abandoned memory branches failed recovery') + } +} + +async function recoverMemoryAttempt(input: { + options: RunAgentMemoryExperimentOptions + candidate: AgentMemoryExperimentCandidate + sequence: AgentMemorySequence + attempt: AgentMemoryAttemptEvent + lease: OwnedMemoryExperimentRunLease + onExternalCall(): void +}): Promise { + const { options, candidate, sequence, attempt, lease, onExternalCall } = input + const cleanupBranches = attempt.cleanupBranches + const cleanupTimeoutMs = resolveMemoryCleanupTimeoutMs( + options.cleanupTimeoutMs, + `${candidate.id}: abandoned branch recovery`, + ) + let rawAdapter: AgentMemoryAdapter | undefined + let adapter: AgentMemoryAdapter | undefined + let memory: AgentMemoryBranch | undefined + let primaryError: unknown + try { + const abortController = new AbortController() + const creation = Promise.resolve().then(() => + candidate.createAdapter({ + branchId: attempt.branchId, + sequence, + rep: attempt.rep, + seed: attempt.seed, + purpose: 'recovery', + signal: abortController.signal, + markExternalCall: onExternalCall, + }), + ) + releaseMemoryAdapterCreatedAfterAbort({ + creation, + signal: abortController.signal, + dispose: candidate.disposeAdapter + ? async (created) => { + onExternalCall() + await candidate.disposeAdapter?.(created) + } + : undefined, + }) + const recovered = await runBoundedMemoryLifecycle({ + operation: `${candidate.id}: recovery adapter creation`, + timeoutMs: cleanupTimeoutMs, + abortController, + run: () => creation, + }) + await lease.assertOwned() + if (recovered === null) return + rawAdapter = recovered + adapter = trackExternalMemoryCalls(recovered, onExternalCall) + const recoveryDelayMs = memoryRecoveryDelayMs(adapter) + await sleepForMemoryRecovery( + recoveryDelayMs, + () => lease.assertOwned(), + cleanupTimeoutMs, + `${candidate.id}: abandoned branch recovery visibility wait`, + ) + if (cleanupBranches) { + if (!adapter.clear) { + throw new Error( + `${candidate.id}: abandoned branch recovery requires an adapter with scoped clear support`, + ) + } + memory = createAgentMemoryBranch({ + adapter, + branchId: attempt.branchId, + lifetime: 'attempt', + policy: candidate.policy, + allowedWriteScopes: sequenceCleanupScopes(sequence), + baseScope: memoryExperimentBaseScope(options, candidate, sequence.id), + }) + await runBoundedMemoryLifecycle({ + operation: `${candidate.id}: abandoned branch cleanup`, + timeoutMs: cleanupTimeoutMs, + resource: adapter, + run: () => clearSequenceScopes(memory!, sequence), + }) + await lease.assertOwned() + } + } catch (error) { + primaryError = error + } + + const cleanupErrors: unknown[] = [] + let cleanupOwned = true + try { + await lease.assertOwned() + } catch (error) { + cleanupOwned = false + cleanupErrors.push(error) + } + if (adapter) { + try { + await runBoundedMemoryLifecycle({ + operation: `${candidate.id}: recovery adapter close`, + timeoutMs: cleanupTimeoutMs, + resource: adapter, + run: async () => { + if (memory && cleanupOwned) { + await memory.close?.() + } else { + if (cleanupOwned && adapter!.flush) { + await adapter!.flush() + } + await adapter!.close?.() + } + }, + }) + } catch (error) { + cleanupErrors.push(error) + } + if (cleanupOwned) { + try { + if (candidate.disposeAdapter) onExternalCall() + await runBoundedMemoryLifecycle({ + operation: `${candidate.id}: recovery adapter disposal`, + timeoutMs: cleanupTimeoutMs, + resource: adapter, + run: () => candidate.disposeAdapter?.(rawAdapter!), + }) + } catch (error) { + cleanupErrors.push(error) + } + } + } else { + cleanupErrors.push( + new Error( + `${candidate.id}: recovery adapter creation failed before cleanup could be confirmed`, + ), + ) + } + if (!cleanupOwned || primaryError || cleanupErrors.length > 0) { + throw new AgentMemoryCleanupError( + [...(primaryError ? [primaryError] : []), ...cleanupErrors], + `${candidate.id}: abandoned memory branch '${attempt.branchId}' recovery failed`, + ) + } +} + +function memoryAttemptCostCallId( + attempt: AgentMemoryAttemptEvent, + purpose: 'execute' | 'recovery', + generation: number, +): string { + return stableId( + 'memory_cost_call', + canonicalJson({ + purpose, + generation, + branchId: attempt.branchId, + candidateId: attempt.candidateId, + candidateRef: attempt.candidateRef, + externalCostUsdPerSequence: attempt.externalCostUsdPerSequence, + externalRecoveryCostUsdPerAttempt: attempt.externalRecoveryCostUsdPerAttempt, + sequenceId: attempt.sequenceId, + rep: attempt.rep, + seed: attempt.seed, + }), + ) +} + +function memoryRecoveryCostTags( + runDir: string, + candidateId: string, + branchId: string, +): Record { + return { + runDir, + candidateId, + branchId, + memoryRecovery: 'attempt', + } +} + +function memoryAttemptEvent(input: { + status: AgentMemoryAttemptEvent['status'] + branchId: string + candidate: AgentMemoryExperimentCandidate + sequence: AgentMemorySequence + rep: number + seed: number + recovery: boolean + cleanupBranches?: boolean + now?: () => Date +}): AgentMemoryAttemptEvent { + return { + schema: 2, + status: input.status, + branchId: input.branchId, + candidateId: input.candidate.id, + candidateRef: input.candidate.ref, + sequenceId: input.sequence.id, + rep: input.rep, + seed: input.seed, + cleanupBranches: input.cleanupBranches ?? true, + externalCostUsdPerSequence: input.candidate.externalCostUsdPerSequence ?? 0, + externalRecoveryCostUsdPerAttempt: input.candidate.externalRecoveryCostUsdPerAttempt ?? 0, + recordedAt: (input.now ?? (() => new Date()))().toISOString(), + recovery: input.recovery, + } +} + +function appendMemoryAttemptEvent( + storage: CampaignStorage, + path: string, + event: AgentMemoryAttemptEvent, +): void { + appendAttemptJournalEvent({ + storage, + path, + event, + label: 'memory attempt log', + }) +} + +function readActiveMemoryAttempts( + storage: CampaignStorage, + path: string, +): AgentMemoryAttemptEvent[] { + return readActiveAttemptJournal({ + storage, + path, + label: 'memory attempt log', + parse: parseMemoryAttemptEvent, + id: (event) => event.branchId, + sameAttempt: sameMemoryAttempt, + }) +} + +function parseMemoryAttemptEvent( + value: unknown, + path: string, + line: number, +): AgentMemoryAttemptEvent { + const event = value as Partial | null + const valid = + typeof event === 'object' && + event !== null && + event.schema === 2 && + (event.status === 'started' || event.status === 'cleaned') && + typeof event.branchId === 'string' && + event.branchId.length > 0 && + typeof event.candidateId === 'string' && + event.candidateId.length > 0 && + typeof event.candidateRef === 'string' && + event.candidateRef.length > 0 && + typeof event.sequenceId === 'string' && + event.sequenceId.length > 0 && + Number.isSafeInteger(event.rep) && + Number.isSafeInteger(event.seed) && + typeof event.cleanupBranches === 'boolean' && + typeof event.externalCostUsdPerSequence === 'number' && + Number.isFinite(event.externalCostUsdPerSequence) && + event.externalCostUsdPerSequence >= 0 && + typeof event.externalRecoveryCostUsdPerAttempt === 'number' && + Number.isFinite(event.externalRecoveryCostUsdPerAttempt) && + event.externalRecoveryCostUsdPerAttempt >= 0 && + typeof event.recordedAt === 'string' && + !Number.isNaN(Date.parse(event.recordedAt)) && + typeof event.recovery === 'boolean' + if (!valid) throw new Error(`invalid memory attempt event in '${path}' line ${line}`) + return value as AgentMemoryAttemptEvent +} + +function sameMemoryAttempt(left: AgentMemoryAttemptEvent, right: AgentMemoryAttemptEvent): boolean { + return ( + left.branchId === right.branchId && + left.candidateId === right.candidateId && + left.candidateRef === right.candidateRef && + left.sequenceId === right.sequenceId && + left.rep === right.rep && + left.seed === right.seed && + left.cleanupBranches === right.cleanupBranches && + left.externalCostUsdPerSequence === right.externalCostUsdPerSequence && + left.externalRecoveryCostUsdPerAttempt === right.externalRecoveryCostUsdPerAttempt + ) +} + +function assertMemoryAttemptCandidateMatches( + attempt: AgentMemoryAttemptEvent, + candidate: AgentMemoryExperimentCandidate, +): void { + if (candidate.ref !== attempt.candidateRef) { + throw new Error( + `cannot recover memory branch '${attempt.branchId}': candidate ref changed from '${attempt.candidateRef}' to '${candidate.ref}'`, + ) + } + const executionCost = candidate.externalCostUsdPerSequence ?? 0 + const recoveryCost = candidate.externalRecoveryCostUsdPerAttempt ?? 0 + if ( + executionCost !== attempt.externalCostUsdPerSequence || + recoveryCost !== attempt.externalRecoveryCostUsdPerAttempt + ) { + throw new Error( + `cannot recover memory branch '${attempt.branchId}': candidate cost settings changed; start a new run or restore the recorded costs`, + ) + } +} + +async function writeStep(memory: AgentMemoryBranch, step: AgentMemorySequenceStep): Promise { + const writes = (step.writes ?? []).map((write) => ({ + ...write, + scope: mergeScopes(step.scope, write.scope), + })) + if (step.parallelWrites) { + await Promise.all(writes.map((write) => memory.write(write))) + return + } + for (const write of writes) await memory.write(write) +} + +async function probeStep( + memory: AgentMemoryBranch, + sequence: AgentMemorySequence, + step: AgentMemorySequenceStep, +): Promise { + const run = async (probe: AgentMemorySequenceProbe): Promise => { + const scope = mergeScopes(step.scope, probe.scope) + const context = await memory.getContext(probe.query, { scope, limit: probe.limit }) + const artifact = { + answer: context.text, + rememberedFacts: context.hits.map((hit) => hit.text), + citedEventIds: uniqueStrings([ + ...context.hits.map((hit) => hit.metadata?.eventId), + ...context.hits.flatMap((hit) => stringArray(hit.metadata?.eventIds)), + ]), + usedMemoryIds: context.hits.map((hit) => hit.id), + actorIds: uniqueStrings([ + ...context.hits.map((hit) => hit.metadata?.actorId), + ...context.hits.flatMap((hit) => stringArray(hit.metadata?.actorIds)), + ]), + } + const evaluation = scoreMemoryBenchmarkArtifact( + { + id: `${sequence.id}:${step.id}:${probe.id}`, + family: sequence.family, + taskKind: probe.taskKind ?? 'memory-recall', + split: sequence.split, + events: [], + prompt: probe.query, + requiredFacts: + probe.requiredFacts && probe.requiredFacts.length > 0 + ? probe.requiredFacts + : probe.referenceAnswer + ? [{ id: `${probe.id}:reference`, anyOf: [probe.referenceAnswer] }] + : undefined, + forbiddenFacts: probe.forbiddenFacts, + expectedEventIds: probe.expectedEventIds, + expectedActorIds: probe.expectedActorIds, + referenceAnswer: probe.referenceAnswer, + }, + artifact, + ) + return { + id: probe.id, + stepId: step.id, + query: probe.query, + score: evaluation.score, + passed: evaluation.passed, + dimensions: evaluation.dimensions, + applicableDimensions: evaluation.applicableDimensions ?? Object.keys(evaluation.dimensions), + notes: evaluation.notes, + hitIds: context.hits.map((hit) => hit.id), + } + } + if (step.parallelProbes === false) { + const results: AgentMemorySequenceProbeResult[] = [] + for (const probe of step.probes ?? []) results.push(await run(probe)) + return results + } + return Promise.all((step.probes ?? []).map(run)) +} + +function rankAgentMemoryExperiment( + candidates: readonly AgentMemoryExperimentCandidate[], + scenarios: readonly AgentMemorySequenceScenario[], + campaign: CampaignResult, + costByCandidate: ReadonlyMap, +): AgentMemoryExperimentRankingRow[] { + const scenarioById = new Map(scenarios.map((scenario) => [scenario.id, scenario])) + const rows = candidates.map((candidate): AgentMemoryExperimentRankingRow => { + const candidateScenarios = scenarios.filter((scenario) => scenario.candidateId === candidate.id) + const cells = campaign.cells.filter( + (cell) => scenarioById.get(cell.scenarioId)?.candidateId === candidate.id, + ) + const successful = cells.filter((cell) => !cell.error && cell.artifact) + const dimensionRows = successful.map((cell) => cell.artifact.dimensions) + return { + rank: 0, + candidateId: candidate.id, + label: candidate.label ?? candidate.id, + scoreMean: mean( + cells.map((cell) => (!cell.error && cell.artifact ? cell.artifact.score : 0)), + ), + passRate: mean(cells.map((cell) => (!cell.error && cell.artifact?.passed ? 1 : 0))), + totalSequences: candidateScenarios.length, + totalCells: cells.length, + totalProbes: successful.reduce((sum, cell) => sum + cell.artifact.probes.length, 0), + cellsFailed: cells.filter((cell) => Boolean(cell.error)).length, + totalCostUsd: normalizeUsd(costByCandidate.get(candidate.id) ?? 0), + durationMs: cells.reduce((sum, cell) => sum + cell.durationMs, 0), + dimensions: meanDimensions(dimensionRows), + } + }) + return rows + .sort( + (a, b) => + Number(a.cellsFailed > 0) - Number(b.cellsFailed > 0) || + b.scoreMean - a.scoreMean || + b.passRate - a.passRate || + a.totalCostUsd - b.totalCostUsd || + a.candidateId.localeCompare(b.candidateId), + ) + .map((row, index) => ({ ...row, rank: index + 1 })) +} + +function memoryExperimentCostByCandidate( + costLedger: CostLedgerHandle, + runDir: string, + scenarios: readonly AgentMemorySequenceScenario[], + candidateIdsInput: readonly string[], +): ReadonlyMap { + const candidateByScenario = new Map( + scenarios.map((scenario) => [scenario.id, scenario.candidateId]), + ) + const candidateIds = new Set(candidateIdsInput) + const totals = new Map() + for (const receipt of costLedger.list()) { + if (receipt.tags?.runDir !== runDir) continue + const scenarioCandidate = receipt.tags.scenarioId + ? candidateByScenario.get(receipt.tags.scenarioId) + : undefined + const recoveryCandidate = + receipt.tags.memoryRecovery === 'attempt' && receipt.tags.candidateId + ? receipt.tags.candidateId + : undefined + const candidateId = scenarioCandidate ?? recoveryCandidate + if (!candidateId || !candidateIds.has(candidateId)) continue + totals.set(candidateId, (totals.get(candidateId) ?? 0) + receipt.costUsd) + } + return totals +} + +function renderAgentMemoryExperimentRanking( + rows: readonly AgentMemoryExperimentRankingRow[], + totalCostUsd: number, + unrankedRecoveryCostUsd: number, +): string { + return [ + '# Agent Memory Experiment', + '', + `- total cost: $${format(totalCostUsd)}`, + `- retired-candidate recovery cost: $${format(unrankedRecoveryCostUsd)}`, + '', + '| rank | candidate | sequences | cells | probes | failed | score | pass rate | cost | duration ms |', + '| ---: | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |', + ...rows.map( + (row) => + `| ${row.rank} | ${row.label} | ${row.totalSequences} | ${row.totalCells} | ${row.totalProbes} | ${row.cellsFailed} | ${format(row.scoreMean)} | ${format(row.passRate)} | $${format(row.totalCostUsd)} | ${format(row.durationMs)} |`, + ), + '', + ].join('\n') +} + +function memoryExperimentDispatchRef(options: RunAgentMemoryExperimentOptions): string { + return stableId( + 'memory_experiment', + canonicalJson({ + implementationRef: MEMORY_EXPERIMENT_IMPLEMENTATION_REF, + experimentId: options.experimentId, + experimentRunId: options.experimentRunId ?? null, + executeStepRef: options.executeStepRef ?? 'fixtures', + cleanupBranches: options.cleanupBranches ?? true, + candidates: options.candidates + .map((candidate) => ({ + id: candidate.id, + ref: candidate.ref, + policy: candidate.policy ?? null, + baseScope: candidate.baseScope ?? null, + externalCostUsdPerSequence: candidate.externalCostUsdPerSequence ?? 0, + externalRecoveryCostUsdPerAttempt: candidate.externalRecoveryCostUsdPerAttempt ?? 0, + })) + .sort((a, b) => a.id.localeCompare(b.id)), + }), + ) +} + +function meanDimensions(rows: readonly Record[]): Record { + const values = new Map() + for (const row of rows) { + for (const [key, value] of Object.entries(row)) { + if (!Number.isFinite(value)) continue + const bucket = values.get(key) ?? [] + bucket.push(value) + values.set(key, bucket) + } + } + return Object.fromEntries([...values].map(([key, bucket]) => [key, mean(bucket)])) +} + +function countDimensions(rows: readonly (readonly string[])[]): Record { + const counts = new Map() + for (const row of rows) { + for (const key of new Set(row)) counts.set(key, (counts.get(key) ?? 0) + 1) + } + return Object.fromEntries(counts) +} + +function mean(values: readonly number[]): number { + return values.length === 0 ? 0 : values.reduce((sum, value) => sum + value, 0) / values.length +} + +function mergeScopes(base?: AgentMemoryScope, extra?: AgentMemoryScope): AgentMemoryScope { + return { + ...(base ?? {}), + ...(extra ?? {}), + tags: { ...(base?.tags ?? {}), ...(extra?.tags ?? {}) }, + } +} + +function memoryExperimentBaseScope( + options: Pick, + candidate: Pick, + sequenceId: string, +): AgentMemoryScope { + return mergeScopes(candidate.baseScope, { + tags: { + memoryExperimentId: options.experimentId, + memoryCandidateId: candidate.id, + memorySequenceId: sequenceId, + }, + }) +} + +function sequenceCleanupScopes(sequence: AgentMemorySequence): AgentMemoryScope[] { + const scopes = new Map() + for (const scope of sequence.cleanupScopes ?? []) { + const normalized = normalizeCleanupScope(scope) + scopes.set(JSON.stringify(normalized), normalized) + } + for (const step of sequence.steps) { + const candidates = [ + ...(step.scope ? [step.scope] : []), + ...(step.writes ?? []).map((write) => mergeScopes(step.scope, write.scope)), + ...(step.probes ?? []).map((probe) => mergeScopes(step.scope, probe.scope)), + ] + for (const scope of candidates) { + const normalized = normalizeCleanupScope(scope) + scopes.set(JSON.stringify(normalized), normalized) + } + } + return [...scopes.values()] +} + +async function clearSequenceScopes( + memory: AgentMemoryBranch, + sequence: AgentMemorySequence, +): Promise { + for (const scope of sequenceCleanupScopes(sequence)) await memory.clear?.(scope) +} + +function assertMemorySequences(sequences: readonly AgentMemorySequence[]): void { + for (const sequence of sequences) { + assertNonEmptyString(sequence.family, `memory experiment sequence ${sequence.id} family`) + if (sequence.steps.length === 0) { + throw new Error(`memory experiment sequence ${sequence.id} has no steps`) + } + assertUnique( + sequence.steps.map((step) => step.id), + `step in sequence ${sequence.id}`, + ) + let probeCount = 0 + for (const step of sequence.steps) { + for (const write of step.writes ?? []) { + assertNonEmptyString(write.text, `memory experiment write in ${sequence.id}/${step.id}`) + if (write.id !== undefined) { + assertNonEmptyString(write.id, `memory experiment write id in ${sequence.id}/${step.id}`) + } + } + assertUnique( + (step.probes ?? []).map((probe) => probe.id), + `probe in sequence ${sequence.id} step ${step.id}`, + ) + for (const probe of step.probes ?? []) { + probeCount += 1 + assertNonEmptyString( + probe.query, + `memory experiment probe query ${sequence.id}/${step.id}/${probe.id}`, + ) + if (probe.limit !== undefined && (!Number.isSafeInteger(probe.limit) || probe.limit <= 0)) { + throw new Error( + `memory experiment probe limit ${sequence.id}/${step.id}/${probe.id} must be a positive safe integer`, + ) + } + assertMemoryFactMatchers( + probe.requiredFacts ?? [], + `${sequence.id}/${step.id}/${probe.id} requiredFacts`, + ) + assertMemoryFactMatchers( + probe.forbiddenFacts ?? [], + `${sequence.id}/${step.id}/${probe.id} forbiddenFacts`, + ) + assertStringList(probe.expectedEventIds, `${sequence.id}/${step.id}/${probe.id} event ids`) + assertStringList(probe.expectedActorIds, `${sequence.id}/${step.id}/${probe.id} actor ids`) + if (probe.referenceAnswer !== undefined) { + assertNonEmptyString( + probe.referenceAnswer, + `memory experiment reference answer ${sequence.id}/${step.id}/${probe.id}`, + ) + } + const hasTarget = + (probe.requiredFacts?.length ?? 0) > 0 || + (probe.forbiddenFacts?.length ?? 0) > 0 || + (probe.expectedEventIds?.length ?? 0) > 0 || + (probe.expectedActorIds?.length ?? 0) > 0 || + Boolean(probe.referenceAnswer?.trim()) + if (!hasTarget) { + throw new Error( + `memory experiment probe ${sequence.id}/${step.id}/${probe.id} has no measurable target`, + ) + } + } + } + if (probeCount === 0) { + throw new Error(`memory experiment sequence ${sequence.id} has no probes`) + } + } +} + +function normalizeCleanupScope(scope: AgentMemoryScope): AgentMemoryScope { + const normalized = compactScope(scope) + if (normalized.tags && Object.keys(normalized.tags).length === 0) { + delete normalized.tags + } + return normalized +} + +function compactScope(scope: AgentMemoryScope): AgentMemoryScope { + return Object.fromEntries( + Object.entries(scope).filter(([, value]) => value !== undefined), + ) as AgentMemoryScope +} + +function compactRecord(record: Record): Record { + return Object.fromEntries(Object.entries(record).filter(([, value]) => value !== undefined)) +} + +function uniqueStrings(values: readonly unknown[]): string[] { + return [...new Set(values.filter((value): value is string => typeof value === 'string'))] +} + +function stringArray(value: unknown): string[] { + return Array.isArray(value) + ? value.filter((entry): entry is string => typeof entry === 'string') + : [] +} + +function assertUnique(values: readonly string[], label: string): void { + const seen = new Set() + for (const value of values) { + assertNonEmptyString(value, `memory experiment ${label} id`) + if (seen.has(value)) throw new Error(`duplicate memory experiment ${label} id: ${value}`) + seen.add(value) + } +} + +function assertMemoryFactMatchers( + matchers: readonly KnowledgeMemoryFactMatcher[], + label: string, +): void { + assertUnique( + matchers.map((matcher) => matcher.id), + `${label} matcher`, + ) + for (const matcher of matchers) { + if (matcher.anyOf.length === 0) { + throw new Error(`memory experiment ${label} matcher ${matcher.id} requires anyOf`) + } + assertStringList(matcher.anyOf, `${label} matcher ${matcher.id} anyOf`) + assertStringList(matcher.sourceEventIds, `${label} matcher ${matcher.id} source event ids`) + if (matcher.weight !== undefined && (!Number.isFinite(matcher.weight) || matcher.weight <= 0)) { + throw new Error( + `memory experiment ${label} matcher ${matcher.id} weight must be a positive finite number`, + ) + } + } +} + +function assertStringList(values: readonly string[] | undefined, label: string): void { + if (values === undefined) return + assertUnique(values, label) +} + +function assertNonEmptyString(value: unknown, label: string): asserts value is string { + if (typeof value !== 'string' || value.trim().length === 0) { + throw new Error(`${label} must be a non-empty string`) + } +} + +function format(value: number): string { + return Number.isFinite(value) ? value.toFixed(4) : '0.0000' +} + +function normalizeUsd(value: number): number { + return Number(value.toFixed(12)) +} diff --git a/src/memory/graphiti.ts b/src/memory/graphiti.ts new file mode 100644 index 0000000..ea9c69a --- /dev/null +++ b/src/memory/graphiti.ts @@ -0,0 +1,569 @@ +import { randomUUID } from 'node:crypto' +import { canonicalJson } from '@tangle-network/agent-eval' +import { sha256, stableId } from '../ids' +import { defaultGetMemoryContext } from './adapter' +import { mergeRankedMemoryHits } from './rank' +import { memoryWriteResultToSourceRecord } from './source-record' +import type { AgentMemoryAdapter, AgentMemoryHit, AgentMemoryScope } from './types' + +export interface GraphitiMcpClientLike { + callTool(request: { name: string; arguments?: Record }): Promise +} + +export interface GraphitiMemoryAdapterOptions { + client: GraphitiMcpClientLike + id?: string + /** Stable server/deployment identity for cache-key helpers. Never put credentials here. */ + backendRef?: string + defaultScope?: AgentMemoryScope + consistency?: 'queued' | 'visible' + ingestionTimeoutMs?: number + pollIntervalMs?: number + /** Maximum episodes inspected to prove a queued write became visible. */ + episodeScanLimit?: number + provenanceEpisodeLimit?: number + search?: readonly ('facts' | 'nodes')[] + toolNames?: Partial +} + +export interface GraphitiToolNames { + addMemory: string + searchFacts: string + searchNodes: string + getEpisodes: string + clearGraph: string +} + +interface GraphitiEpisode { + uuid: string + name?: string + content?: string + source_description?: string + group_id?: string +} + +const DEFAULT_GRAPHITI_INGESTION_TIMEOUT_MS = 120_000 + +/** Connects the official Graphiti MCP server without coupling to its Python internals. */ +export function createGraphitiMemoryAdapter( + options: GraphitiMemoryAdapterOptions, +): AgentMemoryAdapter { + assertGraphitiOptions(options) + const id = options.id ?? 'graphiti' + const consistency = options.consistency ?? 'visible' + const tools = graphitiToolNames(options) + const queued = new Map() + + const adapter: AgentMemoryAdapter = { + id, + branchIsolation: + consistency === 'visible' + ? { + mode: 'scoped', + processExitSafe: false, + recoveryDelayMs: options.ingestionTimeoutMs ?? DEFAULT_GRAPHITI_INGESTION_TIMEOUT_MS, + } + : { + mode: 'unsupported', + reason: + 'queued Graphiti writes can become visible after a process restart; use consistency="visible" for branch experiments', + }, + async search(query, searchOptions = {}) { + if (searchOptions.minScore !== undefined && !Number.isFinite(searchOptions.minScore)) { + throw new Error(`${id}: minScore must be finite`) + } + const scope = mergeScopes(options.defaultScope, searchOptions.scope) + const groupId = graphitiGroupId(id, scope) + const modes = new Set(options.search ?? ['facts', 'nodes']) + const kinds = + searchOptions.kinds && searchOptions.kinds.length > 0 ? new Set(searchOptions.kinds) : null + const searchFacts = modes.has('facts') && (!kinds || kinds.has('fact')) + const searchNodes = modes.has('nodes') && (!kinds || kinds.has('entity')) + const limit = searchOptions.limit ?? 10 + if (!Number.isSafeInteger(limit) || limit < 0) { + throw new Error(`${id}: search limit must be a non-negative safe integer`) + } + if (limit === 0) return [] + const [factPayload, nodePayload] = await Promise.all([ + searchFacts + ? callGraphiti(options.client, tools.searchFacts, { + query, + group_ids: [groupId], + max_facts: limit, + }) + : undefined, + searchNodes + ? callGraphiti(options.client, tools.searchNodes, { + query, + group_ids: [groupId], + max_nodes: limit, + ...(searchOptions.metadata?.entityTypes + ? { entity_types: searchOptions.metadata.entityTypes } + : {}), + }) + : undefined, + ]) + const hits = mergeRankedMemoryHits( + [normalizeGraphitiFacts(factPayload, id), normalizeGraphitiNodes(nodePayload, id)].map( + (group) => + group.filter((hit) => { + if (searchOptions.minScore === undefined) return true + const score = hit.normalizedScore ?? hit.score + return score !== undefined && score >= searchOptions.minScore + }), + ), + limit, + ) + await enrichGraphitiProvenance(options, groupId, hits) + return hits + }, + async getContext(query, searchOptions = {}) { + return defaultGetMemoryContext(adapter, query, searchOptions) + }, + async write(input) { + const scope = mergeScopes(options.defaultScope, input.scope) + const groupId = graphitiGroupId(id, scope) + const eventKey = canonicalJson( + stripUndefined({ + id: input.id, + kind: input.kind, + text: input.text, + title: input.title, + role: input.role, + metadata: input.metadata, + }), + ) + const episodeUuid = + input.id === undefined ? randomUUID() : deterministicUuid(`${id}:${groupId}:${eventKey}`) + const name = input.title ?? `${input.kind}:${input.id ?? stableId('event', eventKey)}` + const sourceDescription = canonicalJson( + stripUndefined({ + adapter: 'agent-knowledge', + memoryKind: input.kind, + eventId: input.id, + actorId: input.metadata?.actorId, + sessionId: scope.sessionId, + role: input.role, + scope, + metadata: input.metadata ?? {}, + }), + ) + await callGraphiti(options.client, tools.addMemory, { + name, + episode_body: input.text, + group_id: groupId, + source: input.kind === 'message' ? 'message' : 'text', + source_description: sourceDescription, + uuid: episodeUuid, + ...(typeof input.metadata?.timestamp === 'string' + ? { reference_time: input.metadata.timestamp } + : {}), + }) + queued.set(episodeUuid, { groupId, name }) + if (consistency === 'visible') { + await waitForGraphitiEpisode(options, groupId, episodeUuid) + queued.delete(episodeUuid) + } + const result = { + accepted: true, + id: episodeUuid, + uri: `memory://${id}/${episodeUuid}`, + kind: input.kind, + metadata: { + provider: 'graphiti', + groupId, + consistency, + queued: consistency === 'queued', + }, + } as const + return { + ...result, + sourceRecord: memoryWriteResultToSourceRecord(result, input.text, { scope }), + } + }, + async clear(scope) { + const groupId = graphitiGroupId(id, mergeScopes(options.defaultScope, scope)) + for (const [episodeUuid, pending] of queued) { + if (pending.groupId !== groupId) continue + await waitForGraphitiEpisode(options, groupId, episodeUuid) + queued.delete(episodeUuid) + } + await callGraphiti(options.client, tools.clearGraph, { group_ids: [groupId] }) + }, + async flush() { + for (const [episodeUuid, pending] of queued) { + await waitForGraphitiEpisode(options, pending.groupId, episodeUuid) + queued.delete(episodeUuid) + } + }, + } + return adapter +} + +async function callGraphiti( + client: GraphitiMcpClientLike, + name: string, + args: Record, +): Promise { + const raw = await client.callTool({ name, arguments: args }) + const payload = graphitiPayload(raw) + if (isRecord(payload) && typeof payload.error === 'string') { + throw new Error(`Graphiti ${name}: ${payload.error}`) + } + return payload +} + +function graphitiPayload(raw: unknown): unknown { + if (!isRecord(raw)) return raw + if (raw.isError === true) throw new Error(`Graphiti MCP: ${toolText(raw) || 'tool call failed'}`) + if (isRecord(raw.structuredContent)) return unwrapResult(raw.structuredContent) + const text = toolText(raw) + if (text) { + try { + return unwrapResult(JSON.parse(text)) + } catch { + return { message: text } + } + } + return unwrapResult(raw) +} + +function unwrapResult(value: unknown): unknown { + if (isRecord(value) && Object.keys(value).length === 1 && 'result' in value) { + return value.result + } + return value +} + +function toolText(raw: Record): string { + if (!Array.isArray(raw.content)) return '' + return raw.content + .filter(isRecord) + .map((part) => (typeof part.text === 'string' ? part.text : '')) + .filter(Boolean) + .join('\n') +} + +function normalizeGraphitiFacts(raw: unknown, adapterId: string): AgentMemoryHit[] { + const facts = isRecord(raw) && Array.isArray(raw.facts) ? raw.facts.filter(isRecord) : [] + return facts.flatMap((fact) => { + const text = stringField(fact, ['fact', 'text']) + if (!text) return [] + const factId = + stringField(fact, ['uuid', 'id']) ?? + stableId('fact', canonicalJson(stripUndefined({ adapterId, text, fact }))) + const episodeUuids = stringArray(fact.episodes ?? fact.episode_uuids) + const score = numberField(fact, ['score', 'relevance_score']) + const normalizedScore = normalizedScoreOf(score) + return [ + { + id: factId, + uri: `memory://${adapterId}/fact/${factId}`, + kind: 'fact' as const, + text, + ...(stringField(fact, ['name']) ? { title: stringField(fact, ['name']) } : {}), + ...(stringField(fact, ['created_at']) + ? { createdAt: stringField(fact, ['created_at']) } + : {}), + ...(stringField(fact, ['invalid_at']) + ? { validUntil: stringField(fact, ['invalid_at']) } + : stringField(fact, ['expired_at']) + ? { validUntil: stringField(fact, ['expired_at']) } + : {}), + ...(score !== undefined ? { score } : {}), + ...(normalizedScore !== undefined ? { normalizedScore } : {}), + metadata: { + provider: 'graphiti', + groupId: fact.group_id, + sourceNodeUuid: fact.source_node_uuid, + targetNodeUuid: fact.target_node_uuid, + validAt: fact.valid_at, + invalidAt: fact.invalid_at, + episodeUuids, + attributes: fact.attributes, + }, + }, + ] + }) +} + +function normalizeGraphitiNodes(raw: unknown, adapterId: string): AgentMemoryHit[] { + const nodes = isRecord(raw) && Array.isArray(raw.nodes) ? raw.nodes.filter(isRecord) : [] + return nodes.flatMap((node) => { + const name = stringField(node, ['name']) + const summary = stringField(node, ['summary']) + const text = summary ?? name + if (!text) return [] + const nodeId = + stringField(node, ['uuid', 'id']) ?? + stableId('node', canonicalJson(stripUndefined({ adapterId, text, node }))) + const score = numberField(node, ['score', 'relevance_score']) + const normalizedScore = normalizedScoreOf(score) + return [ + { + id: nodeId, + uri: `memory://${adapterId}/entity/${nodeId}`, + kind: 'entity' as const, + text, + ...(name ? { title: name } : {}), + ...(stringField(node, ['created_at']) + ? { createdAt: stringField(node, ['created_at']) } + : {}), + ...(score !== undefined ? { score } : {}), + ...(normalizedScore !== undefined ? { normalizedScore } : {}), + metadata: { + provider: 'graphiti', + groupId: node.group_id, + labels: node.labels, + attributes: node.attributes, + }, + }, + ] + }) +} + +async function enrichGraphitiProvenance( + options: GraphitiMemoryAdapterOptions, + groupId: string, + hits: AgentMemoryHit[], +): Promise { + const wanted = new Set(hits.flatMap((hit) => stringArray(hit.metadata?.episodeUuids))) + if (wanted.size === 0) return + const scan = await scanGraphitiEpisodes( + options, + groupId, + wanted, + options.provenanceEpisodeLimit ?? options.episodeScanLimit ?? 10_000, + ) + const episodes = scan.episodes + const byId = new Map(episodes.map((episode) => [episode.uuid, episode])) + for (const hit of hits) { + const episodeUuids = stringArray(hit.metadata?.episodeUuids) + const sourceEpisodes = episodeUuids + .map((uuid) => byId.get(uuid)) + .filter((episode): episode is GraphitiEpisode => episode !== undefined) + const provenanceIncomplete = episodeUuids.some((uuid) => !byId.has(uuid)) + if (sourceEpisodes.length === 0) { + if (provenanceIncomplete) { + hit.metadata = { ...hit.metadata, provenanceIncomplete: true } + } + continue + } + const descriptions = sourceEpisodes + .map((episode) => parseDescription(episode.source_description)) + .filter(isRecord) + const first = descriptions[0] + hit.metadata = { + ...hit.metadata, + sourceEpisodes, + ...(provenanceIncomplete || !scan.complete ? { provenanceIncomplete: true } : {}), + ...(first ?? {}), + eventIds: uniqueStrings(descriptions.map((value) => value.eventId)), + actorIds: uniqueStrings(descriptions.map((value) => value.actorId)), + ...(typeof first?.eventId === 'string' ? { eventId: first.eventId } : {}), + ...(typeof first?.actorId === 'string' ? { actorId: first.actorId } : {}), + } + } +} + +async function getGraphitiEpisodes( + options: GraphitiMemoryAdapterOptions, + groupId: string, + limit: number, +): Promise { + const raw = await callGraphiti(options.client, graphitiToolNames(options).getEpisodes, { + group_ids: [groupId], + max_episodes: limit, + }) + if (!isRecord(raw) || !Array.isArray(raw.episodes)) return [] + return raw.episodes.filter(isRecord).flatMap((episode) => { + const uuid = stringField(episode, ['uuid', 'id']) + return uuid ? [{ ...episode, uuid } as GraphitiEpisode] : [] + }) +} + +async function scanGraphitiEpisodes( + options: GraphitiMemoryAdapterOptions, + groupId: string, + wanted: ReadonlySet, + maximum: number, +): Promise<{ episodes: GraphitiEpisode[]; complete: boolean }> { + let limit = Math.min(100, maximum) + for (;;) { + const episodes = await getGraphitiEpisodes(options, groupId, limit) + const foundAll = [...wanted].every((uuid) => episodes.some((episode) => episode.uuid === uuid)) + if (foundAll || episodes.length < limit) return { episodes, complete: foundAll } + if (limit === maximum) return { episodes, complete: false } + limit = Math.min(maximum, limit * 2) + } +} + +async function waitForGraphitiEpisode( + options: GraphitiMemoryAdapterOptions, + groupId: string, + episodeUuid: string, +): Promise { + const timeoutMs = options.ingestionTimeoutMs ?? DEFAULT_GRAPHITI_INGESTION_TIMEOUT_MS + const pollIntervalMs = options.pollIntervalMs ?? 500 + const maximum = options.episodeScanLimit ?? 10_000 + const deadline = Date.now() + timeoutMs + let scanLimitReached = false + for (;;) { + const scan = await scanGraphitiEpisodes(options, groupId, new Set([episodeUuid]), maximum) + if (scan.complete) return + scanLimitReached ||= scan.episodes.length >= maximum + const remainingMs = deadline - Date.now() + if (remainingMs <= 0) break + await new Promise((resolve) => setTimeout(resolve, Math.min(pollIntervalMs, remainingMs))) + } + if (scanLimitReached) { + throw new Error( + `Graphiti episode ${episodeUuid} is outside episodeScanLimit=${maximum}; raise the limit or use a smaller group`, + ) + } + throw new Error(`Graphiti episode ${episodeUuid} was not visible after ${timeoutMs}ms`) +} + +function graphitiGroupId(adapterId: string, scope: AgentMemoryScope): string { + return `ak_${sha256( + canonicalJson( + stripUndefined({ + adapterId, + tenantId: scope.tenantId, + userId: scope.userId, + agentId: scope.agentId, + teamId: scope.teamId, + runId: scope.runId, + sessionId: scope.sessionId, + namespace: scope.namespace, + tags: scope.tags, + }), + ), + ).slice(0, 32)}` +} + +function graphitiToolNames(options: GraphitiMemoryAdapterOptions): GraphitiToolNames { + return { + addMemory: options.toolNames?.addMemory ?? 'add_memory', + searchFacts: options.toolNames?.searchFacts ?? 'search_memory_facts', + searchNodes: options.toolNames?.searchNodes ?? 'search_nodes', + getEpisodes: options.toolNames?.getEpisodes ?? 'get_episodes', + clearGraph: options.toolNames?.clearGraph ?? 'clear_graph', + } +} + +function assertGraphitiOptions(options: GraphitiMemoryAdapterOptions): void { + if (options.id !== undefined && (typeof options.id !== 'string' || !options.id.trim())) { + throw new Error('Graphiti id must be a non-empty string') + } + if ( + options.backendRef !== undefined && + (typeof options.backendRef !== 'string' || !options.backendRef.trim()) + ) { + throw new Error('Graphiti backendRef must be a non-empty string') + } + if (options.consistency !== undefined && !['queued', 'visible'].includes(options.consistency)) { + throw new Error('Graphiti consistency must be queued or visible') + } + for (const [name, value] of [ + ['ingestionTimeoutMs', options.ingestionTimeoutMs], + ['pollIntervalMs', options.pollIntervalMs], + ['episodeScanLimit', options.episodeScanLimit], + ['provenanceEpisodeLimit', options.provenanceEpisodeLimit], + ] as const) { + if (value !== undefined && (!Number.isSafeInteger(value) || value <= 0)) { + throw new Error(`Graphiti ${name} must be a positive safe integer`) + } + } + for (const [name, value] of Object.entries(options.toolNames ?? {})) { + if (typeof value !== 'string' || value.trim().length === 0) { + throw new Error(`Graphiti toolNames.${name} must be a non-empty string`) + } + } +} + +function deterministicUuid(value: string): string { + const hex = sha256(value) + return `${hex.slice(0, 8)}-${hex.slice(8, 12)}-5${hex.slice(13, 16)}-a${hex.slice(17, 20)}-${hex.slice(20, 32)}` +} + +function parseDescription(value: string | undefined): unknown { + if (!value) return undefined + try { + return JSON.parse(value) + } catch { + return undefined + } +} + +function mergeScopes(base?: AgentMemoryScope, extra?: AgentMemoryScope): AgentMemoryScope { + return { + ...(base ?? {}), + ...(extra ?? {}), + tags: { ...(base?.tags ?? {}), ...(extra?.tags ?? {}) }, + } +} + +function uniqueStrings(values: unknown[]): string[] { + return [...new Set(values.filter((value): value is string => typeof value === 'string'))] +} + +function stringArray(value: unknown): string[] { + return Array.isArray(value) + ? value.filter((entry): entry is string => typeof entry === 'string') + : [] +} + +function stringField(value: Record, keys: readonly string[]): string | undefined { + for (const key of keys) { + const field = value[key] + if (typeof field === 'string' && field.length > 0) return field + } + return undefined +} + +function numberField(value: Record, keys: readonly string[]): number | undefined { + for (const key of keys) { + const field = value[key] + if (typeof field === 'number' && Number.isFinite(field)) return field + } + return undefined +} + +function normalizedScoreOf(score: number | undefined): number | undefined { + return score !== undefined && score >= 0 && score <= 1 ? score : undefined +} + +function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value) +} + +function stripUndefined(value: unknown): unknown { + if (Array.isArray(value)) return value.map(stripUndefined) + if (!isRecord(value)) return value + return Object.fromEntries( + Object.entries(value) + .filter(([, child]) => child !== undefined) + .map(([key, child]) => [key, stripUndefined(child)]), + ) +} + +export function graphitiMemoryAdapterIdentity( + options: Pick< + GraphitiMemoryAdapterOptions, + | 'id' + | 'consistency' + | 'ingestionTimeoutMs' + | 'pollIntervalMs' + | 'episodeScanLimit' + | 'provenanceEpisodeLimit' + | 'search' + | 'toolNames' + | 'defaultScope' + > & { backendRef: string }, +): string { + if (typeof options.backendRef !== 'string' || !options.backendRef.trim()) { + throw new Error('Graphiti backendRef must be a non-empty string') + } + return stableId('graphiti', canonicalJson(stripUndefined(options))) +} diff --git a/src/memory/improvement.ts b/src/memory/improvement.ts new file mode 100644 index 0000000..1e6029c --- /dev/null +++ b/src/memory/improvement.ts @@ -0,0 +1,1249 @@ +import { join } from 'node:path' +import { canonicalJson } from '@tangle-network/agent-eval' +import { + type CampaignStorage, + type CostLedgerHandle, + campaignLineageStore, + createRunCostLedger, + fsCampaignStorage, + type Governor, + type GovernorContext, + type GovernorOp, + type HeldoutSignificance, + type HeldoutSignificanceOptions, + heldoutSignificance, + type Lineage, + type LineageStore, + type MutableSurface, + memLineageStore, + type PairedHoldout, + resolveRunDir, + runLineageLoop, + type Scenario, + type SurfaceProposer, + surfaceHash, +} from '@tangle-network/agent-eval/campaign' +import { + appendDurableJournalEvent, + assertNoInterruptedPaidCalls, + reconcileInterruptedRunPaidCalls, +} from './attempt-log' +import type { + AgentMemoryExperimentCandidate, + AgentMemorySequence, + AgentMemorySequenceArtifact, + AgentMemorySequenceProbe, + RunAgentMemoryExperimentOptions, + RunAgentMemoryExperimentResult, +} from './experiment' +import { runAgentMemoryExperiment } from './experiment' +import { runBoundedMemoryLifecycle } from './lifecycle' +import { + type AgentMemoryAcquireRunLease, + type AgentMemoryControllerMode, + type AgentMemoryRunLease, + acquireAgentMemoryRunLease, + type OwnedAgentMemoryRunLease, +} from './run-control' + +export interface AgentMemoryImprovementSeed { + config: TConfig + track: string + vision?: string + proposer: string +} + +export interface AgentMemoryDimensionComparison { + dimension: string + n: number + expectedN: number + measured: boolean + meanDelta: number + low: number + high: number + tolerance: number + regressed: boolean +} + +export interface AgentMemoryPromotionDecision { + status: 'promote' | 'hold' | 'no-change' + reasons: readonly string[] + baselineScore: number + winnerScore: number + lift: number + significance?: HeldoutSignificance + criticalDimensions: readonly AgentMemoryDimensionComparison[] +} + +export interface AgentMemoryActivation { + id: string + status: + | 'not-eligible' + | 'not-configured' + | 'pending' + | 'activated' + | 'recovered' + | 'already-activated' + journalPath: string +} + +export interface AgentMemoryActivationDriver { + /** Change whenever activation behavior or the external target changes. */ + ref: string + /** Return the exact currently active configuration. */ + readCurrent(): Promise + /** Atomically replace expectedConfig with config, or fail on a concurrent change. */ + compareAndSet(input: { + activationId: string + expectedConfig: TConfig + expectedSurfaceHash: string + config: TConfig + surfaceHash: string + decision: AgentMemoryPromotionDecision + lineage: Lineage + holdout: RunAgentMemoryExperimentResult + }): Promise +} + +interface AgentMemoryActivationEvent { + schema: 1 + status: 'prepared' | 'activated' + activationId: string + experimentId: string + activationRef: string + baselineSurfaceHash: string + winnerSurfaceHash: string + holdoutManifestHash: string + recordedAt: string + outcome?: 'applied' | 'recovered' | 'already-current' +} + +interface AgentMemoryActivationJournalState { + prepared: boolean + activated?: AgentMemoryActivationEvent +} + +export type AgentMemoryImprovementRunLease = AgentMemoryRunLease + +export interface AgentMemoryGovernor { + decide( + context: GovernorContext & { + costLedger: CostLedgerHandle + costPhase: string + }, + ): GovernorOp | Promise +} + +export interface RunAgentMemoryImprovementOptions { + experimentId: string + trainSequences: readonly AgentMemorySequence[] + holdoutSequences: readonly AgentMemorySequence[] + /** First entry is the current baseline; remaining entries seed independent search tracks. */ + seeds: readonly AgentMemoryImprovementSeed[] + createCandidate(input: { + config: TConfig + candidateId: string + surfaceHash: string + }): + | Omit + | Promise> + proposer: SurfaceProposer + /** Optional proposer implementations keyed by seed and branch proposer labels. */ + proposers?: Readonly> + /** Stable version or commit for the candidate factory, proposer, and governor. */ + improvementRef: string + governor?: AgentMemoryGovernor + budget: { maxSteps: number } + populationSize?: number + candidateConcurrency?: number + sequenceConcurrency?: number + runDir: string + repo?: string + storage?: CampaignStorage + lineageStore?: LineageStore + /** Required with custom storage when all controllers are confined to one process. */ + controllerMode?: AgentMemoryControllerMode + /** Required for distributed controllers using custom storage. Worker concurrency is independent. */ + acquireRunLease?: AgentMemoryAcquireRunLease + seed?: number + reps?: number + resumable?: boolean + dispatchTimeoutMs?: number + cleanupTimeoutMs?: number + maxRecoveryAttempts?: number + maxRecoveryRetriesPerAttempt?: number + maxTotalCostUsd?: number + executeStep?: RunAgentMemoryExperimentOptions['executeStep'] + executeStepRef?: string + onBranchSnapshot?: RunAgentMemoryExperimentOptions['onBranchSnapshot'] + cleanupBranches?: boolean + serializeConfig?: (config: TConfig) => string + parseConfig?: (surface: string) => TConfig + significance?: HeldoutSignificanceOptions + criticalDimensions?: readonly string[] + criticalDimensionTolerance?: number + minHoldoutScore?: number + activation?: AgentMemoryActivationDriver + activationTimeoutMs?: number + now?: () => Date +} + +export interface RunAgentMemoryImprovementResult { + lineage: Lineage + baselineConfig: TConfig + winnerConfig: TConfig + baselineSurface: string + winnerSurface: string + baselineSurfaceHash: string + winnerSurfaceHash: string + decision: AgentMemoryPromotionDecision + activation: AgentMemoryActivation + holdout?: RunAgentMemoryExperimentResult + totalCostUsd: number + resultJsonPath: string +} + +interface MemoryConfigScenario extends Scenario { + kind: 'agent-memory-config-search' + sequenceId: string +} + +const DEFAULT_CRITICAL_DIMENSIONS = [ + 'memory_stale_safe', + 'memory_actor_recall', + 'memory_event_recall', +] as const +const MEMORY_IMPROVEMENT_IMPLEMENTATION_REF = 'agent-knowledge:memory-improvement:v2' + +/** Searches branchable memory configurations and activates only a fresh holdout win. */ +export async function runAgentMemoryImprovement( + options: RunAgentMemoryImprovementOptions, +): Promise> { + if ('onPromote' in options) { + throw new Error( + 'memory improvement onPromote was removed; use activation.readCurrent and activation.compareAndSet', + ) + } + if (options.seeds.length === 0) throw new Error('memory improvement requires seed configs') + if (options.trainSequences.length === 0) { + throw new Error('memory improvement requires training sequences') + } + if (options.holdoutSequences.length === 0) { + throw new Error('memory improvement requires holdout sequences') + } + const trainIds = new Set(options.trainSequences.map((sequence) => sequence.id)) + const overlap = options.holdoutSequences + .map((sequence) => sequence.id) + .filter((id) => trainIds.has(id)) + if (overlap.length > 0) { + throw new Error(`memory improvement train/holdout overlap: ${overlap.join(', ')}`) + } + const trainFingerprints = new Map( + options.trainSequences.map((sequence) => [memorySequenceFingerprint(sequence), sequence.id]), + ) + const duplicateHistories = options.holdoutSequences.flatMap((sequence) => { + const trainId = trainFingerprints.get(memorySequenceFingerprint(sequence)) + return trainId ? [`${trainId}/${sequence.id}`] : [] + }) + if (duplicateHistories.length > 0) { + throw new Error( + `memory improvement train/holdout histories duplicate content: ${duplicateHistories.join(', ')}`, + ) + } + if (typeof options.improvementRef !== 'string' || !options.improvementRef.trim()) { + throw new Error('memory improvement improvementRef must be a non-empty string') + } + assertMemoryImprovementOptions(options) + const storage = options.storage ?? fsCampaignStorage() + const runDir = resolveRunDir(options.runDir, options.repo) + storage.ensureDir(runDir) + const lease = await acquireAgentMemoryRunLease({ + experimentId: options.experimentId, + runDir, + storage, + customStorage: options.storage !== undefined, + lockFileName: 'memory-improvement.lock', + label: 'memory improvement', + controllerMode: options.controllerMode, + acquireRunLease: options.acquireRunLease, + }) + let result: RunAgentMemoryImprovementResult | undefined + let primaryError: unknown + try { + result = await runAgentMemoryImprovementOwned(options, storage, runDir, lease) + } catch (error) { + primaryError = error + } + let releaseError: unknown + try { + await lease.release() + } catch (error) { + releaseError = error + } + if (primaryError && releaseError) { + throw new AggregateError( + [primaryError, releaseError], + 'memory improvement failed and its controller lease could not be released', + ) + } + if (primaryError) throw primaryError + if (releaseError) throw releaseError + if (!result) throw new Error('memory improvement produced no result') + return result +} + +async function runAgentMemoryImprovementOwned( + options: RunAgentMemoryImprovementOptions, + storage: CampaignStorage, + runDir: string, + lease: OwnedRunLease, +): Promise> { + await lease.assertOwned() + const serializeRaw = options.serializeConfig ?? ((config: TConfig) => canonicalJson(config)) + const parseRaw = options.parseConfig ?? ((surface: string) => JSON.parse(surface) as TConfig) + const serialize = (config: TConfig): string => serializeMemoryConfig(serializeRaw, config) + const parse = (surface: string): TConfig => parseMemoryConfig(parseRaw, surface) + for (const seed of options.seeds) assertMemoryConfigRoundTrip(seed.config, serialize, parse) + if (options.activation && !storage.append) { + throw new Error('memory activation requires CampaignStorage.append') + } + if (options.resumable !== false && !options.lineageStore && !storage.append) { + throw new Error('resumable memory improvement requires CampaignStorage.append') + } + assertMemoryImprovementIdentity(options, storage, runDir, serialize) + const costLedger = createRunCostLedger({ + storage, + runDir, + costCeilingUsd: options.maxTotalCostUsd ?? 0, + }) + reconcileInterruptedRunPaidCalls(costLedger, 'memory improvement run') + assertNoInterruptedPaidCalls(costLedger, 'memory improvement recovery') + const trainScenarios: MemoryConfigScenario[] = options.trainSequences.map((sequence) => ({ + id: sequence.id, + kind: 'agent-memory-config-search', + sequenceId: sequence.id, + })) + const evaluations = new Map>() + + const evaluateSurface = async ( + surface: MutableSurface, + ): Promise<{ score: number; scoreVector: number[] }> => { + await lease.assertOwned() + const text = requireStringSurface(surface) + const config = parse(text) + const canonicalSurface = serialize(config) + const hash = surfaceHash(canonicalSurface) + let pending = evaluations.get(hash) + if (!pending) { + pending = (async () => { + const candidate = await buildCandidate(options, config, hash, `search-${hash}`) + await lease.assertOwned() + const experiment = await runAgentMemoryExperiment({ + ...experimentOptions(options, costLedger, storage, lease), + experimentId: `${options.experimentId}:search:${hash}`, + sequences: options.trainSequences, + candidates: [candidate], + runDir: join(runDir, 'search', hash), + costPhase: `memory.search.${hash}`, + }) + await lease.assertOwned() + return experiment + })() + evaluations.set(hash, pending) + void pending.catch(() => evaluations.delete(hash)) + } + const result = await pending + await lease.assertOwned() + const row = result.rows[0] + if (!row || row.cellsFailed > 0) { + throw new Error(`${hash}: memory candidate did not complete every training cell`) + } + return { + score: row.scoreMean, + scoreVector: sequenceScores(result, options.trainSequences, row.candidateId), + } + } + + const lineageStore = + options.lineageStore ?? + (options.resumable === false + ? memLineageStore() + : campaignLineageStore(storage, join(runDir, 'lineage.jsonl'))) + const lineageOptions = { + seeds: options.seeds.map((seed) => ({ + surface: serialize(seed.config), + track: seed.track, + proposer: seed.proposer, + ...(seed.vision !== undefined ? { vision: seed.vision } : {}), + })), + scenarios: trainScenarios, + proposer: withCostContext(options.proposer, costLedger, lease, 'default'), + proposers: options.proposers + ? Object.fromEntries( + Object.entries(options.proposers).map(([name, proposer]) => [ + name, + withCostContext(proposer, costLedger, lease, name), + ]), + ) + : undefined, + scoreSurface: evaluateSurface, + governor: options.governor + ? withGovernorCostContext(options.governor, costLedger, lease) + : undefined, + budget: { + ...options.budget, + maxNodes: options.seeds.length + options.budget.maxSteps, + }, + store: lineageStore, + populationSize: options.populationSize, + candidateConcurrency: options.candidateConcurrency, + } + const search = await runLineageLoop(lineageOptions) + await lease.assertOwned() + const best = search.best + if (!best) throw new Error('memory improvement produced no measured config') + + const baselineSurface = serialize(options.seeds[0]!.config) + const baselineHash = surfaceHash(baselineSurface) + const winnerConfig = parse(requireStringSurface(best.surface)) + const winnerSurface = serialize(winnerConfig) + const winnerHash = surfaceHash(winnerSurface) + const baselineConfig = parse(baselineSurface) + + let holdout: RunAgentMemoryExperimentResult | undefined + let decision: AgentMemoryPromotionDecision + if (winnerHash === baselineHash) { + const baselineMeasurement = await evaluateSurface(baselineSurface) + decision = { + status: 'no-change', + reasons: ['search did not find a config better than the baseline'], + baselineScore: baselineMeasurement.score, + winnerScore: baselineMeasurement.score, + lift: 0, + criticalDimensions: [], + } + } else { + await lease.assertOwned() + const [baselineCandidate, winnerCandidate] = await Promise.all([ + buildCandidate(options, baselineConfig, baselineHash, 'baseline'), + buildCandidate(options, winnerConfig, winnerHash, 'winner'), + ]) + await lease.assertOwned() + holdout = await runAgentMemoryExperiment({ + ...experimentOptions(options, costLedger, storage, lease), + experimentId: `${options.experimentId}:holdout`, + sequences: options.holdoutSequences, + candidates: [baselineCandidate, winnerCandidate], + runDir: join(runDir, 'holdout'), + costPhase: 'memory.holdout', + }) + decision = decidePromotion({ + options, + result: holdout, + baselineId: baselineCandidate.id, + winnerId: winnerCandidate.id, + }) + } + + const resultJsonPath = join(runDir, 'memory-improvement-result.json') + const activationRef = options.activation?.ref ?? 'not-configured' + const activationId = `memory-activation-${surfaceHash( + canonicalJson({ + experimentId: options.experimentId, + improvementRef: options.improvementRef, + activationRef, + baselineSurfaceHash: baselineHash, + winnerSurfaceHash: winnerHash, + holdoutManifestHash: holdout?.campaign.manifestHash ?? null, + promotionPolicy: normalizedPromotionPolicy(options), + }), + )}` + const activationJournalDir = join(runDir, 'activations') + const activationJournalPath = join(activationJournalDir, `${activationId}.jsonl`) + const activationEligible = decision.status === 'promote' && holdout !== undefined + const activationEventIdentity = holdout + ? { + schema: 1 as const, + activationId, + experimentId: options.experimentId, + activationRef, + baselineSurfaceHash: baselineHash, + winnerSurfaceHash: winnerHash, + holdoutManifestHash: holdout.campaign.manifestHash, + } + : undefined + const activationJournal = + activationEligible && activationEventIdentity + ? readMemoryActivationJournal(storage, activationJournalPath, activationEventIdentity) + : { prepared: false } + const activation: AgentMemoryActivation = { + id: activationId, + status: !activationEligible + ? 'not-eligible' + : activationJournal.activated + ? 'already-activated' + : options.activation + ? 'pending' + : 'not-configured', + journalPath: activationJournalPath, + } + const result = { + lineage: search.lineage, + baselineConfig, + winnerConfig, + baselineSurface, + winnerSurface, + baselineSurfaceHash: baselineHash, + winnerSurfaceHash: winnerHash, + decision, + activation, + ...(holdout ? { holdout } : {}), + totalCostUsd: costLedger.summary().totalCostUsd, + resultJsonPath, + } satisfies RunAgentMemoryImprovementResult + await lease.assertOwned() + writeMemoryImprovementResult(storage, options.experimentId, result) + if ( + activationEligible && + !activationJournal.activated && + activationEventIdentity && + holdout && + options.activation + ) { + const activationDriver = options.activation + const activationTimeoutMs = options.activationTimeoutMs ?? 60_000 + const hadPreparedEvent = activationJournal.prepared + if (!hadPreparedEvent) { + await lease.assertOwned() + storage.ensureDir(activationJournalDir) + appendMemoryActivationEvent(storage, activationJournalPath, { + ...activationEventIdentity, + status: 'prepared', + recordedAt: (options.now ?? (() => new Date()))().toISOString(), + }) + } + + await lease.assertOwned() + const currentConfig = await runBoundedMemoryLifecycle({ + operation: `${activationDriver.ref}: read current memory configuration`, + timeoutMs: activationTimeoutMs, + run: () => activationDriver.readCurrent(), + }) + await lease.assertOwned() + const currentHash = surfaceHash(serialize(currentConfig)) + if (currentHash !== baselineHash && currentHash !== winnerHash) { + throw new Error( + `memory activation target '${activationDriver.ref}' changed concurrently; expected '${baselineHash}' or '${winnerHash}', found '${currentHash}'`, + ) + } + + let outcome: NonNullable + if (currentHash === winnerHash) { + outcome = hadPreparedEvent ? 'recovered' : 'already-current' + activation.status = 'recovered' + } else { + let compareError: unknown + try { + await runBoundedMemoryLifecycle({ + operation: `${activationDriver.ref}: activate memory configuration`, + timeoutMs: activationTimeoutMs, + run: () => + activationDriver.compareAndSet({ + activationId, + expectedConfig: baselineConfig, + expectedSurfaceHash: baselineHash, + config: winnerConfig, + surfaceHash: winnerHash, + decision, + lineage: search.lineage, + holdout, + }), + }) + } catch (error) { + compareError = error + } + await lease.assertOwned() + + let observedConfig: TConfig + try { + observedConfig = await runBoundedMemoryLifecycle({ + operation: `${activationDriver.ref}: confirm memory configuration`, + timeoutMs: activationTimeoutMs, + run: () => activationDriver.readCurrent(), + }) + } catch (error) { + if (compareError) { + throw new AggregateError( + [compareError, error], + `memory activation '${activationId}' failed and its live state could not be confirmed`, + ) + } + throw error + } + await lease.assertOwned() + const observedHash = surfaceHash(serialize(observedConfig)) + if (observedHash !== winnerHash) { + const mismatch = new Error( + `memory activation '${activationId}' did not install the measured winner; found '${observedHash}'`, + ) + if (compareError) { + throw new AggregateError( + [compareError, mismatch], + `memory activation '${activationId}' failed without applying the measured winner`, + ) + } + throw mismatch + } + outcome = compareError ? 'recovered' : 'applied' + activation.status = compareError ? 'recovered' : 'activated' + } + + await lease.assertOwned() + appendMemoryActivationEvent(storage, activationJournalPath, { + ...activationEventIdentity, + status: 'activated', + outcome, + recordedAt: (options.now ?? (() => new Date()))().toISOString(), + }) + writeMemoryImprovementResult(storage, options.experimentId, result) + } + return result +} + +function withCostContext( + proposer: SurfaceProposer, + costLedger: CostLedgerHandle, + lease: OwnedRunLease, + label: string, +): SurfaceProposer { + return { + kind: proposer.kind, + async propose(context) { + await lease.assertOwned() + const proposal = await proposer.propose({ + ...context, + costLedger, + costPhase: `memory.proposal.${context.track?.id ?? label}`, + }) + await lease.assertOwned() + return proposal + }, + ...(proposer.decide ? { decide: (input) => proposer.decide!(input) } : {}), + } +} + +function withGovernorCostContext( + governor: AgentMemoryGovernor, + costLedger: CostLedgerHandle, + lease: OwnedRunLease, +): Governor { + return { + async decide(context) { + await lease.assertOwned() + const decision = await governor.decide({ + ...context, + costLedger, + costPhase: 'memory.governor', + }) + await lease.assertOwned() + return decision + }, + } +} + +async function buildCandidate( + options: RunAgentMemoryImprovementOptions, + config: TConfig, + hash: string, + role: string, +): Promise { + const built = await options.createCandidate({ + config, + candidateId: `${role}-${hash}`, + surfaceHash: hash, + }) + return { + ...built, + id: `${role}-${hash}`, + ref: built.ref, + } +} + +function experimentOptions( + options: RunAgentMemoryImprovementOptions, + costLedger: ReturnType, + storage: CampaignStorage, + lease: OwnedRunLease, +): Pick< + RunAgentMemoryExperimentOptions, + | 'storage' + | 'seed' + | 'reps' + | 'resumable' + | 'maxConcurrency' + | 'dispatchTimeoutMs' + | 'cleanupTimeoutMs' + | 'maxRecoveryAttempts' + | 'maxRecoveryRetriesPerAttempt' + | 'executeStep' + | 'executeStepRef' + | 'onBranchSnapshot' + | 'cleanupBranches' + | 'costLedger' + | 'acquireRunLease' + | 'now' +> { + return { + storage, + seed: options.seed, + reps: options.reps, + resumable: options.resumable, + maxConcurrency: options.sequenceConcurrency, + dispatchTimeoutMs: options.dispatchTimeoutMs, + cleanupTimeoutMs: options.cleanupTimeoutMs, + maxRecoveryAttempts: options.maxRecoveryAttempts, + maxRecoveryRetriesPerAttempt: options.maxRecoveryRetriesPerAttempt, + executeStep: options.executeStep, + executeStepRef: options.executeStepRef, + onBranchSnapshot: options.onBranchSnapshot, + cleanupBranches: options.cleanupBranches ?? true, + costLedger, + acquireRunLease: async () => ({ + assertOwned: () => lease.assertOwned(), + release() {}, + }), + now: options.now, + } +} + +type OwnedRunLease = OwnedAgentMemoryRunLease + +function assertMemoryImprovementIdentity( + options: RunAgentMemoryImprovementOptions, + storage: CampaignStorage, + runDir: string, + serialize: (config: TConfig) => string, +): void { + const path = join(runDir, 'memory-improvement-manifest.json') + const identity = { + schema: 6, + implementationRef: MEMORY_IMPROVEMENT_IMPLEMENTATION_REF, + experimentId: options.experimentId, + improvementRef: options.improvementRef, + activationRef: options.activation?.ref ?? null, + proposerKind: options.proposer.kind, + proposerKinds: Object.fromEntries( + Object.entries(options.proposers ?? {}) + .sort(([a], [b]) => a.localeCompare(b)) + .map(([name, proposer]) => [name, proposer.kind]), + ), + populationSize: options.populationSize ?? 4, + budget: { maxSteps: options.budget.maxSteps }, + executeStepRef: options.executeStepRef ?? null, + cleanupBranches: options.cleanupBranches ?? true, + promotionPolicy: normalizedPromotionPolicy(options), + seed: options.seed ?? null, + reps: options.reps ?? 1, + seeds: options.seeds.map((entry) => ({ + config: serialize(entry.config), + track: entry.track, + vision: entry.vision ?? null, + proposer: entry.proposer, + })), + trainSequences: options.trainSequences, + holdoutSequences: options.holdoutSequences, + } + const identityHash = surfaceHash(canonicalJson(identity)) + const stored = storage.read(path) + if (stored === undefined) { + if (storage.exists(path)) throw new Error(`cannot read memory improvement manifest '${path}'`) + if ( + storage.exists(join(runDir, 'lineage.jsonl')) || + storage.exists(join(runDir, 'memory-improvement-result.json')) + ) { + throw new Error(`memory improvement run '${runDir}' has state without an identity manifest`) + } + storage.write(path, `${JSON.stringify({ identityHash, identity }, null, 2)}\n`) + return + } + let manifest: unknown + try { + manifest = JSON.parse(stored) + } catch (error) { + throw new Error(`invalid memory improvement manifest '${path}'`, { cause: error }) + } + if ( + !manifest || + typeof manifest !== 'object' || + (manifest as Record).identityHash !== identityHash || + canonicalJson((manifest as Record).identity) !== canonicalJson(identity) + ) { + throw new Error( + `memory improvement run '${runDir}' does not match its persisted inputs or implementationRef`, + ) + } +} + +function appendMemoryActivationEvent( + storage: CampaignStorage, + path: string, + event: AgentMemoryActivationEvent, +): void { + appendDurableJournalEvent({ + storage, + path, + event, + label: 'memory activation journal', + }) +} + +function readMemoryActivationJournal( + storage: CampaignStorage, + path: string, + expected: Omit, +): AgentMemoryActivationJournalState { + const stored = storage.read(path) + if (stored === undefined) { + if (storage.exists(path)) throw new Error(`cannot read memory activation journal '${path}'`) + return { prepared: false } + } + let prepared = false + let activated: AgentMemoryActivationEvent | undefined + for (const [index, line] of stored.split('\n').entries()) { + if (!line) continue + let raw: unknown + try { + raw = JSON.parse(line) + } catch (error) { + throw new Error(`invalid memory activation journal '${path}' line ${index + 1}`, { + cause: error, + }) + } + const event = parseMemoryActivationEvent(raw, path, index + 1, expected) + if (event.status === 'prepared') { + if (prepared || activated) { + throw new Error(`memory activation journal '${path}' repeats its prepared event`) + } + prepared = true + continue + } + if (!prepared || activated) { + throw new Error(`memory activation journal '${path}' has an out-of-order activated event`) + } + activated = event + } + return { prepared, ...(activated ? { activated } : {}) } +} + +function parseMemoryActivationEvent( + value: unknown, + path: string, + line: number, + expected: Omit, +): AgentMemoryActivationEvent { + if (!value || typeof value !== 'object' || Array.isArray(value)) { + throw new Error(`invalid memory activation journal '${path}' line ${line}`) + } + const event = value as Record + for (const [key, expectedValue] of Object.entries(expected)) { + if (event[key] !== expectedValue) { + throw new Error( + `memory activation journal '${path}' line ${line} does not match the measured winner`, + ) + } + } + if (event.status !== 'prepared' && event.status !== 'activated') { + throw new Error(`invalid memory activation journal '${path}' line ${line} status`) + } + if (typeof event.recordedAt !== 'string' || !Number.isFinite(Date.parse(event.recordedAt))) { + throw new Error(`invalid memory activation journal '${path}' line ${line} recordedAt`) + } + if ( + event.status === 'activated' && + event.outcome !== 'applied' && + event.outcome !== 'recovered' && + event.outcome !== 'already-current' + ) { + throw new Error(`invalid memory activation journal '${path}' line ${line} outcome`) + } + if (event.status === 'prepared' && event.outcome !== undefined) { + throw new Error(`invalid memory activation journal '${path}' line ${line} prepared outcome`) + } + return value as AgentMemoryActivationEvent +} + +function writeMemoryImprovementResult( + storage: CampaignStorage, + experimentId: string, + result: RunAgentMemoryImprovementResult, +): void { + storage.write( + result.resultJsonPath, + `${JSON.stringify( + { + experimentId, + baselineSurfaceHash: result.baselineSurfaceHash, + winnerSurfaceHash: result.winnerSurfaceHash, + baselineSurface: result.baselineSurface, + winnerSurface: result.winnerSurface, + decision: result.decision, + activation: result.activation, + totalCostUsd: result.totalCostUsd, + lineage: result.lineage.toGraph(), + }, + null, + 2, + )}\n`, + ) +} + +function decidePromotion(input: { + options: RunAgentMemoryImprovementOptions + result: RunAgentMemoryExperimentResult + baselineId: string + winnerId: string +}): AgentMemoryPromotionDecision { + const { options, result, baselineId, winnerId } = input + const baselineRow = result.rows.find((row) => row.candidateId === baselineId) + const winnerRow = result.rows.find((row) => row.candidateId === winnerId) + if (!baselineRow || !winnerRow) throw new Error('holdout result is missing a comparison arm') + const paired = pairedArtifacts(result, baselineId, winnerId, (artifact) => artifact.score) + const significance = heldoutSignificance(paired, options.significance) + const tolerance = options.criticalDimensionTolerance ?? 0.05 + const criticalDimensions = (options.criticalDimensions ?? DEFAULT_CRITICAL_DIMENSIONS).map( + (dimension) => { + const expectedN = + applicableSequenceCount(options.holdoutSequences, dimension) * (options.reps ?? 1) + const dimensionPairs = pairedArtifacts(result, baselineId, winnerId, (artifact) => + (artifact.dimensionSampleCounts?.[dimension] ?? 0) > 0 + ? artifact.dimensions[dimension] + : undefined, + ) + const comparison = heldoutSignificance(dimensionPairs, { + ...options.significance, + deltaThreshold: 0, + }) + return { + dimension, + n: comparison.n, + expectedN, + measured: expectedN > 0 && comparison.n === expectedN, + meanDelta: comparison.bootstrap.mean, + low: comparison.bootstrap.low, + high: comparison.bootstrap.high, + tolerance, + regressed: + expectedN > 0 && comparison.n === expectedN && comparison.bootstrap.low < -tolerance, + } + }, + ) + const reasons: string[] = [] + if (baselineRow.cellsFailed > 0 || winnerRow.cellsFailed > 0) { + reasons.push('at least one holdout cell failed') + } + if (!significance.significant) { + reasons.push( + significance.fewRuns + ? `only ${significance.n} paired holdout cells; more are required` + : 'holdout lift is not confidently above the promotion threshold', + ) + } + if (winnerRow.scoreMean < (options.minHoldoutScore ?? 0)) { + reasons.push(`winner holdout score ${winnerRow.scoreMean} is below the required minimum`) + } + for (const dimension of criticalDimensions) { + if (!dimension.measured) { + reasons.push( + dimension.expectedN === 0 + ? `critical dimension ${dimension.dimension} has no applicable holdout histories` + : `critical dimension ${dimension.dimension} was measured on ${dimension.n}/${dimension.expectedN} applicable paired holdout cells`, + ) + } else if (dimension.regressed) { + reasons.push(`${dimension.dimension} may regress beyond ${tolerance}`) + } + } + return { + status: reasons.length === 0 ? 'promote' : 'hold', + reasons, + baselineScore: baselineRow.scoreMean, + winnerScore: winnerRow.scoreMean, + lift: winnerRow.scoreMean - baselineRow.scoreMean, + significance, + criticalDimensions, + } +} + +function applicableSequenceCount( + sequences: readonly AgentMemorySequence[], + dimension: string, +): number { + return sequences.filter((sequence) => + sequence.steps.some((step) => + (step.probes ?? []).some((probe) => probeAppliesToDimension(probe, dimension)), + ), + ).length +} + +function probeAppliesToDimension(probe: AgentMemorySequenceProbe, dimension: string): boolean { + switch (dimension) { + case 'memory_fact_recall': + case 'memory_required_fact_count': + case 'memory_matched_fact_count': + return Boolean( + (probe.requiredFacts && probe.requiredFacts.length > 0) || probe.referenceAnswer, + ) + case 'memory_event_recall': + return Boolean(probe.expectedEventIds && probe.expectedEventIds.length > 0) + case 'memory_actor_recall': + return Boolean(probe.expectedActorIds && probe.expectedActorIds.length > 0) + case 'memory_stale_safe': + case 'memory_stale_rate': + case 'memory_forbidden_fact_count': + case 'memory_matched_forbidden_fact_count': + return Boolean(probe.forbiddenFacts && probe.forbiddenFacts.length > 0) + default: + return true + } +} + +function normalizedPromotionPolicy( + options: RunAgentMemoryImprovementOptions, +): Record { + return { + significance: { + deltaThreshold: options.significance?.deltaThreshold ?? 0, + minProductiveRuns: options.significance?.minProductiveRuns ?? 3, + confidence: options.significance?.confidence ?? 0.95, + resamples: options.significance?.resamples ?? 2000, + seed: options.significance?.seed ?? 1337, + statistic: options.significance?.statistic ?? 'mean', + }, + criticalDimensions: [...(options.criticalDimensions ?? DEFAULT_CRITICAL_DIMENSIONS)], + criticalDimensionTolerance: options.criticalDimensionTolerance ?? 0.05, + minHoldoutScore: options.minHoldoutScore ?? 0, + } +} + +function pairedArtifacts( + result: RunAgentMemoryExperimentResult, + baselineId: string, + winnerId: string, + select: (artifact: AgentMemorySequenceArtifact) => number | undefined, +): PairedHoldout { + const baseline = artifactValues(result, baselineId, select) + const winner = artifactValues(result, winnerId, select) + const keys = [...baseline.keys()].filter((key) => winner.has(key)).sort() + return { + before: keys.map((key) => baseline.get(key)!), + after: keys.map((key) => winner.get(key)!), + cellIds: keys, + } +} + +function artifactValues( + result: RunAgentMemoryExperimentResult, + candidateId: string, + select: (artifact: AgentMemorySequenceArtifact) => number | undefined, +): Map { + const values = new Map() + for (const cell of result.campaign.cells) { + if (cell.error || cell.artifact.candidateId !== candidateId) continue + const value = select(cell.artifact) + if (value === undefined || !Number.isFinite(value)) continue + values.set(`${cell.artifact.sequenceId}:${cell.rep}`, value) + } + return values +} + +function sequenceScores( + result: RunAgentMemoryExperimentResult, + sequences: readonly AgentMemorySequence[], + candidateId: string, +): number[] { + const bySequence = new Map() + for (const cell of result.campaign.cells) { + if (cell.error || cell.artifact.candidateId !== candidateId) continue + const bucket = bySequence.get(cell.artifact.sequenceId) ?? [] + bucket.push(cell.artifact.score) + bySequence.set(cell.artifact.sequenceId, bucket) + } + return sequences.map((sequence) => mean(bySequence.get(sequence.id) ?? [])) +} + +function requireStringSurface(surface: MutableSurface): string { + if (typeof surface !== 'string' || surface.trim().length === 0) { + throw new Error('memory config proposer must return a JSON string surface') + } + return surface +} + +function serializeMemoryConfig( + serialize: (config: TConfig) => string, + config: TConfig, +): string { + let surface: unknown + try { + surface = serialize(config) + } catch (error) { + throw new Error('memory config serializer failed', { cause: error }) + } + if (typeof surface !== 'string' || surface.trim().length === 0) { + throw new Error('memory config serializer must return a non-empty string') + } + return surface +} + +function parseMemoryConfig(parse: (surface: string) => TConfig, surface: string): TConfig { + try { + const config = parse(surface) + if (config === undefined) throw new Error('parser returned undefined') + return config + } catch (error) { + throw new Error('memory config surface could not be parsed', { cause: error }) + } +} + +function assertMemoryConfigRoundTrip( + config: TConfig, + serialize: (config: TConfig) => string, + parse: (surface: string) => TConfig, +): void { + const first = serialize(config) + const second = serialize(parse(first)) + if (first !== second) { + throw new Error('memory config serializer and parser must round-trip seed configs exactly') + } +} + +function memorySequenceFingerprint(sequence: AgentMemorySequence): string { + const { id: _id, split: _split, tags: _tags, ...content } = sequence + return surfaceHash(canonicalJson(content)) +} + +function assertMemoryImprovementOptions( + options: RunAgentMemoryImprovementOptions, +): void { + for (const [name, value] of [ + ['experimentId', options.experimentId], + ['runDir', options.runDir], + ] as const) { + if (typeof value !== 'string' || !value.trim()) { + throw new Error(`memory improvement ${name} must be a non-empty string`) + } + } + if (typeof options.proposer?.kind !== 'string' || !options.proposer.kind.trim()) { + throw new Error('memory improvement proposer.kind must be a non-empty string') + } + if (typeof options.proposer?.propose !== 'function') { + throw new Error('memory improvement proposer.propose must be a function') + } + if (options.governor !== undefined && typeof options.governor.decide !== 'function') { + throw new Error('memory improvement governor.decide must be a function') + } + if (options.activation !== undefined) { + if (typeof options.activation.ref !== 'string' || !options.activation.ref.trim()) { + throw new Error('memory improvement activation.ref must be a non-empty string') + } + if (typeof options.activation.readCurrent !== 'function') { + throw new Error('memory improvement activation.readCurrent must be a function') + } + if (typeof options.activation.compareAndSet !== 'function') { + throw new Error('memory improvement activation.compareAndSet must be a function') + } + } + for (const [name, proposer] of Object.entries(options.proposers ?? {})) { + if (!name.trim()) throw new Error('memory improvement proposer labels must be non-empty') + if ( + !proposer || + typeof proposer.kind !== 'string' || + !proposer.kind.trim() || + typeof proposer.propose !== 'function' + ) { + throw new Error(`memory improvement proposer '${name}' is invalid`) + } + } + if (options.serializeConfig !== undefined && typeof options.serializeConfig !== 'function') { + throw new Error('memory improvement serializeConfig must be a function') + } + if (options.parseConfig !== undefined && typeof options.parseConfig !== 'function') { + throw new Error('memory improvement parseConfig must be a function') + } + if (!Number.isSafeInteger(options.budget.maxSteps) || options.budget.maxSteps < 0) { + throw new Error('memory improvement budget.maxSteps must be a non-negative safe integer') + } + for (const [name, value] of [ + ['populationSize', options.populationSize], + ['candidateConcurrency', options.candidateConcurrency], + ['sequenceConcurrency', options.sequenceConcurrency], + ['reps', options.reps], + ['maxRecoveryAttempts', options.maxRecoveryAttempts], + ['maxRecoveryRetriesPerAttempt', options.maxRecoveryRetriesPerAttempt], + ] as const) { + if (value !== undefined && (!Number.isSafeInteger(value) || value <= 0)) { + throw new Error(`memory improvement ${name} must be a positive safe integer`) + } + } + if ( + options.maxTotalCostUsd !== undefined && + (!Number.isFinite(options.maxTotalCostUsd) || options.maxTotalCostUsd < 0) + ) { + throw new Error('memory improvement maxTotalCostUsd must be a non-negative finite number') + } + if ( + options.activationTimeoutMs !== undefined && + (!Number.isSafeInteger(options.activationTimeoutMs) || options.activationTimeoutMs <= 0) + ) { + throw new Error('memory improvement activationTimeoutMs must be a positive safe integer') + } + const tolerance = options.criticalDimensionTolerance + if (tolerance !== undefined && (!Number.isFinite(tolerance) || tolerance < 0 || tolerance > 1)) { + throw new Error('memory improvement criticalDimensionTolerance must be between 0 and 1') + } + const minimum = options.minHoldoutScore + if (minimum !== undefined && (!Number.isFinite(minimum) || minimum < 0 || minimum > 1)) { + throw new Error('memory improvement minHoldoutScore must be between 0 and 1') + } + for (const seed of options.seeds) { + if ( + typeof seed.track !== 'string' || + !seed.track.trim() || + typeof seed.proposer !== 'string' || + !seed.proposer.trim() + ) { + throw new Error('memory improvement seeds require non-empty track and proposer values') + } + if (seed.vision !== undefined && (typeof seed.vision !== 'string' || !seed.vision.trim())) { + throw new Error('memory improvement seed vision must be a non-empty string when provided') + } + } + const dimensions = options.criticalDimensions ?? DEFAULT_CRITICAL_DIMENSIONS + if (dimensions.some((dimension) => typeof dimension !== 'string' || !dimension.trim())) { + throw new Error('memory improvement criticalDimensions must contain non-empty names') + } + if (new Set(dimensions).size !== dimensions.length) { + throw new Error('memory improvement criticalDimensions must be unique') + } + assertDistinctSequenceContent(options.trainSequences, 'train') + assertDistinctSequenceContent(options.holdoutSequences, 'holdout') +} + +function assertDistinctSequenceContent( + sequences: readonly AgentMemorySequence[], + split: string, +): void { + const idsByFingerprint = new Map() + for (const sequence of sequences) { + const fingerprint = memorySequenceFingerprint(sequence) + const prior = idsByFingerprint.get(fingerprint) + if (prior) { + throw new Error( + `memory improvement ${split} histories duplicate content: ${prior}/${sequence.id}`, + ) + } + idsByFingerprint.set(fingerprint, sequence.id) + } +} + +function mean(values: readonly number[]): number { + return values.length === 0 ? 0 : values.reduce((sum, value) => sum + value, 0) / values.length +} diff --git a/src/memory/index.ts b/src/memory/index.ts index da6fffb..6820790 100644 --- a/src/memory/index.ts +++ b/src/memory/index.ts @@ -1,6 +1,22 @@ export * from './adapter' +export * from './branch' +export * from './experiment' +export * from './graphiti' export * from './holdout' +export * from './improvement' +export { + AgentMemoryLifecycleTimeoutError, + AgentMemoryLifecycleUnsafeError, + createMemoryExecutionPool, + DEFAULT_MEMORY_CLEANUP_TIMEOUT_MS, + memoryRecoveryDelayMs, + resolveMemoryCleanupTimeoutMs, + runBoundedMemoryLifecycle, + sleepForMemoryRecovery, +} from './lifecycle' +export * from './mem0' export * from './neo4j' +export * from './run-control' export * from './schemas' export * from './source-record' export * from './types' diff --git a/src/memory/lifecycle.ts b/src/memory/lifecycle.ts new file mode 100644 index 0000000..1a8ab5e --- /dev/null +++ b/src/memory/lifecycle.ts @@ -0,0 +1,145 @@ +import type { AgentMemoryAdapter } from './types' + +export const DEFAULT_MEMORY_CLEANUP_TIMEOUT_MS = 180_000 + +export class AgentMemoryLifecycleTimeoutError extends Error { + constructor( + readonly operation: string, + readonly timeoutMs: number, + ) { + super(`${operation} did not finish within ${timeoutMs}ms`) + this.name = 'AgentMemoryLifecycleTimeoutError' + } +} + +export class AgentMemoryLifecycleUnsafeError extends Error { + constructor( + readonly operation: string, + readonly priorTimeout: AgentMemoryLifecycleTimeoutError, + ) { + super( + `${operation} cannot start because '${priorTimeout.operation}' is still running after its ${priorTimeout.timeoutMs}ms timeout`, + { cause: priorTimeout }, + ) + this.name = 'AgentMemoryLifecycleUnsafeError' + } +} + +const timedOutResources = new WeakMap() + +export function releaseMemoryAdapterCreatedAfterAbort(input: { + creation: Promise + signal: AbortSignal + dispose?: (adapter: AgentMemoryAdapter) => Promise +}): void { + void input.creation.then( + async (adapter) => { + if (!adapter || !input.signal.aborted) return + try { + await adapter.close?.() + } catch {} + try { + await input.dispose?.(adapter) + } catch {} + }, + () => undefined, + ) +} + +export function resolveMemoryCleanupTimeoutMs(value: number | undefined, label: string): number { + const timeoutMs = value ?? DEFAULT_MEMORY_CLEANUP_TIMEOUT_MS + if (!Number.isSafeInteger(timeoutMs) || timeoutMs <= 0) { + throw new Error(`${label} cleanupTimeoutMs must be a positive safe integer`) + } + return timeoutMs +} + +export async function runBoundedMemoryLifecycle(input: { + operation: string + timeoutMs: number + /** Prevent later operations from using the same client while timed-out work may still be running. */ + resource?: object + /** Cooperatively cancel provider work before reporting a timeout. */ + abortController?: AbortController + run(): Promise | T +}): Promise { + const priorTimeout = input.resource ? timedOutResources.get(input.resource) : undefined + if (priorTimeout) throw new AgentMemoryLifecycleUnsafeError(input.operation, priorTimeout) + let timeout: ReturnType | undefined + let timeoutError: AgentMemoryLifecycleTimeoutError | undefined + const work = Promise.resolve().then(input.run) + if (input.resource) { + const resource = input.resource + void work.then( + () => { + if (timeoutError && timedOutResources.get(resource) === timeoutError) { + timedOutResources.delete(resource) + } + }, + () => { + if (timeoutError && timedOutResources.get(resource) === timeoutError) { + timedOutResources.delete(resource) + } + }, + ) + } + const deadline = new Promise((_, reject) => { + timeout = setTimeout(() => { + timeoutError = new AgentMemoryLifecycleTimeoutError(input.operation, input.timeoutMs) + if (input.resource) timedOutResources.set(input.resource, timeoutError) + input.abortController?.abort(timeoutError) + reject(timeoutError) + }, input.timeoutMs) + }) + try { + return await Promise.race([work, deadline]) + } finally { + if (timeout) clearTimeout(timeout) + } +} + +export function memoryRecoveryDelayMs(adapter: AgentMemoryAdapter): number { + const isolation = adapter.branchIsolation + if (isolation?.mode !== 'scoped' || isolation.processExitSafe !== false) return 0 + const delayMs = isolation.recoveryDelayMs + if (typeof delayMs !== 'number' || !Number.isSafeInteger(delayMs) || delayMs <= 0) { + throw new Error( + `${adapter.id}: processExitSafe=false requires a positive recoveryDelayMs for abandoned writes`, + ) + } + return delayMs +} + +export async function sleepForMemoryRecovery( + delayMs: number, + assertOwned: () => Promise, + timeoutMs = delayMs, + operation = 'memory recovery visibility wait', +): Promise { + if (delayMs <= 0) return + if (!Number.isSafeInteger(timeoutMs) || timeoutMs <= 0) { + throw new Error(`${operation} timeout must be a positive safe integer`) + } + await new Promise((resolve) => setTimeout(resolve, Math.min(delayMs, timeoutMs))) + await assertOwned() + if (delayMs > timeoutMs) throw new AgentMemoryLifecycleTimeoutError(operation, timeoutMs) +} + +export function createMemoryExecutionPool(limit: number): { + run(operation: () => Promise): Promise +} { + let active = 0 + const waiters: Array<() => void> = [] + return { + async run(operation: () => Promise): Promise { + if (active >= limit) await new Promise((resolve) => waiters.push(resolve)) + active += 1 + try { + return await operation() + } finally { + active -= 1 + waiters.shift()?.() + } + }, + } +} diff --git a/src/memory/mem0.ts b/src/memory/mem0.ts new file mode 100644 index 0000000..d390fa6 --- /dev/null +++ b/src/memory/mem0.ts @@ -0,0 +1,678 @@ +import { randomUUID } from 'node:crypto' +import { canonicalJson } from '@tangle-network/agent-eval' +import { stableId } from '../ids' +import { defaultGetMemoryContext } from './adapter' +import { memoryWriteResultToSourceRecord } from './source-record' +import type { + AgentMemoryAdapter, + AgentMemoryHit, + AgentMemoryKind, + AgentMemoryScope, + AgentMemorySearchOptions, + AgentMemoryWriteInput, +} from './types' + +export type Mem0ClientMode = 'hosted' | 'oss' + +export interface Mem0ClientLike { + add( + messages: Array<{ role: string; content: string }>, + options?: Record, + ): Promise + search(query: string, options?: Record): Promise + getAll?(options?: Record): Promise + delete?(memoryId: string, options?: Record): Promise +} + +export interface Mem0MemoryAdapterOptions { + client: Mem0ClientLike + mode: Mem0ClientMode + id?: string + appId?: string + infer?: boolean + rerank?: boolean + latestOnly?: boolean + /** Bounds delayed-delete visibility checks and abandoned hosted-write recovery waits. */ + ingestionTimeoutMs?: number + pollIntervalMs?: number + /** Stable deployment/account identity for cache-key helpers. Never put credentials here. */ + backendRef?: string + defaultScope?: AgentMemoryScope +} + +interface Mem0PendingWriteProbe { + text: string + providerIds: Set + filters: Record + expiresAt: number +} + +/** Connects both the hosted Mem0 client and the open-source `Memory` class. */ +export function createMem0MemoryAdapter(options: Mem0MemoryAdapterOptions): AgentMemoryAdapter { + assertMem0Options(options) + const id = options.id ?? `mem0-${options.mode}` + const pendingWrites = new Set() + + const adapter: AgentMemoryAdapter = { + id, + branchIsolation: + options.mode === 'oss' + ? { mode: 'scoped' } + : { + mode: 'scoped', + processExitSafe: false, + recoveryDelayMs: options.ingestionTimeoutMs ?? 30_000, + }, + async search(query, searchOptions = {}) { + assertMem0SearchOptions(searchOptions, id) + if (searchOptions.limit === 0) return [] + const scope = mergeScopes(options.defaultScope, searchOptions.scope) + assertMem0ProviderScope(scope, options.mode, id, 'search') + const requestedKinds = [...new Set(searchOptions.kinds ?? [])] + const kindQueries = requestedKinds.length > 0 ? requestedKinds : [undefined] + const payloads = await Promise.all( + kindQueries.map((kind) => + options.client.search(query, { + filters: mem0Filters(scope, options.appId, kind), + topK: searchOptions.limit, + threshold: searchOptions.minScore, + ...(options.rerank !== undefined ? { rerank: options.rerank } : {}), + ...(options.mode === 'hosted' && options.latestOnly !== undefined + ? { latestOnly: options.latestOnly } + : {}), + }), + ), + ) + return mergeMem0Hits( + payloads.flatMap((raw) => normalizeMem0Hits(raw, id, searchOptions)), + searchOptions.limit, + ) + }, + async getContext(query, searchOptions = {}) { + return defaultGetMemoryContext(adapter, query, searchOptions) + }, + async write(input) { + const scope = mergeScopes(options.defaultScope, input.scope) + assertMem0ProviderScope(scope, options.mode, id, 'write') + pruneExpiredMem0PendingWrites(pendingWrites) + const writeId = input.id ?? randomUUID() + const writeFilters = mem0Filters(scope, options.appId) + const pendingProbe: Mem0PendingWriteProbe = { + text: input.text, + providerIds: new Set(), + filters: writeFilters, + expiresAt: Number.POSITIVE_INFINITY, + } + pendingWrites.add(pendingProbe) + const scopeMetadata = mem0ScopeMetadata(scope, options.appId) + const metadata = compactRecord({ + ...input.metadata, + agentKnowledgeWriteId: writeId, + memoryKind: input.kind, + memoryTitle: input.title, + entityName: input.entityName, + entityType: input.entityType, + category: input.category, + predicate: input.predicate, + subject: input.subject, + object: input.object, + confidence: input.confidence, + originalRole: input.role, + ...scopeMetadata, + }) + const entity = mem0Entity(scope, options.appId) + let raw: unknown + try { + raw = await options.client.add([{ role: mem0Role(input.role), content: input.text }], { + ...entity, + metadata, + infer: options.infer ?? true, + ...(options.mode === 'oss' ? { filters: writeFilters } : {}), + }) + } finally { + pendingProbe.expiresAt = Date.now() + (options.ingestionTimeoutMs ?? 30_000) + } + if (options.mode === 'hosted' && !Array.isArray(raw)) { + throw new Error( + `${id}: hosted Mem0 add returned an unsupported response; mem0ai 3.x must return a memory array`, + ) + } + const items = extractMem0Items(raw) + const mutations = items.filter((candidate) => mem0Event(candidate) !== 'NOOP') + const durableMutations = mutations.filter((candidate) => mem0Event(candidate) !== 'DELETE') + for (const candidate of durableMutations) { + const providerId = stringField(candidate, ['id']) + if (providerId) pendingProbe.providerIds.add(providerId) + } + if (durableMutations.length === 0) { + pendingWrites.delete(pendingProbe) + } + const item = mutations[0] ?? items[0] + const itemId = stringField(item, ['id']) ?? writeId + const event = mem0Event(item) + const events = [...new Set(mutations.map(mem0Event).filter(Boolean))] + const result = { + accepted: mutations.length > 0, + id: itemId, + uri: `memory://${id}/${encodeURIComponent(itemId)}`, + kind: input.kind, + metadata: { + provider: 'mem0', + mode: options.mode, + ...(event ? { event } : {}), + ...(events.length > 1 ? { events } : {}), + }, + } as const + return { + ...result, + sourceRecord: memoryWriteResultToSourceRecord(result, input.text, { scope }), + } + }, + async clear(scope) { + const mergedScope = mergeScopes(options.defaultScope, scope) + assertMem0ProviderScope(mergedScope, options.mode, id, 'clear') + pruneExpiredMem0PendingWrites(pendingWrites) + const clearFilters = mem0Filters(mergedScope, options.appId) + if (Object.keys(clearFilters).length === 0) { + throw new Error(`${id}: refusing an unscoped Mem0 clear`) + } + if (options.client.getAll && options.client.delete) { + const pendingForClear = [...pendingWrites].filter((pending) => + mem0FiltersInclude(pending.filters, clearFilters), + ) + const deletedIds = new Set() + const searchProbes = new Map>() + const observedPending = new Set() + for (const pending of pendingForClear) { + const ids = searchProbes.get(pending.text) ?? new Set() + for (const providerId of pending.providerIds) ids.add(providerId) + searchProbes.set(pending.text, ids) + if (pending.providerIds.size > 0) observedPending.add(pending) + } + for (const pending of pendingForClear) { + for (const providerId of pending.providerIds) { + if (deletedIds.has(providerId)) continue + await options.client.delete(providerId) + deletedIds.add(providerId) + } + } + const visibilityTimeoutMs = options.ingestionTimeoutMs ?? 30_000 + const pollIntervalMs = options.pollIntervalMs ?? 250 + let visibilityDeadline = Date.now() + visibilityTimeoutMs + for (let batch = 0; batch < 100; ) { + const raw = await options.client.getAll( + options.mode === 'hosted' + ? { + filters: mem0Filters(mergedScope, options.appId), + page: 1, + pageSize: 100, + latestOnly: false, + showExpired: true, + } + : { + filters: mem0Filters(mergedScope, options.appId), + topK: 100, + showExpired: true, + }, + ) + const items = extractMem0Items(raw) + if (items.length === 0) { + const searchableByQuery = await searchableMem0IdsForCleanup( + options, + clearFilters, + searchProbes, + ) + const searchableIds = new Set( + [...searchableByQuery.values()].flatMap((ids) => [...ids]), + ) + const unseenSearchIds = [...searchableIds].filter( + (memoryId) => !deletedIds.has(memoryId), + ) + for (const [query, ids] of searchableByQuery) { + const known = searchProbes.get(query) ?? new Set() + for (const memoryId of ids) known.add(memoryId) + searchProbes.set(query, known) + if (ids.size > 0) { + for (const pending of pendingForClear) { + if (pending.text === query) observedPending.add(pending) + } + } + } + if (unseenSearchIds.length > 0) { + for (const memoryId of unseenSearchIds) { + await options.client.delete(memoryId) + deletedIds.add(memoryId) + } + batch += 1 + visibilityDeadline = Date.now() + visibilityTimeoutMs + continue + } + const unresolvedPending = pendingForClear.some( + (pending) => !observedPending.has(pending), + ) + const needsQuietWindow = pendingForClear.some( + (pending) => pending.providerIds.size === 0, + ) + const remainingMs = visibilityDeadline - Date.now() + if (remainingMs <= 0) { + if (!unresolvedPending && searchableIds.size === 0) { + removeMem0PendingWrites(pendingWrites, pendingForClear) + return + } + if (searchableIds.size > 0) { + throw new Error( + `${id}: deleted Mem0 memories remained searchable after ${visibilityTimeoutMs}ms`, + ) + } + throw new Error( + `${id}: a Mem0 write never became visible for exact cleanup within ${visibilityTimeoutMs}ms`, + ) + } + if (!unresolvedPending && searchableIds.size === 0 && !needsQuietWindow) { + removeMem0PendingWrites(pendingWrites, pendingForClear) + return + } + await sleep(Math.min(pollIntervalMs, remainingMs)) + continue + } + const rawIds = items.map((item) => stringField(item, ['id'])) + if (!rawIds.every((memoryId): memoryId is string => memoryId !== undefined)) { + throw new Error(`${id}: Mem0 returned memories without ids during scoped clear`) + } + const ids = [...new Set(rawIds)] + for (const item of items) { + const memoryId = stringField(item, ['id'])! + const text = + stringField(item, ['memory', 'text', 'content']) ?? + stringField(recordField(item, 'data'), ['memory', 'text', 'content']) + if (!text) continue + const idsForText = searchProbes.get(text) ?? new Set() + idsForText.add(memoryId) + searchProbes.set(text, idsForText) + for (const pending of pendingForClear) { + if (pending.text === text || pending.providerIds.has(memoryId)) { + observedPending.add(pending) + } + } + } + const unseenIds = ids.filter((memoryId) => !deletedIds.has(memoryId)) + if (unseenIds.length === 0) { + const remainingMs = visibilityDeadline - Date.now() + if (remainingMs <= 0) { + throw new Error( + `${id}: deleted Mem0 memories remained visible after ${visibilityTimeoutMs}ms`, + ) + } + await sleep(Math.min(pollIntervalMs, remainingMs)) + continue + } + for (const memoryId of unseenIds) { + await options.client.delete(memoryId) + deletedIds.add(memoryId) + } + batch += 1 + visibilityDeadline = Date.now() + visibilityTimeoutMs + } + throw new Error( + `${id}: refused to clear more than ${deletedIds.size} Mem0 memories in one call`, + ) + } + throw new Error(`${id}: exact scoped clear requires Mem0 getAll plus per-memory delete`) + }, + } + return adapter +} + +function assertMem0Options(options: Mem0MemoryAdapterOptions): void { + if (options.id !== undefined && (typeof options.id !== 'string' || !options.id.trim())) { + throw new Error('Mem0 id must be a non-empty string') + } + if (options.appId !== undefined && (typeof options.appId !== 'string' || !options.appId.trim())) { + throw new Error('Mem0 appId must be a non-empty string') + } + if ( + options.backendRef !== undefined && + (typeof options.backendRef !== 'string' || !options.backendRef.trim()) + ) { + throw new Error('Mem0 backendRef must be a non-empty string') + } + for (const [name, value] of [ + ['ingestionTimeoutMs', options.ingestionTimeoutMs], + ['pollIntervalMs', options.pollIntervalMs], + ] as const) { + if (value !== undefined && (!Number.isSafeInteger(value) || value <= 0)) { + throw new Error(`Mem0 ${name} must be a positive safe integer`) + } + } +} + +function assertMem0SearchOptions(options: AgentMemorySearchOptions, adapterId: string): void { + if (options.limit !== undefined && (!Number.isSafeInteger(options.limit) || options.limit < 0)) { + throw new Error(`${adapterId}: search limit must be a non-negative safe integer`) + } + if (options.minScore !== undefined && !Number.isFinite(options.minScore)) { + throw new Error(`${adapterId}: minScore must be finite`) + } +} + +function assertMem0ProviderScope( + scope: AgentMemoryScope, + mode: Mem0ClientMode, + adapterId: string, + operation: 'search' | 'write' | 'clear', +): void { + if (scope.userId || scope.agentId || scope.runId || scope.namespace) return + if (operation === 'clear') { + throw new Error( + `${adapterId}: refusing an unscoped Mem0 clear; provide scope.userId, scope.agentId, scope.runId, or scope.namespace`, + ) + } + throw new Error( + `${adapterId}: Mem0 ${mode} ${operation} requires scope.userId, scope.agentId, scope.runId, or scope.namespace`, + ) +} + +function normalizeMem0Hits( + raw: unknown, + adapterId: string, + options: AgentMemorySearchOptions, +): AgentMemoryHit[] { + const limit = options.limit ?? Number.POSITIVE_INFINITY + const kinds = options.kinds && options.kinds.length > 0 ? new Set(options.kinds) : null + return extractMem0Items(raw) + .map((item): AgentMemoryHit | undefined => { + const text = + stringField(item, ['memory', 'text', 'content']) ?? + stringField(recordField(item, 'data'), ['memory', 'text', 'content']) + if (!text) return undefined + const metadata = recordField(item, 'metadata') + const kind = mem0Kind(metadata?.memoryKind ?? item?.memoryType) + if (kinds && !kinds.has(kind)) return undefined + const score = numberField(item, ['rerankScore', 'rerank_score', 'score']) + if (options.minScore !== undefined && (score === undefined || score < options.minScore)) { + return undefined + } + const memoryId = + stringField(item, ['id']) ?? + stableId('mem', canonicalJson(compactRecord({ adapterId, text, metadata, item }))) + const normalizedScore = score !== undefined && score >= 0 && score <= 1 ? score : undefined + return { + id: memoryId, + uri: `memory://${adapterId}/${encodeURIComponent(memoryId)}`, + kind, + text, + ...(stringField(metadata, ['memoryTitle']) ? { title: String(metadata!.memoryTitle) } : {}), + ...(score !== undefined ? { score } : {}), + ...(normalizedScore !== undefined ? { normalizedScore } : {}), + ...(dateField(item, ['createdAt', 'created_at']) + ? { createdAt: dateField(item, ['createdAt', 'created_at']) } + : {}), + ...(dateField(item, ['expirationDate', 'expiration_date']) + ? { validUntil: dateField(item, ['expirationDate', 'expiration_date']) } + : {}), + metadata: compactRecord({ + provider: 'mem0', + ...metadata, + categories: item?.categories, + owner: item?.owner, + runId: item?.runId, + agentId: item?.agentId, + userId: item?.userId, + }), + } + }) + .filter((hit): hit is AgentMemoryHit => hit !== undefined) + .slice(0, limit) +} + +function mergeMem0Hits(hits: readonly AgentMemoryHit[], limit?: number): AgentMemoryHit[] { + const unique = new Map() + for (const [index, hit] of hits.entries()) { + const key = JSON.stringify([hit.uri, hit.text]) + const score = hit.normalizedScore ?? hit.score + const finiteScore = score !== undefined && Number.isFinite(score) ? score : undefined + const prior = unique.get(key) + if (!prior) { + unique.set(key, { hit, index, score: finiteScore }) + } else if ( + finiteScore !== undefined && + (prior.score === undefined || finiteScore > prior.score) + ) { + unique.set(key, { hit, index: prior.index, score: finiteScore }) + } + } + return [...unique.values()] + .sort( + (left, right) => + Number(right.score !== undefined) - Number(left.score !== undefined) || + (right.score ?? 0) - (left.score ?? 0) || + left.index - right.index, + ) + .map(({ hit }) => hit) + .slice(0, limit ?? Number.POSITIVE_INFINITY) +} + +function extractMem0Items(raw: unknown): Array> { + if (Array.isArray(raw)) return raw.filter(isRecord) + if (!isRecord(raw)) return [] + for (const key of ['results', 'memories', 'data']) { + if (Array.isArray(raw[key])) return raw[key].filter(isRecord) + } + return [raw] +} + +function mem0Event(item: Record | undefined): string | undefined { + return ( + stringField(item, ['event']) ?? stringField(recordField(item, 'metadata'), ['event']) + )?.toUpperCase() +} + +function mem0Entity(scope: AgentMemoryScope, appId?: string): Record { + return compactStringRecord({ + userId: scope.userId, + agentId: scope.agentId, + appId, + runId: scope.namespace ?? scope.runId, + }) +} + +function mem0Filters( + scope: AgentMemoryScope, + appId: string | undefined, + kind?: AgentMemoryKind, +): Record { + const entity = mem0EntityFilters(scope, appId) + const customMetadata = compactRecord({ + ...mem0CustomScopeMetadata(scope), + memoryKind: kind, + }) + return compactRecord({ ...entity, ...customMetadata }) +} + +function mem0ScopeMetadata(scope: AgentMemoryScope, appId?: string): Record { + return { ...mem0EntityFilters(scope, appId), ...mem0CustomScopeMetadata(scope) } +} + +function mem0EntityFilters(scope: AgentMemoryScope, appId?: string): Record { + return compactStringRecord({ + user_id: scope.userId, + agent_id: scope.agentId, + run_id: scope.namespace ?? scope.runId, + app_id: appId, + }) +} + +function mem0CustomScopeMetadata(scope: AgentMemoryScope): Record { + const metadata = compactStringRecord({ + logical_run_id: scope.namespace ? scope.runId : undefined, + tenant_id: scope.tenantId, + team_id: scope.teamId, + session_id: scope.sessionId, + }) + for (const [key, value] of Object.entries(scope.tags ?? {})) { + metadata[`tag_${key}`] = value + } + return metadata +} + +async function searchableMem0IdsForCleanup( + options: Mem0MemoryAdapterOptions, + filters: Record, + probes: ReadonlyMap>, +): Promise>> { + const searchable = new Map>() + const entries = [...probes] + for (let offset = 0; offset < entries.length; offset += 8) { + const results = await Promise.all( + entries.slice(offset, offset + 8).map(async ([query, expectedIds]) => { + const raw = await options.client.search(query, { + filters, + topK: Math.min(100, Math.max(10, expectedIds.size)), + ...(options.mode === 'hosted' ? { latestOnly: false, showExpired: true } : {}), + }) + return extractMem0Items(raw).map((item) => { + const memoryId = stringField(item, ['id']) + if (!memoryId) { + throw new Error( + `${options.id ?? `mem0-${options.mode}`}: Mem0 search returned a memory without an id during clear verification`, + ) + } + return memoryId + }) + }), + ) + for (const [index, ids] of results.entries()) { + const query = entries[offset + index]![0] + searchable.set(query, new Set(ids)) + } + } + return searchable +} + +function mem0FiltersInclude( + candidate: Readonly>, + requested: Readonly>, +): boolean { + return Object.entries(requested).every(([key, value]) => Object.is(candidate[key], value)) +} + +function removeMem0PendingWrites( + pendingWrites: Set, + removed: readonly Mem0PendingWriteProbe[], +): void { + for (const pending of removed) pendingWrites.delete(pending) +} + +function pruneExpiredMem0PendingWrites(pendingWrites: Set): void { + const now = Date.now() + for (const pending of pendingWrites) { + if (pending.expiresAt <= now) pendingWrites.delete(pending) + } +} + +function mem0Role(role: AgentMemoryWriteInput['role']): 'user' | 'assistant' { + return role === 'assistant' || role === 'tool' ? 'assistant' : 'user' +} + +function mem0Kind(value: unknown): AgentMemoryKind { + switch (value) { + case 'message': + case 'entity': + case 'fact': + case 'preference': + case 'observation': + case 'reasoning-trace': + return value + default: + return 'fact' + } +} + +function mergeScopes(base?: AgentMemoryScope, extra?: AgentMemoryScope): AgentMemoryScope { + return { + ...(base ?? {}), + ...(extra ?? {}), + tags: { ...(base?.tags ?? {}), ...(extra?.tags ?? {}) }, + } +} + +function sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)) +} + +function compactRecord(input: Record): Record { + return Object.fromEntries(Object.entries(input).filter(([, value]) => value !== undefined)) +} + +function compactStringRecord(input: Record): Record { + return Object.fromEntries( + Object.entries(input).filter((entry): entry is [string, string] => entry[1] !== undefined), + ) +} + +function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value) +} + +function recordField( + value: Record | undefined, + key: string, +): Record | undefined { + const field = value?.[key] + return isRecord(field) ? field : undefined +} + +function stringField( + value: Record | undefined, + keys: readonly string[], +): string | undefined { + for (const key of keys) { + const field = value?.[key] + if (typeof field === 'string' && field.length > 0) return field + } + return undefined +} + +function numberField( + value: Record | undefined, + keys: readonly string[], +): number | undefined { + for (const key of keys) { + const field = value?.[key] + if (typeof field === 'number' && Number.isFinite(field)) return field + } + return undefined +} + +function dateField( + value: Record | undefined, + keys: readonly string[], +): string | undefined { + for (const key of keys) { + const field = value?.[key] + if (typeof field === 'string' && field.length > 0) return field + if (field instanceof Date && !Number.isNaN(field.valueOf())) return field.toISOString() + } + return undefined +} + +/** Stable identity useful for dispatch/cache keys without exposing client credentials. */ +export function mem0MemoryAdapterIdentity( + options: Pick< + Mem0MemoryAdapterOptions, + | 'mode' + | 'id' + | 'appId' + | 'infer' + | 'rerank' + | 'latestOnly' + | 'ingestionTimeoutMs' + | 'pollIntervalMs' + | 'defaultScope' + > & { backendRef: string }, +): string { + if (typeof options.backendRef !== 'string' || !options.backendRef.trim()) { + throw new Error('Mem0 backendRef must be a non-empty string') + } + return stableId('mem0', canonicalJson(compactRecord(options))) +} diff --git a/src/memory/neo4j.ts b/src/memory/neo4j.ts index a6e73db..cb3b40a 100644 --- a/src/memory/neo4j.ts +++ b/src/memory/neo4j.ts @@ -1,6 +1,8 @@ +import { canonicalJson } from '@tangle-network/agent-eval' import { stableId } from '../ids' import { defaultGetMemoryContext } from './adapter' import { emitRetrievalHoldoutBypass } from './holdout' +import { mergeRankedMemoryHits } from './rank' import { memoryHitToSourceRecord, memoryWriteResultToSourceRecord } from './source-record' import type { AgentMemoryAdapter, @@ -12,218 +14,315 @@ import type { } from './types' export interface Neo4jAgentMemoryAdapterOptions { - client: Record + client: object + /** Match the transport passed to the official MemoryClient. */ + transport: 'rest' | 'bridge' + /** Use Neo4j's whole-conversation context instead of query-based search. */ + contextMode?: 'search' | 'native' id?: string + /** Assert that the client owns disposable external state for this exact branch. */ + branchId?: string } export function createNeo4jAgentMemoryAdapter( options: Neo4jAgentMemoryAdapterOptions, ): AgentMemoryAdapter { - const client = options.client + assertNeo4jOptions(options) + const client = options.client as Record const id = options.id ?? 'neo4j-agent-memory' - return { + const isolatedInstance = options.branchId !== undefined + const adapter: AgentMemoryAdapter = { id, + branchIsolation: options.branchId + ? { mode: 'instance', branchId: options.branchId, supportsLogicalScopes: false } + : { + mode: 'unsupported', + reason: 'create a separate MemoryClient namespace per branch and pass branchId', + }, async search(query, searchOptions = {}) { - const sdkHits = await searchNeo4jMemory(client, query, searchOptions, id) - if (sdkHits.length > 0) return sdkHits - - const result = await callOptional( + assertNeo4jSearchOptions(searchOptions, id) + return searchNeo4jMemory( client, - ['search', 'memory_search'], - [query, neo4jOptions(searchOptions)], + query, + searchOptions, + id, + options.transport, + isolatedInstance, ) - if (result !== undefined) return normalizeHits(result, searchOptions, id) - - const context = await callOptional( - client, - ['getContext', 'get_context'], - [query, neo4jOptions(searchOptions)], - ) - if (context !== undefined) return normalizeHits(context, searchOptions, id) - return [] }, async getContext(query, searchOptions = {}) { + assertNeo4jSearchOptions(searchOptions, id) + assertNeo4jSearchKinds(searchOptions.kinds, options.transport, id) const shortTerm = nested(client, ['shortTerm', 'short_term'], {}) const sessionId = searchOptions.scope?.sessionId - if (sessionId) { - const conversationContext = await callOptional( - shortTerm, - ['getContext', 'get_context'], - [sessionId], - ) - if (conversationContext !== undefined) { - const hits = normalizeConversationContextHits(conversationContext, searchOptions, id) - const text = - textFromConversationContext(conversationContext) ?? - (typeof conversationContext === 'string' - ? conversationContext - : hits.length > 0 - ? renderHits(hits) - : '') - if (searchOptions.holdout) { - emitRetrievalHoldoutBypass( - hits, - searchOptions.holdout, - holdoutBypassContext(query, searchOptions), - 'short-term-context', - ) - } - return { - query, - text, - hits, - sourceRecords: hits.map((hit) => - memoryHitToSourceRecord(hit, { scope: searchOptions.scope }), - ), - metadata: { adapter: id, rawContext: conversationContext }, - } - } + if (options.contextMode === 'native' && options.transport !== 'rest') { + throw new Error(`${id}: Neo4j native context requires transport="rest"`) } - - const result = await callOptional( - client, - ['getContext', 'get_context'], - [query, neo4jOptions(searchOptions)], - ) - if (result === undefined) { - return defaultGetMemoryContext( - { - search: async () => { - const sdkHits = await searchNeo4jMemory(client, query, searchOptions, id) - if (sdkHits.length > 0) return sdkHits - const searchResult = await callOptional( - client, - ['search', 'memory_search'], - [query, neo4jOptions(searchOptions)], - ) - return normalizeHits(searchResult, searchOptions, id) - }, - }, - query, - searchOptions, + if (options.contextMode === 'native' && !sessionId) { + throw new Error(`${id}: Neo4j native context requires scope.sessionId`) + } + if ( + options.contextMode === 'native' && + searchOptions.kinds?.some((kind) => kind !== 'message' && kind !== 'observation') + ) { + throw new Error( + `${id}: Neo4j native context only returns message and observation memory kinds`, ) } - if (typeof result === 'string') { - const hit: AgentMemoryHit = { - id: stableId('mem', `${id}:${query}:${result}`), - uri: `memory://${id}/context/${stableId('ctx', query)}`, - kind: 'fact', - text: result, - title: 'Memory context', - normalizedScore: 1, - } + if (options.contextMode === 'native' && options.transport === 'rest' && sessionId) { + assertNeo4jScopeSupported( + searchOptions.scope, + ['sessionId'], + id, + 'native context read', + isolatedInstance, + ) + const conversationContext = await callRequired(shortTerm, ['getContext'], [sessionId]) + const hits = normalizeConversationContextHits(conversationContext, searchOptions, id) + const text = renderHits(hits) if (searchOptions.holdout) { emitRetrievalHoldoutBypass( - [hit], + hits, searchOptions.holdout, holdoutBypassContext(query, searchOptions), - 'raw-string-context', + 'short-term-context', ) } return { query, - text: result, - hits: [hit], - sourceRecords: [memoryHitToSourceRecord(hit, { scope: searchOptions.scope })], - metadata: { adapter: id }, + text, + hits, + sourceRecords: hits.map((hit) => + memoryHitToSourceRecord(hit, { scope: searchOptions.scope }), + ), + metadata: { adapter: id, rawContext: conversationContext }, } } - const hits = normalizeHits(result, searchOptions, id) - if (hits.length > 0) - return defaultGetMemoryContext({ search: async () => hits }, query, searchOptions) - return defaultGetMemoryContext({ search: async () => [] }, query, searchOptions) + + return defaultGetMemoryContext(adapter, query, searchOptions) }, async write(input) { - const result = await writeNeo4jMemory(client, input, id) + const result = await writeNeo4jMemory(client, input, id, options.transport, isolatedInstance) return { ...result, sourceRecord: memoryWriteResultToSourceRecord(result, input.text, { scope: input.scope }), } }, + async close() { + await callOptional(client, ['close'], []) + }, + } + return adapter +} + +function assertNeo4jOptions(options: Neo4jAgentMemoryAdapterOptions): void { + if (typeof options.client !== 'object' || options.client === null) { + throw new Error('Neo4j agent-memory client must be an object') + } + if (options.id !== undefined && (typeof options.id !== 'string' || !options.id.trim())) { + throw new Error('Neo4j agent-memory id must be a non-empty string') + } + if ( + options.branchId !== undefined && + (typeof options.branchId !== 'string' || !options.branchId.trim()) + ) { + throw new Error('Neo4j agent-memory branchId must be a non-empty string') + } + if (options.contextMode !== undefined && !['search', 'native'].includes(options.contextMode)) { + throw new Error('Neo4j agent-memory contextMode must be search or native') + } +} + +function assertNeo4jSearchOptions(options: AgentMemorySearchOptions, adapterId: string): void { + if (options.limit !== undefined && (!Number.isSafeInteger(options.limit) || options.limit < 0)) { + throw new Error(`${adapterId}: search limit must be a non-negative safe integer`) } + if (options.minScore !== undefined && !Number.isFinite(options.minScore)) { + throw new Error(`${adapterId}: minScore must be finite`) + } +} + +function assertNeo4jSearchKinds( + kinds: AgentMemorySearchOptions['kinds'], + transport: 'rest' | 'bridge', + adapterId: string, +): void { + if (!kinds?.length) return + const supported = new Set( + transport === 'rest' + ? ['message', 'observation', 'entity'] + : ['message', 'observation', 'entity', 'preference', 'reasoning-trace'], + ) + const unsupported = [...new Set(kinds.filter((kind) => !supported.has(kind)))] + if (unsupported.length > 0) { + throw new Error( + `${adapterId}: Neo4j ${transport} cannot search memory kinds: ${unsupported.join(', ')}`, + ) + } +} + +function assertNeo4jBridgeCapability( + transport: 'rest' | 'bridge', + kind: AgentMemoryWriteInput['kind'], + adapterId: string, +): void { + if (transport === 'bridge') return + throw new Error( + `${adapterId}: Neo4j REST cannot write ${kind}; use transport="bridge" or map it to an entity/message`, + ) } async function writeNeo4jMemory( client: Record, input: AgentMemoryWriteInput, adapterId: string, + transport: 'rest' | 'bridge', + isolatedInstance: boolean, ): Promise { const scope = input.scope ?? {} const sessionId = scope.sessionId ?? input.metadata?.sessionId + assertNeo4jScopeSupported( + input.scope, + input.kind === 'message' || input.kind === 'observation' + ? ['sessionId'] + : input.kind === 'reasoning-trace' + ? ['sessionId', 'runId'] + : [], + adapterId, + `${input.kind} write`, + isolatedInstance, + ) + if ( + !isolatedInstance && + input.kind === 'reasoning-trace' && + scope.sessionId !== undefined && + scope.runId !== undefined + ) { + throw new Error( + `${adapterId}: Neo4j reasoning write cannot enforce both sessionId and runId; provide one`, + ) + } let result: unknown - if (input.kind === 'message') { + if (input.kind === 'message' || input.kind === 'observation') { + if (transport === 'rest' && typeof sessionId !== 'string') { + throw new Error( + `${adapterId}: Neo4j REST message writes require scope.sessionId as the conversation id`, + ) + } const shortTerm = nested(client, ['shortTerm', 'short_term'], client) - result = await callOptional( + const metadata = { ...input.metadata, agentKnowledgeKind: input.kind } + result = await callRequired( shortTerm, ['addMessage'], [ sessionId, - input.role ?? 'user', + neo4jMessageRole(input.role), input.text, - { - metadata: input.metadata, - conversationId: sessionId, - userId: scope.userId, - }, + { metadata, conversationId: sessionId }, ], ) - if (result === undefined) { - result = await callRequired( - shortTerm, - ['add_message'], - [ - { - session_id: sessionId, - sessionId, - role: input.role ?? 'user', - content: input.text, - user_identifier: scope.userId, - userIdentifier: scope.userId, - metadata: input.metadata, - }, - ], - ) - } } else if (input.kind === 'entity') { const longTerm = nested(client, ['longTerm', 'long_term'], client) - result = await callOptional( + result = await callRequired( longTerm, ['addEntity'], [ input.entityName ?? input.title ?? input.text, - input.entityType ?? 'ENTITY', + input.entityType ?? 'custom', neo4jEntityOptions(input), ], ) - if (result === undefined) { - result = await callRequired( - longTerm, - ['add_entity'], - [ - input.entityName ?? input.title ?? input.text, - input.entityType ?? 'ENTITY', - neo4jWriteOptions(input), - ], - ) - } } else if (input.kind === 'preference') { + assertNeo4jBridgeCapability(transport, input.kind, adapterId) const longTerm = nested(client, ['longTerm', 'long_term'], client) - result = await callOptional( + result = await callRequired( longTerm, ['addPreference'], [input.category ?? 'general', input.text, neo4jPreferenceOptions(input)], ) - if (result === undefined) { + } else if (input.kind === 'reasoning-trace') { + const reasoning = nested(client, ['reasoning'], client) + const conversationId = sessionId ?? scope.runId + if (transport === 'rest') { + if (typeof conversationId !== 'string') { + throw new Error( + `${adapterId}: Neo4j REST reasoning writes require scope.sessionId or scope.runId`, + ) + } result = await callRequired( - longTerm, - ['add_preference'], - [input.category ?? 'general', input.text, neo4jWriteOptions(input)], + reasoning, + ['recordStep'], + [ + { + conversationId, + reasoning: input.text, + actionTaken: stringMetadata(input.metadata, 'action') ?? input.title ?? 'memory', + result: + stringMetadata(input.metadata, 'result') ?? + stringMetadata(input.metadata, 'observation') ?? + stringMetadata(input.metadata, 'outcome'), + }, + ], + ) + } else { + const trace = await callRequired( + reasoning, + ['startTrace'], + [ + conversationId ?? stableId('session', input.text), + stringMetadata(input.metadata, 'task') ?? input.title ?? input.text, + ], + ) + assertNeo4jWriteSucceeded(trace, adapterId, 'startTrace') + const traceId = idFromResult(trace) + if (!traceId) throw new Error(`${adapterId}: Neo4j startTrace returned no trace id`) + try { + const step = await callRequired( + reasoning, + ['addStep'], + [ + traceId, + { + thought: input.text, + observation: stringMetadata(input.metadata, 'observation'), + action: stringMetadata(input.metadata, 'action'), + }, + ], + ) + assertNeo4jWriteSucceeded(step, adapterId, 'addStep') + } catch (error) { + try { + await callRequired( + reasoning, + ['completeTrace'], + [traceId, { outcome: 'agent-knowledge addStep failed', success: false }], + ) + } catch (cleanupError) { + throw new AggregateError( + [error, cleanupError], + `${adapterId}: Neo4j reasoning step and trace cleanup failed`, + ) + } + throw error + } + result = await callRequired( + reasoning, + ['completeTrace'], + [ + traceId, + { + outcome: stringMetadata(input.metadata, 'outcome'), + success: + typeof input.metadata?.success === 'boolean' ? input.metadata.success : undefined, + }, + ], ) } } else { + assertNeo4jBridgeCapability(transport, input.kind, adapterId) result = await callRequired( nested(client, ['longTerm', 'long_term'], client), - ['addFact', 'add_fact'], + ['addFact'], [ input.subject ?? input.title ?? input.kind, input.predicate ?? 'states', @@ -232,13 +331,40 @@ async function writeNeo4jMemory( ) } - const id = idFromResult(result) ?? input.id ?? stableId('mem', `${input.kind}:${input.text}`) + assertNeo4jWriteSucceeded(result, adapterId, input.kind) + + const id = + idFromResult(result) ?? + input.id ?? + stableId( + 'mem', + canonicalJson({ adapterId, kind: input.kind, text: input.text, scope: input.scope ?? {} }), + ) return { accepted: true, id, uri: `memory://${adapterId}/${encodeURIComponent(id)}`, kind: input.kind, - metadata: { rawResult: result }, + metadata: { provider: 'neo4j-agent-memory', transport }, + } +} + +function assertNeo4jWriteSucceeded(value: unknown, adapterId: string, operation: string): void { + const response = record(value) + if (!response) return + const status = stringField(response, ['status'])?.toLowerCase() + if ( + response.success === false || + response.ok === false || + response.isError === true || + status === 'failed' || + status === 'error' + ) { + const nestedError = record(response.error) + const detail = + stringField(response, ['error', 'message', 'detail']) ?? + (nestedError ? stringField(nestedError, ['message', 'detail']) : undefined) + throw new Error(`${adapterId}: Neo4j ${operation} failed${detail ? `: ${detail}` : ''}`) } } @@ -247,60 +373,81 @@ async function searchNeo4jMemory( query: string, options: AgentMemorySearchOptions, adapterId: string, + transport: 'rest' | 'bridge', + isolatedInstance: boolean, ): Promise { + const limit = options.limit ?? 10 + if (limit === 0) return [] + assertNeo4jSearchKinds(options.kinds, transport, adapterId) + const onlyConversationKinds = + options.kinds !== undefined && + options.kinds.length > 0 && + options.kinds.every((kind) => kind === 'message' || kind === 'observation') + assertNeo4jScopeSupported( + options.scope, + onlyConversationKinds ? ['sessionId'] : [], + adapterId, + 'search', + isolatedInstance, + ) const searches: Promise[] = [] const kinds = options.kinds const includeKind = (kind: AgentMemoryHit['kind']) => !kinds?.length || kinds.includes(kind) const shortTerm = nested(client, ['shortTerm', 'short_term'], {}) const longTerm = nested(client, ['longTerm', 'long_term'], {}) const reasoning = nested(client, ['reasoning'], {}) + const wantsMessages = includeKind('message') || includeKind('observation') + const hasConversation = typeof options.scope?.sessionId === 'string' + + if ( + transport === 'rest' && + wantsMessages && + !hasConversation && + options.kinds?.some((kind) => kind === 'message' || kind === 'observation') + ) { + throw new Error( + `${adapterId}: Neo4j REST message search requires scope.sessionId as the conversation id`, + ) + } - if (includeKind('message')) { + if (wantsMessages && (transport === 'bridge' || hasConversation)) { + const messageKinds = options.kinds?.filter( + (kind) => kind === 'message' || kind === 'observation', + ) ?? ['message', 'observation'] searches.push( - callOptional( - shortTerm, - ['searchMessages', 'search_messages'], - [query, searchMessagesOptions(options)], - ).then((result) => normalizeHits(result, { ...options, kinds: ['message'] }, adapterId)), + callRequired(shortTerm, ['searchMessages'], [query, searchMessagesOptions(options)]).then( + (result) => normalizeHits(result, { ...options, kinds: messageKinds }, adapterId), + ), ) } if (includeKind('entity')) { searches.push( - callOptional( - longTerm, - ['searchEntities', 'search_entities'], - [query, searchEntitiesOptions(options)], - ).then((result) => normalizeHits(result, { ...options, kinds: ['entity'] }, adapterId)), + callRequired(longTerm, ['searchEntities'], [query, searchEntitiesOptions(options)]).then( + (result) => normalizeHits(result, { ...options, kinds: ['entity'] }, adapterId), + ), ) } - if (includeKind('preference')) { + if (transport === 'bridge' && includeKind('preference')) { searches.push( - callOptional( + callRequired( longTerm, - ['searchPreferences', 'search_preferences'], + ['searchPreferences'], [query, searchPreferencesOptions(options)], ).then((result) => normalizeHits(result, { ...options, kinds: ['preference'] }, adapterId)), ) } - if (includeKind('reasoning-trace')) { + if (transport === 'bridge' && includeKind('reasoning-trace')) { searches.push( - callOptional( - reasoning, - ['getSimilarTraces', 'get_similar_traces'], - [query, similarTracesOptions(options)], - ).then((result) => - normalizeHits(result, { ...options, kinds: ['reasoning-trace'] }, adapterId), + callRequired(reasoning, ['getSimilarTraces'], [query, similarTracesOptions(options)]).then( + (result) => normalizeHits(result, { ...options, kinds: ['reasoning-trace'] }, adapterId), ), ) } - const hits = (await Promise.all(searches)).flat() - return hits - .sort((a, b) => (b.normalizedScore ?? b.score ?? 0) - (a.normalizedScore ?? a.score ?? 0)) - .slice(0, options.limit) + return mergeRankedMemoryHits(await Promise.all(searches), limit) } // Mirrors defaultGetMemoryContext's holdout call context so bypass events join the same log stream. @@ -315,31 +462,12 @@ function holdoutBypassContext( } } -function neo4jOptions(options: AgentMemorySearchOptions): Record { - return { - limit: options.limit, - k: options.limit, - min_score: options.minScore, - minScore: options.minScore, - user_identifier: options.scope?.userId, - userIdentifier: options.scope?.userId, - session_id: options.scope?.sessionId, - sessionId: options.scope?.sessionId, - namespace: options.scope?.namespace, - tenant_id: options.scope?.tenantId, - tenantId: options.scope?.tenantId, - kinds: options.kinds, - metadata: options.metadata, - } -} - function searchMessagesOptions(options: AgentMemorySearchOptions): Record { return { limit: options.limit, sessionId: options.scope?.sessionId, conversationId: options.scope?.sessionId, threshold: options.minScore, - metadata: options.metadata, } } @@ -360,21 +488,35 @@ function searchPreferencesOptions(options: AgentMemorySearchOptions): Record { return { limit: options.limit, - sessionId: options.scope?.sessionId, successOnly: options.metadata?.successOnly, } } -function neo4jWriteOptions(input: AgentMemoryWriteInput): Record { - return { - user_identifier: input.scope?.userId, - userIdentifier: input.scope?.userId, - session_id: input.scope?.sessionId, - sessionId: input.scope?.sessionId, - namespace: input.scope?.namespace, - confidence: input.confidence, - metadata: input.metadata, - } +function assertNeo4jScopeSupported( + scope: AgentMemoryScope | undefined, + allowedKeys: readonly (keyof AgentMemoryScope)[], + adapterId: string, + operation: string, + isolatedInstance: boolean, +): void { + if (isolatedInstance || !scope) return + const allowed = new Set(allowedKeys) + const scopeKeys: readonly (keyof AgentMemoryScope)[] = [ + 'tenantId', + 'userId', + 'agentId', + 'teamId', + 'runId', + 'sessionId', + 'namespace', + ] + const supplied = scopeKeys.filter((key) => scope[key] !== undefined) + if (scope.tags && Object.keys(scope.tags).length > 0) supplied.push('tags') + const unsupported = supplied.filter((key) => !allowed.has(key)) + if (unsupported.length === 0) return + throw new Error( + `${adapterId}: Neo4j ${operation} cannot enforce scope fields: ${unsupported.join(', ')}; create a dedicated client and pass branchId`, + ) } function neo4jEntityOptions(input: AgentMemoryWriteInput): Record { @@ -406,8 +548,10 @@ async function callRequired( names: string[], args: unknown[], ): Promise { - const result = await callOptional(target, names, args) - if (result !== undefined) return result + for (const name of names) { + const fn = target[name] + if (typeof fn === 'function') return await fn.apply(target, args) + } throw new Error(`Neo4j agent-memory client is missing method ${names.join(' or ')}`) } @@ -438,7 +582,7 @@ function normalizeHits( : (hit.normalizedScore ?? hit.score ?? 0) >= options.minScore!, ) .filter((hit) => (options.kinds?.length ? options.kinds.includes(hit.kind) : true)) - .slice(0, options.limit) + .slice(0, options.limit ?? 10) } function normalizeHit(value: unknown, index: number, adapterId: string): AgentMemoryHit | null { @@ -453,7 +597,11 @@ function normalizeHit(value: unknown, index: number, adapterId: string): AgentMe } const obj = record(value) if (!obj) return null - const kind = memoryKind(stringField(obj, ['kind', 'type', 'label']), obj) + const metadata = record(obj.metadata) ?? {} + const kind = memoryKind( + stringField(metadata, ['agentKnowledgeKind']) ?? stringField(obj, ['kind', 'type', 'label']), + obj, + ) const text = textFromHitObject(obj, kind) if (!text) return null const id = stringField(obj, ['id', 'memoryId', 'uuid']) ?? stableId('mem', text) @@ -536,17 +684,6 @@ function textFromHitObject( return undefined } -function textFromConversationContext(value: unknown): string | undefined { - const obj = record(value) - if (!obj) return undefined - const parts = [ - ...rawHitsFrom(obj.reflections).map((hit) => normalizeContextPart(hit)), - ...rawHitsFrom(obj.observations).map((hit) => normalizeContextPart(hit)), - ...rawHitsFrom(obj.recentMessages).map((hit) => normalizeContextPart(hit)), - ].filter(Boolean) - return parts.length > 0 ? parts.join('\n\n') : undefined -} - function normalizeConversationContextHits( value: unknown, options: AgentMemorySearchOptions, @@ -574,14 +711,6 @@ function tagContextHit( : { kind, title, content: value } } -function normalizeContextPart(value: unknown): string | undefined { - if (typeof value === 'string') return value - const obj = record(value) - return obj - ? textFromHitObject(obj, memoryKind(stringField(obj, ['kind', 'type', 'label']), obj)) - : undefined -} - function renderHits(hits: AgentMemoryHit[]): string { return hits.map((hit) => hit.text).join('\n\n') } @@ -612,3 +741,15 @@ function numberField(obj: Record, names: string[]): number | un } return undefined } + +function neo4jMessageRole(role: AgentMemoryWriteInput['role']): 'system' | 'user' | 'assistant' { + return role === 'tool' ? 'assistant' : (role ?? 'user') +} + +function stringMetadata( + metadata: Record | undefined, + key: string, +): string | undefined { + const value = metadata?.[key] + return typeof value === 'string' && value.length > 0 ? value : undefined +} diff --git a/src/memory/rank.ts b/src/memory/rank.ts new file mode 100644 index 0000000..ef3ab9e --- /dev/null +++ b/src/memory/rank.ts @@ -0,0 +1,62 @@ +import type { AgentMemoryHit } from './types' + +/** Merges independently ranked memory lists without letting the first unscored list dominate. */ +export function mergeRankedMemoryHits( + groups: readonly (readonly AgentMemoryHit[])[], + limit?: number, +): AgentMemoryHit[] { + const byIdentity = new Map< + string, + { + hit: AgentMemoryHit + score?: number + reciprocalRank: number + bestRank: number + firstGroup: number + encounter: number + } + >() + let encounter = 0 + for (const [groupIndex, group] of groups.entries()) { + for (const [rank, hit] of group.entries()) { + const key = JSON.stringify([hit.uri, hit.text]) + const score = memoryHitScore(hit) + const prior = byIdentity.get(key) + if (!prior) { + byIdentity.set(key, { + hit, + score, + reciprocalRank: 1 / (60 + rank + 1), + bestRank: rank, + firstGroup: groupIndex, + encounter, + }) + } else { + prior.reciprocalRank += 1 / (60 + rank + 1) + prior.bestRank = Math.min(prior.bestRank, rank) + if (score !== undefined && (prior.score === undefined || score > prior.score)) { + prior.hit = hit + prior.score = score + } + } + encounter += 1 + } + } + return [...byIdentity.values()] + .sort( + (left, right) => + right.reciprocalRank - left.reciprocalRank || + Number(right.score !== undefined) - Number(left.score !== undefined) || + (right.score ?? 0) - (left.score ?? 0) || + left.bestRank - right.bestRank || + left.firstGroup - right.firstGroup || + left.encounter - right.encounter, + ) + .map(({ hit }) => hit) + .slice(0, limit ?? Number.POSITIVE_INFINITY) +} + +function memoryHitScore(hit: AgentMemoryHit): number | undefined { + const score = hit.normalizedScore ?? hit.score + return score !== undefined && Number.isFinite(score) ? score : undefined +} diff --git a/src/memory/run-control.ts b/src/memory/run-control.ts new file mode 100644 index 0000000..bf78273 --- /dev/null +++ b/src/memory/run-control.ts @@ -0,0 +1,93 @@ +import { join } from 'node:path' +import { acquireSingleRunLock, type CampaignStorage } from '@tangle-network/agent-eval/campaign' + +export interface AgentMemoryRunLease { + assertOwned(): Promise | void + release(): Promise | void +} + +export type AgentMemoryAcquireRunLease = (input: { + experimentId: string + runDir: string +}) => AgentMemoryRunLease | Promise + +export type AgentMemoryControllerMode = 'process-local' + +export interface OwnedAgentMemoryRunLease { + assertOwned(): Promise + release(): Promise +} + +const activeProcessLocalRuns = new Set() + +export async function acquireAgentMemoryRunLease(input: { + experimentId: string + runDir: string + storage: CampaignStorage + customStorage: boolean + lockFileName: string + label: string + controllerMode?: AgentMemoryControllerMode + acquireRunLease?: AgentMemoryAcquireRunLease +}): Promise { + if (input.acquireRunLease && input.controllerMode) { + throw new Error(`${input.label} cannot combine acquireRunLease with controllerMode`) + } + if (input.acquireRunLease) { + const lease = await input.acquireRunLease({ + experimentId: input.experimentId, + runDir: input.runDir, + }) + if (!lease || typeof lease.assertOwned !== 'function' || typeof lease.release !== 'function') { + throw new Error(`${input.label} acquireRunLease must return assertOwned and release`) + } + return { + async assertOwned() { + await lease.assertOwned() + }, + async release() { + await lease.release() + }, + } + } + + if (!input.customStorage) { + if (input.controllerMode) { + throw new Error(`${input.label} controllerMode is only valid with custom storage`) + } + const lock = acquireSingleRunLock({ + lockPath: join(input.runDir, input.lockFileName), + releaseOnExit: false, + }) + return { + async assertOwned() {}, + async release() { + lock.release() + }, + } + } + + if (input.controllerMode !== 'process-local') { + throw new Error( + `${input.label} custom storage requires acquireRunLease or controllerMode='process-local'`, + ) + } + const runKey = `${input.label}:${input.runDir}` + if (activeProcessLocalRuns.has(runKey)) { + throw new Error(`${input.label} run '${input.runDir}' already has an active controller`) + } + activeProcessLocalRuns.add(runKey) + let released = false + return { + async assertOwned() { + if (released || !activeProcessLocalRuns.has(runKey)) { + throw new Error(`${input.label} run '${input.runDir}' controller lease was lost`) + } + }, + async release() { + if (released) return + released = true + activeProcessLocalRuns.delete(runKey) + }, + } +} diff --git a/src/memory/schemas.ts b/src/memory/schemas.ts index 6f6e791..b6e5da4 100644 --- a/src/memory/schemas.ts +++ b/src/memory/schemas.ts @@ -12,6 +12,9 @@ export const AgentMemoryKindSchema = z.enum([ export const AgentMemoryScopeSchema = z.object({ tenantId: z.string().optional(), userId: z.string().optional(), + agentId: z.string().optional(), + teamId: z.string().optional(), + runId: z.string().optional(), sessionId: z.string().optional(), namespace: z.string().optional(), tags: z.record(z.string(), z.string()).optional(), diff --git a/src/memory/types.ts b/src/memory/types.ts index a538d1f..0a73408 100644 --- a/src/memory/types.ts +++ b/src/memory/types.ts @@ -12,6 +12,9 @@ export type AgentMemoryKind = export interface AgentMemoryScope { tenantId?: string userId?: string + agentId?: string + teamId?: string + runId?: string sessionId?: string namespace?: string tags?: Record @@ -79,10 +82,31 @@ export interface AgentMemoryWriteResult { metadata?: Record } +export type AgentMemoryBranchIsolation = + | { + mode: 'scoped' + /** False when writes may outlive the worker process that issued them. */ + processExitSafe?: boolean + /** Wait before clearing an abandoned branch so accepted asynchronous writes become visible. */ + recoveryDelayMs?: number + } + | { + mode: 'instance' + branchId: string + /** True only when the dedicated instance also enforces every logical scope. */ + supportsLogicalScopes?: boolean + } + | { mode: 'unsupported'; reason: string } + export interface AgentMemoryAdapter { readonly id: string + /** How this adapter prevents candidate branches from reading each other's state. */ + readonly branchIsolation?: AgentMemoryBranchIsolation search(query: string, options?: AgentMemorySearchOptions): Promise getContext(query: string, options?: AgentMemorySearchOptions): Promise write(input: AgentMemoryWriteInput): Promise + /** Delete exactly this scope. Repeated and concurrent calls for the same scope must be safe. */ + clear?(scope?: AgentMemoryScope): Promise flush?(): Promise + close?(): Promise } diff --git a/tests/benchmarks.test.ts b/tests/benchmarks.test.ts index 6df91a8..e0a7944 100644 --- a/tests/benchmarks.test.ts +++ b/tests/benchmarks.test.ts @@ -1,4 +1,4 @@ -import { inMemoryCampaignStorage } from '@tangle-network/agent-eval/campaign' +import { createRunCostLedger, inMemoryCampaignStorage } from '@tangle-network/agent-eval/campaign' import { describe, expect, it } from 'vitest' import { buildFirstPartyMemoryLifecycleBenchmarkCases, @@ -12,17 +12,44 @@ import { isKnowledgeMemoryBenchmarkCase, type KnowledgeAnswerBenchmarkCase, type KnowledgeMemoryBenchmarkCase, + knowledgeBenchmarkJudge, parseKnowledgeBenchmarkJsonl, parseKnowledgeBenchmarkQrels, + type RunMemoryAdapterBenchmarkOptions, respondToIndustryMemoryBenchmarkSmokeCase, respondToIndustryRagBenchmarkSmokeCase, runKnowledgeBenchmarkSuite, - runMemoryAdapterBenchmark, + runMemoryAdapterBenchmark as runMemoryAdapterBenchmarkRaw, scoreKnowledgeBenchmarkArtifact, scoreMemoryBenchmarkArtifact, } from '../src/benchmarks/index' +import type { AgentMemoryAdapter, AgentMemoryHit, AgentMemoryScope } from '../src/memory/types' + +function runMemoryAdapterBenchmark(options: RunMemoryAdapterBenchmarkOptions) { + return runMemoryAdapterBenchmarkRaw({ + ...options, + ...(options.storage && !options.controllerMode && !options.acquireRunLease + ? { controllerMode: 'process-local' as const } + : {}), + }) +} describe('knowledge benchmark adapters', () => { + it('versions benchmark scoring for resumable cache safety', () => { + expect(knowledgeBenchmarkJudge().judgeVersion).toBe('agent-knowledge:knowledge-benchmark:v2') + }) + + it('requires a responder version for resumable benchmark rows', async () => { + await expect( + runKnowledgeBenchmarkSuite({ + cases: buildIndustryRagBenchmarkSmokeCases().slice(0, 1), + runDir: '/runs/missing-responder-ref', + storage: inMemoryCampaignStorage(), + respond: respondToIndustryRagBenchmarkSmokeCase, + }), + ).rejects.toThrow('respondRef is required when resumable is enabled') + }) + it('parses qrels/jsonl and builds retrieval cases for public benchmark formats', () => { expect(parseKnowledgeBenchmarkJsonl('{"id":"q1"}\n{"id":"q2"}\n')).toEqual([ { id: 'q1' }, @@ -76,6 +103,8 @@ describe('knowledge benchmark adapters', () => { cases, runDir: '/runs/knowledge-benchmark-smoke', storage, + respondRef: 'retriever-fixture:v1', + costCeiling: 1, respond: async ({ case: testCase, context }) => { const paid = await context.cost.runPaidCall({ actor: 'benchmark-retriever', @@ -170,6 +199,7 @@ describe('knowledge benchmark adapters', () => { cases, runDir: '/runs/knowledge-benchmark-family-smoke', storage, + respondRef: 'industry-rag-smoke:v1', respond: respondToIndustryRagBenchmarkSmokeCase, }) @@ -233,6 +263,22 @@ describe('knowledge benchmark adapters', () => { expect(weak.passed).toBe(false) expect(strong.score - weak.score).toBeGreaterThanOrEqual(0.4) expect(weak.dimensions.memory_stale_safe).toBe(0) + expect(strong.applicableDimensions).toEqual( + expect.arrayContaining([ + 'memory_fact_recall', + 'memory_event_recall', + 'memory_actor_recall', + 'memory_stale_safe', + ]), + ) + + const recallOnly = scoreMemoryBenchmarkArtifact( + { ...testCase, forbiddenFacts: undefined, expectedActorIds: undefined }, + { answer: 'The user currently prefers email briefings.', citedEventIds: ['e-new'] }, + ) + expect(recallOnly.dimensions).not.toHaveProperty('memory_stale_safe') + expect(recallOnly.dimensions).not.toHaveProperty('memory_actor_recall') + expect(recallOnly.applicableDimensions).not.toContain('memory_stale_safe') }) it('runs one persisted benchmark cell for every declared memory benchmark family', async () => { @@ -242,6 +288,7 @@ describe('knowledge benchmark adapters', () => { cases, runDir: '/runs/memory-benchmark-family-smoke', storage, + respondRef: 'industry-memory-smoke:v1', respond: ({ case: testCase }) => { expect(isKnowledgeMemoryBenchmarkCase(testCase)).toBe(true) if (!isKnowledgeMemoryBenchmarkCase(testCase)) { @@ -285,10 +332,12 @@ describe('knowledge benchmark adapters', () => { candidates: [ { id: 'no-memory', + ref: 'no-memory:v1', createAdapter: () => createNoopMemoryBenchmarkAdapter(), }, { id: 'in-memory', + ref: 'in-memory:v1', createAdapter: () => createInMemoryBenchmarkAdapter(), searchLimit: 1, }, @@ -307,4 +356,851 @@ describe('knowledge benchmark adapters', () => { expect(storage.read(result.rankingMarkdownPath)).toContain('| 1 | in-memory |') expect(storage.read(result.rows[0]!.reportJsonPath)).toContain('"memory_stale_safe"') }) + + it('requires an explicit controller policy for custom benchmark storage', async () => { + await expect( + runMemoryAdapterBenchmarkRaw({ + cases: buildFirstPartyMemoryLifecycleBenchmarkCases().slice(0, 1), + runDir: '/runs/custom-benchmark-storage', + storage: inMemoryCampaignStorage(), + candidates: [ + { + id: 'in-memory', + ref: 'in-memory:v1', + createAdapter: () => createInMemoryBenchmarkAdapter(), + }, + ], + }), + ).rejects.toThrow("requires acquireRunLease or controllerMode='process-local'") + }) + + it('recovers an unfinished provider scope before retrying a direct benchmark cell', async () => { + const storage = inMemoryCampaignStorage() + const providerRows = new Map() + const operations: string[] = [] + const purposes: string[] = [] + let firstExecution = true + const candidate = { + id: 'recoverable', + ref: 'recoverable:v1', + adapterId: 'recoverable-provider', + recoveryCostUsdPerAttempt: 0.1, + createAdapter({ purpose }: { purpose: 'execute' | 'recovery' }) { + purposes.push(purpose) + const label = purpose === 'recovery' ? 'recovery' : firstExecution ? 'first' : 'retry' + if (purpose === 'execute') firstExecution = false + const adapter: AgentMemoryAdapter = { + id: 'recoverable-provider', + branchIsolation: { mode: 'scoped' }, + async search(_query, options) { + return [...(providerRows.get(options?.scope?.namespace ?? '') ?? [])] + }, + async getContext(query, options) { + const hits = await adapter.search(query, options) + return { query, text: hits.map((hit) => hit.text).join('\n'), hits, sourceRecords: [] } + }, + async write(input) { + operations.push(`write:${label}`) + const namespace = input.scope?.namespace ?? '' + const hit: AgentMemoryHit = { + id: input.id ?? `${label}:memory`, + uri: `memory://recoverable/${label}`, + kind: input.kind, + text: input.text, + metadata: input.metadata, + } + providerRows.set(namespace, [...(providerRows.get(namespace) ?? []), hit]) + return { accepted: true, id: hit.id, uri: hit.uri, kind: hit.kind } + }, + async clear(scope) { + operations.push(`clear:${label}`) + if (label === 'first') throw new Error('provider cleanup unavailable') + providerRows.delete(scope?.namespace ?? '') + }, + async close() {}, + } + return adapter + }, + } + const run = () => + runMemoryAdapterBenchmark({ + cases: buildFirstPartyMemoryLifecycleBenchmarkCases().slice(0, 1), + runDir: '/runs/direct-benchmark-recovery', + storage, + candidates: [candidate], + costCeiling: 1, + }) + + await expect(run()).rejects.toThrow('memory benchmark attempt cleanup failed') + expect(providerRows.size).toBe(1) + + const result = await run() + expect(result.rows[0]).toMatchObject({ candidateId: 'recoverable', cellsFailed: 0 }) + expect(result.rows[0]?.totalCostUsd).toBe(0.1) + expect(result.totalCostUsd).toBe(0.1) + expect(purposes).toEqual(['execute', 'recovery', 'execute']) + expect(operations.indexOf('clear:recovery')).toBeLessThan(operations.indexOf('write:retry')) + expect(providerRows.size).toBe(0) + const events = storage + .read(result.attemptLogPath)! + .trim() + .split('\n') + .map((line) => JSON.parse(line) as { status: string; recovery: boolean }) + expect(events.map(({ status, recovery }) => ({ status, recovery }))).toEqual([ + { status: 'started', recovery: false }, + { status: 'cleaned', recovery: true }, + { status: 'started', recovery: false }, + { status: 'cleaned', recovery: false }, + ]) + }) + + it('includes paid cleanup for a retired candidate in the benchmark total', async () => { + const storage = inMemoryCampaignStorage() + const runDir = '/runs/retired-candidate-recovery-cost' + storage.write( + `${runDir}/memory-adapter-attempts.jsonl`, + `${JSON.stringify({ + schema: 3, + status: 'started', + attemptId: 'retired-attempt', + candidateId: 'retired', + candidateRef: 'retired:v1', + adapterId: 'retired-provider', + caseId: 'old-case', + cellId: 'old-cell', + scope: { namespace: 'retired-scope' }, + adapterCreationCostUsd: 0.05, + costUsdPerCase: 0, + recoveryCostUsdPerAttempt: 0.1, + recordedAt: '2026-01-01T00:00:00.000Z', + recovery: false, + })}\n`, + ) + let retiredClears = 0 + const result = await runMemoryAdapterBenchmark({ + cases: buildFirstPartyMemoryLifecycleBenchmarkCases().slice(0, 1), + runDir, + storage, + costCeiling: 1, + candidates: [ + { + id: 'active', + ref: 'active:v1', + adapterId: 'active-provider', + createAdapter: () => createInMemoryBenchmarkAdapter({ id: 'active-provider' }), + }, + ], + recoveryCandidates: [ + { + id: 'retired', + ref: 'retired:v1', + adapterId: 'retired-provider', + adapterCreationCostUsd: 0.05, + recoveryCostUsdPerAttempt: 0.1, + createAdapter({ markExternalCall }) { + markExternalCall() + const adapter = createInMemoryBenchmarkAdapter({ id: 'retired-provider' }) + adapter.clear = async () => { + retiredClears += 1 + } + return adapter + }, + }, + ], + }) + + expect(retiredClears).toBe(1) + expect(result.rows[0]).toMatchObject({ candidateId: 'active', totalCostUsd: 0 }) + expect(result).toMatchObject({ totalCostUsd: 0.15, unrankedRecoveryCostUsd: 0.15 }) + expect(storage.read(result.rankingJsonPath)).toContain('"unrankedRecoveryCostUsd": 0.15') + }) + + it('refuses benchmark recovery when candidate cost settings changed', async () => { + const storage = inMemoryCampaignStorage() + const runDir = '/runs/benchmark-changed-recovery-costs' + storage.write( + `${runDir}/memory-adapter-attempts.jsonl`, + `${JSON.stringify({ + schema: 3, + status: 'started', + attemptId: 'unfinished-attempt', + candidateId: 'memory', + candidateRef: 'memory:v1', + adapterId: 'memory-provider', + caseId: 'old-case', + cellId: 'old-cell', + scope: { namespace: 'unfinished-scope' }, + adapterCreationCostUsd: 0, + costUsdPerCase: 0, + recoveryCostUsdPerAttempt: 0, + recordedAt: '2026-01-01T00:00:00.000Z', + recovery: false, + })}\n`, + ) + let adapterCreates = 0 + + await expect( + runMemoryAdapterBenchmark({ + cases: buildFirstPartyMemoryLifecycleBenchmarkCases().slice(0, 1), + runDir, + storage, + costCeiling: 1, + candidates: [ + { + id: 'memory', + ref: 'memory:v1', + adapterId: 'memory-provider', + costUsdPerCase: 0.1, + createAdapter() { + adapterCreates += 1 + return createInMemoryBenchmarkAdapter({ id: 'memory-provider' }) + }, + }, + ], + }), + ).rejects.toThrow('candidate cost settings changed') + + expect(adapterCreates).toBe(0) + expect( + storage.read(`${runDir}/memory-adapter-attempts.jsonl`)?.trim().split('\n'), + ).toHaveLength(1) + }) + + it('bounds repeated direct benchmark recovery across process restarts', async () => { + const storage = inMemoryCampaignStorage() + const runDir = '/runs/benchmark-recovery-retry-limit' + storage.write( + `${runDir}/memory-adapter-attempts.jsonl`, + `${JSON.stringify({ + schema: 3, + status: 'started', + attemptId: 'unfinished-attempt', + candidateId: 'memory', + candidateRef: 'memory:v1', + adapterId: 'memory-provider', + caseId: 'old-case', + cellId: 'old-cell', + scope: { namespace: 'unfinished-scope' }, + adapterCreationCostUsd: 0, + costUsdPerCase: 0, + recoveryCostUsdPerAttempt: 0, + recordedAt: '2026-01-01T00:00:00.000Z', + recovery: false, + })}\n`, + ) + let recoveryCreates = 0 + const run = () => + runMemoryAdapterBenchmark({ + cases: buildFirstPartyMemoryLifecycleBenchmarkCases().slice(0, 1), + runDir, + storage, + maxRecoveryRetriesPerAttempt: 2, + candidates: [ + { + id: 'memory', + ref: 'memory:v1', + adapterId: 'memory-provider', + createAdapter({ purpose }) { + if (purpose === 'recovery') recoveryCreates += 1 + throw new Error('provider recovery unavailable') + }, + }, + ], + }) + + await expect(run()).rejects.toThrow('provider recovery unavailable') + await expect(run()).rejects.toThrow('provider recovery unavailable') + await expect(run()).rejects.toThrow('exhausted 2 recovery attempts') + expect(recoveryCreates).toBe(2) + expect( + storage.read(`${runDir}/memory-adapter-recovery-attempts.jsonl`)?.trim().split('\n'), + ).toHaveLength(2) + }) + + it('closes a recovery adapter that arrives after its factory timeout', async () => { + const storage = inMemoryCampaignStorage() + const runDir = '/runs/late-benchmark-recovery-adapter' + storage.write( + `${runDir}/memory-adapter-attempts.jsonl`, + `${JSON.stringify({ + schema: 3, + status: 'started', + attemptId: 'unfinished-attempt', + candidateId: 'memory', + candidateRef: 'memory:v1', + adapterId: 'memory-provider', + caseId: 'old-case', + cellId: 'old-cell', + scope: { namespace: 'unfinished-scope' }, + adapterCreationCostUsd: 0, + costUsdPerCase: 0, + recoveryCostUsdPerAttempt: 0, + recordedAt: '2026-01-01T00:00:00.000Z', + recovery: false, + })}\n`, + ) + let resolveCreation!: (adapter: AgentMemoryAdapter) => void + const creation = new Promise((resolve) => { + resolveCreation = resolve + }) + let reportClosed!: () => void + const closed = new Promise((resolve) => { + reportClosed = resolve + }) + let closeCalls = 0 + const lateAdapter = createInMemoryBenchmarkAdapter({ id: 'memory-provider' }) + lateAdapter.close = async () => { + closeCalls += 1 + reportClosed() + } + + await expect( + runMemoryAdapterBenchmark({ + cases: buildFirstPartyMemoryLifecycleBenchmarkCases().slice(0, 1), + runDir, + storage, + cleanupTimeoutMs: 10, + candidates: [ + { + id: 'memory', + ref: 'memory:v1', + adapterId: 'memory-provider', + createAdapter: ({ purpose }) => + purpose === 'recovery' ? creation : createInMemoryBenchmarkAdapter(), + }, + ], + }), + ).rejects.toThrow('benchmark recovery adapter creation did not finish within 10ms') + + resolveCreation(lateAdapter) + await closed + expect(closeCalls).toBe(1) + }) + + it('reconciles a crash after direct recovery but before its cost receipt', async () => { + const storage = inMemoryCampaignStorage() + const append = storage.append!.bind(storage) + let failRecoveryReceipt = false + storage.append = (path, value, expectedBytes) => { + if ( + failRecoveryReceipt && + path.endsWith('/cost-ledger.jsonl') && + value.includes('"status":"settled"') && + value.includes('memory-adapter-recovery') + ) { + failRecoveryReceipt = false + throw new Error('simulated process exit before benchmark recovery receipt') + } + return append(path, value, expectedBytes) + } + + let firstExecution = true + let recoveryClears = 0 + const candidate = { + id: 'receipt-crash', + ref: 'receipt-crash:v1', + adapterId: 'receipt-crash-provider', + costUsdPerCase: 0.1, + recoveryCostUsdPerAttempt: 0.1, + createAdapter({ purpose }: { purpose: 'execute' | 'recovery' }) { + const adapter = createInMemoryBenchmarkAdapter({ id: 'receipt-crash-provider' }) + const clear = adapter.clear! + const failThisExecution = purpose === 'execute' && firstExecution + if (purpose === 'execute') firstExecution = false + adapter.clear = async (scope) => { + if (purpose === 'recovery') recoveryClears += 1 + if (failThisExecution) throw new Error('leave direct benchmark state active') + await clear(scope) + } + return adapter + }, + } + const run = () => + runMemoryAdapterBenchmark({ + cases: buildFirstPartyMemoryLifecycleBenchmarkCases().slice(0, 1), + runDir: '/runs/direct-recovery-receipt-crash', + storage, + costCeiling: 1, + candidates: [candidate], + }) + + await expect(run()).rejects.toThrow('memory benchmark attempt cleanup failed') + failRecoveryReceipt = true + await expect(run()).rejects.toThrow('failed to persist') + expect( + storage + .read('/runs/direct-recovery-receipt-crash/memory-adapter-attempts.jsonl') + ?.trim() + .split('\n'), + ).toHaveLength(2) + + const result = await run() + const costLedger = createRunCostLedger({ + storage, + runDir: '/runs/direct-recovery-receipt-crash', + costCeilingUsd: 1, + }) + + expect(result.rows[0]).toMatchObject({ + candidateId: 'receipt-crash', + cellsFailed: 0, + }) + expect(result.rows[0]?.totalCostUsd).toBeCloseTo(0.3) + expect(result.totalCostUsd).toBeCloseTo(0.3) + expect(recoveryClears).toBe(1) + expect(costLedger.summary()).toMatchObject({ unresolvedCalls: 0, accountingComplete: true }) + expect(costLedger.summary().totalCostUsd).toBeCloseTo(0.3) + }) + + it('bounds direct benchmark cleanup and preserves its recovery record', async () => { + const storage = inMemoryCampaignStorage() + const adapter = createInMemoryBenchmarkAdapter({ id: 'hung-cleanup' }) + let closeCalls = 0 + adapter.clear = () => new Promise(() => {}) + adapter.close = async () => { + closeCalls += 1 + } + const startedAt = Date.now() + + await expect( + runMemoryAdapterBenchmark({ + cases: buildFirstPartyMemoryLifecycleBenchmarkCases().slice(0, 1), + runDir: '/runs/direct-benchmark-cleanup-timeout', + storage, + cleanupTimeoutMs: 10, + candidates: [ + { + id: 'hung-cleanup', + ref: 'hung-cleanup:v1', + createAdapter: () => adapter, + }, + ], + }), + ).rejects.toThrow('memory adapter benchmark cleanup failed') + + expect(Date.now() - startedAt).toBeLessThan(500) + expect( + storage + .read('/runs/direct-benchmark-cleanup-timeout/memory-adapter-attempts.jsonl') + ?.trim() + .split('\n'), + ).toHaveLength(1) + expect(closeCalls).toBe(0) + }) + + it('counts billable adapter creation in the shared benchmark budget', async () => { + const storage = inMemoryCampaignStorage() + let creates = 0 + const result = await runMemoryAdapterBenchmark({ + cases: buildFirstPartyMemoryLifecycleBenchmarkCases().slice(0, 1), + runDir: '/runs/paid-adapter-creation', + storage, + costCeiling: 0.5, + candidates: [ + { + id: 'paid-creation', + ref: 'paid-creation:v1', + adapterId: 'paid-creation-provider', + adapterCreationCostUsd: 0.2, + createAdapter({ markExternalCall }) { + markExternalCall() + creates += 1 + return createInMemoryBenchmarkAdapter({ id: 'paid-creation-provider' }) + }, + }, + ], + }) + + const ledger = createRunCostLedger({ + storage, + runDir: '/runs/paid-adapter-creation', + costCeilingUsd: 0.5, + }) + expect(creates).toBe(1) + expect(result.rows[0]).toMatchObject({ candidateId: 'paid-creation', totalCostUsd: 0.2 }) + expect(result.totalCostUsd).toBe(0.2) + expect(ledger.list()).toMatchObject([ + { + actor: 'agent-knowledge:memory-adapter:paid-creation', + costUsd: 0.2, + tags: { + candidateId: 'paid-creation', + memoryAdapterCreation: 'execute', + runDir: '/runs/paid-adapter-creation/paid-creation', + }, + }, + ]) + }) + + it('does not recreate or recharge a billable adapter when every cell resumes', async () => { + const storage = inMemoryCampaignStorage() + let creates = 0 + const run = () => + runMemoryAdapterBenchmark({ + cases: buildFirstPartyMemoryLifecycleBenchmarkCases().slice(0, 1), + runDir: '/runs/resumed-paid-adapter-creation', + storage, + costCeiling: 0.5, + candidates: [ + { + id: 'paid-resume', + ref: 'paid-resume:v1', + adapterId: 'paid-resume-provider', + adapterCreationCostUsd: 0.2, + createAdapter({ markExternalCall }) { + markExternalCall() + creates += 1 + return createInMemoryBenchmarkAdapter({ id: 'paid-resume-provider' }) + }, + }, + ], + }) + + const initial = await run() + const resumed = await run() + + expect(creates).toBe(1) + expect(initial).toMatchObject({ totalCostUsd: 0.2 }) + expect(resumed).toMatchObject({ totalCostUsd: 0.2 }) + expect(resumed.rows[0]).toMatchObject({ adapterId: 'paid-resume-provider' }) + expect(resumed.rows[0]?.report.cellsCached).toBe(1) + }) + + it('closes a mismatched lazy adapter before any benchmark case writes', async () => { + const storage = inMemoryCampaignStorage() + let closes = 0 + let writes = 0 + const adapter = createInMemoryBenchmarkAdapter({ id: 'actual-provider' }) + const write = adapter.write.bind(adapter) + adapter.write = async (input) => { + writes += 1 + return write(input) + } + adapter.close = async () => { + closes += 1 + } + + await expect( + runMemoryAdapterBenchmark({ + cases: buildFirstPartyMemoryLifecycleBenchmarkCases().slice(0, 1), + runDir: '/runs/mismatched-lazy-adapter', + storage, + candidates: [ + { + id: 'candidate', + ref: 'candidate:v1', + adapterId: 'expected-provider', + createAdapter: () => adapter, + }, + ], + }), + ).rejects.toThrow("returned id 'actual-provider', expected 'expected-provider'") + expect({ closes, writes }).toEqual({ closes: 1, writes: 0 }) + expect( + storage.read('/runs/mismatched-lazy-adapter/memory-adapter-attempts.jsonl'), + ).toBeUndefined() + }) + + it('aborts execute adapter creation at the configured timeout', async () => { + const storage = inMemoryCampaignStorage() + let aborted = false + const startedAt = Date.now() + + await expect( + runMemoryAdapterBenchmark({ + cases: buildFirstPartyMemoryLifecycleBenchmarkCases().slice(0, 1), + runDir: '/runs/timed-out-adapter-creation', + storage, + cleanupTimeoutMs: 10, + candidates: [ + { + id: 'hung-factory', + ref: 'hung-factory:v1', + createAdapter: ({ signal }) => + new Promise((_resolve, reject) => { + signal.addEventListener( + 'abort', + () => { + aborted = true + reject(signal.reason) + }, + { once: true }, + ) + }), + }, + ], + }), + ).rejects.toThrow('benchmark execute adapter creation did not finish within 10ms') + expect(aborted).toBe(true) + expect(Date.now() - startedAt).toBeLessThan(500) + expect( + storage.read('/runs/timed-out-adapter-creation/memory-adapter-attempts.jsonl'), + ).toBeUndefined() + }) + + it('does not charge a local adapter factory failure', async () => { + const storage = inMemoryCampaignStorage() + await expect( + runMemoryAdapterBenchmark({ + cases: buildFirstPartyMemoryLifecycleBenchmarkCases().slice(0, 1), + runDir: '/runs/local-adapter-creation-failure', + storage, + costCeiling: 0.5, + candidates: [ + { + id: 'local-failure', + ref: 'local-failure:v1', + adapterCreationCostUsd: 0.2, + createAdapter() { + throw new Error('invalid local configuration') + }, + }, + ], + }), + ).rejects.toThrow('invalid local configuration') + + const ledger = createRunCostLedger({ + storage, + runDir: '/runs/local-adapter-creation-failure', + costCeilingUsd: 0.5, + }) + expect(ledger.summary()).toMatchObject({ totalCostUsd: 0, unresolvedCalls: 0 }) + }) + + it('enforces one cost ceiling across every compared adapter', async () => { + const result = await runMemoryAdapterBenchmark({ + cases: buildFirstPartyMemoryLifecycleBenchmarkCases().slice(0, 1), + runDir: '/runs/shared-adapter-cost-ceiling', + storage: inMemoryCampaignStorage(), + costCeiling: 0.75, + candidates: ['first', 'second'].map((id) => ({ + id, + ref: `${id}:v1`, + costUsdPerCase: 0.5, + createAdapter: () => createInMemoryBenchmarkAdapter({ id }), + })), + }) + + expect(result.totalCostUsd).toBe(0.5) + expect(result.rows).toHaveLength(2) + expect(result.rows.find((row) => row.candidateId === 'first')).toMatchObject({ + cellsFailed: 0, + totalCostUsd: 0.5, + }) + expect(result.rows.find((row) => row.candidateId === 'second')).toMatchObject({ + cellsFailed: 1, + totalCostUsd: 0, + }) + }) + + it('runs sequential paid adapter cases after exact external-cost receipts', async () => { + const result = await runMemoryAdapterBenchmark({ + cases: buildFirstPartyMemoryLifecycleBenchmarkCases().slice(0, 2), + runDir: '/runs/sequential-adapter-cost-accounting', + storage: inMemoryCampaignStorage(), + costCeiling: 0.2, + maxConcurrency: 1, + candidates: [ + { + id: 'sequential-paid', + ref: 'sequential-paid:v1', + costUsdPerCase: 0.1, + createAdapter: () => createInMemoryBenchmarkAdapter({ id: 'sequential-paid' }), + }, + ], + }) + + expect(result.rows[0]).toMatchObject({ cellsFailed: 0, totalCostUsd: 0.2 }) + expect(result.totalCostUsd).toBe(0.2) + }) + + it('refuses paid adapter calls when no dollar limit is configured', async () => { + let providerCalls = 0 + const adapter = createInMemoryBenchmarkAdapter({ id: 'paid-by-default' }) + const write = adapter.write.bind(adapter) + adapter.write = async (input) => { + providerCalls += 1 + return write(input) + } + + const result = await runMemoryAdapterBenchmark({ + cases: buildFirstPartyMemoryLifecycleBenchmarkCases().slice(0, 1), + runDir: '/runs/default-zero-cost-ceiling', + storage: inMemoryCampaignStorage(), + candidates: [ + { + id: 'paid-by-default', + ref: 'paid-by-default:v1', + costUsdPerCase: 0.01, + createAdapter: () => adapter, + }, + ], + }) + + expect(result.rows[0]).toMatchObject({ cellsFailed: 1, totalCostUsd: 0 }) + expect(providerCalls).toBe(0) + }) + + it('does not reuse resumable rows when adapter benchmark options change', async () => { + const storage = inMemoryCampaignStorage() + const run = (searchLimit: number) => + runMemoryAdapterBenchmark({ + cases: buildFirstPartyMemoryLifecycleBenchmarkCases().slice(0, 1), + runDir: '/runs/memory-adapter-cache-identity', + storage, + candidates: [ + { + id: 'in-memory', + ref: 'in-memory:v1', + createAdapter: () => createInMemoryBenchmarkAdapter(), + searchLimit, + }, + ], + }) + + const initial = await run(1) + const changed = await run(2) + const repeated = await run(2) + + expect(initial.rows[0]?.report.cellsCached).toBe(0) + expect(changed.rows[0]?.report.cellsCached).toBe(0) + expect(repeated.rows[0]?.report.cellsCached).toBe(1) + }) + + it('isolates and clears every concurrent adapter repetition', async () => { + const adapter = createInMemoryBenchmarkAdapter() + const clearedScopes: AgentMemoryScope[] = [] + const clear = adapter.clear! + adapter.clear = async (scope) => { + clearedScopes.push(scope ?? {}) + await clear(scope) + } + + const result = await runMemoryAdapterBenchmark({ + cases: buildFirstPartyMemoryLifecycleBenchmarkCases().slice(0, 1), + runDir: '/runs/isolated-memory-repetitions', + storage: inMemoryCampaignStorage(), + reps: 3, + maxConcurrency: 3, + candidates: [ + { + id: 'in-memory', + ref: 'in-memory:v1', + createAdapter: () => adapter, + }, + ], + }) + + expect(result.rows[0]?.totalCells).toBe(3) + expect(result.rows[0]?.cellsFailed).toBe(0) + expect(clearedScopes).toHaveLength(3) + expect(new Set(clearedScopes.map((scope) => scope.namespace)).size).toBe(3) + expect(new Set(clearedScopes.map((scope) => scope.tags?.benchmarkAttemptId)).size).toBe(3) + }) + + it('settles timed-out provider work before clearing and closing the adapter', async () => { + let closed = false + let writes = 0 + let writesAfterClose = 0 + let clears = 0 + let clearsAfterClose = 0 + let contextReads = 0 + const adapter: AgentMemoryAdapter = { + id: 'delayed-provider', + branchIsolation: { mode: 'scoped' }, + async search() { + return [] + }, + async getContext(query) { + contextReads += 1 + return { query, text: '', hits: [], sourceRecords: [] } + }, + async write(input) { + await new Promise((resolve) => setTimeout(resolve, 25)) + writes += 1 + if (closed) writesAfterClose += 1 + return { + accepted: true, + id: input.id ?? String(writes), + uri: `memory://delayed-provider/${writes}`, + kind: input.kind, + } + }, + async clear() { + await new Promise((resolve) => setTimeout(resolve, 1)) + clears += 1 + if (closed) clearsAfterClose += 1 + }, + async close() { + closed = true + }, + } + + const result = await runMemoryAdapterBenchmark({ + cases: buildFirstPartyMemoryLifecycleBenchmarkCases().slice(0, 1), + runDir: '/runs/timed-out-memory-provider', + storage: inMemoryCampaignStorage(), + dispatchTimeoutMs: 5, + candidates: [ + { + id: 'delayed-provider', + ref: 'delayed-provider:v1', + createAdapter: () => adapter, + }, + { + id: 'no-memory', + ref: 'no-memory:v1', + createAdapter: () => createNoopMemoryBenchmarkAdapter(), + }, + ], + }) + + expect(result.rows[0]).toMatchObject({ candidateId: 'no-memory', cellsFailed: 0 }) + expect(result.rows[1]).toMatchObject({ candidateId: 'delayed-provider', cellsFailed: 1 }) + expect({ closed, writes, writesAfterClose, clears, clearsAfterClose, contextReads }).toEqual({ + closed: true, + writes: 1, + writesAfterClose: 0, + clears: 1, + clearsAfterClose: 0, + contextReads: 0, + }) + await new Promise((resolve) => setTimeout(resolve, 50)) + expect({ writes, writesAfterClose, clears, clearsAfterClose }).toEqual({ + writes: 1, + writesAfterClose: 0, + clears: 1, + clearsAfterClose: 0, + }) + }) + + it('rejects every unsafe candidate id before creating any adapter', async () => { + let creates = 0 + await expect( + runMemoryAdapterBenchmark({ + cases: buildFirstPartyMemoryLifecycleBenchmarkCases().slice(0, 1), + runDir: '/runs/invalid-memory-candidate', + storage: inMemoryCampaignStorage(), + candidates: [ + { + id: 'valid', + ref: 'valid:v1', + createAdapter() { + creates += 1 + return createInMemoryBenchmarkAdapter() + }, + }, + { + id: '../invalid', + ref: 'invalid:v1', + createAdapter() { + creates += 1 + return createInMemoryBenchmarkAdapter() + }, + }, + ], + }), + ).rejects.toThrow('must be a safe directory segment') + expect(creates).toBe(0) + }) }) diff --git a/tests/memory-adapter.test.ts b/tests/memory-adapter.test.ts index 5df869a..5902bac 100644 --- a/tests/memory-adapter.test.ts +++ b/tests/memory-adapter.test.ts @@ -48,29 +48,26 @@ describe('memory adapters', () => { expect(similarTraceArgs[1].successOnly).toBe(true) }) - it('wraps a Neo4j Agent Memory-like client for search and context', async () => { + it('renders bridge preference search results as memory context', async () => { const client = { - async search(query: string, options: Record) { - return [ - { - id: 'pref-1', - type: 'preference', - text: `${query}: likes concise answers`, - score: 0.8, - normalizedScore: 1, - userIdentifier: options.userIdentifier, - }, - ] + longTerm: { + async searchPreferences(query: string) { + return [{ id: 'pref-1', category: 'writing', preference: `${query}: concise answers` }] + }, }, } - const adapter = createNeo4jAgentMemoryAdapter({ client }) + const adapter = createNeo4jAgentMemoryAdapter({ + client, + transport: 'bridge', + branchId: 'branch-1', + }) const hits = await adapter.search('writing style', { - scope: { userId: 'user-1' }, + kinds: ['preference'], limit: 3, }) const context = await adapter.getContext('writing style', { - scope: { userId: 'user-1' }, + kinds: ['preference'], limit: 3, }) @@ -78,10 +75,9 @@ describe('memory adapters', () => { expect(hits[0]).toMatchObject({ id: 'pref-1', kind: 'preference', - text: 'writing style: likes concise answers', - normalizedScore: 1, + text: 'writing style: concise answers', }) - expect(context.text).toContain('likes concise answers') + expect(context.text).toContain('concise answers') expect(context.sourceRecords[0]?.uri).toBe('memory://neo4j-agent-memory/pref-1') }) @@ -113,7 +109,11 @@ describe('memory adapters', () => { }, }, } - const adapter = createNeo4jAgentMemoryAdapter({ client }) + const adapter = createNeo4jAgentMemoryAdapter({ + client, + transport: 'bridge', + branchId: 'branch-1', + }) const hits = await adapter.search('project preferences', { scope: { sessionId: 'session-1' }, @@ -149,7 +149,11 @@ describe('memory adapters', () => { }, }, } - const adapter = createNeo4jAgentMemoryAdapter({ client }) + const adapter = createNeo4jAgentMemoryAdapter({ + client, + transport: 'rest', + contextMode: 'native', + }) const context = await adapter.getContext('style', { scope: { sessionId: 'conversation-1' } }) @@ -160,6 +164,21 @@ describe('memory adapters', () => { expect(context.sourceRecords[0]?.metadata?.source).toBe('agent-memory') }) + it('keeps provider-native context explicit when requested kinds require query search', async () => { + const adapter = createNeo4jAgentMemoryAdapter({ + client: { shortTerm: { async getContext() {} } }, + transport: 'rest', + contextMode: 'native', + }) + + await expect( + adapter.getContext('company', { + scope: { sessionId: 'conversation-1' }, + kinds: ['entity'], + }), + ).rejects.toThrow('native context only returns message and observation') + }) + it('delegates message writes using the published TypeScript SDK positional signature', async () => { const calls: unknown[][] = [] const client = { @@ -170,13 +189,13 @@ describe('memory adapters', () => { }, }, } - const adapter = createNeo4jAgentMemoryAdapter({ client }) + const adapter = createNeo4jAgentMemoryAdapter({ client, transport: 'rest' }) const result = await adapter.write({ kind: 'message', role: 'assistant', text: 'I will keep updates short.', - scope: { sessionId: 'session-1', userId: 'user-1' }, + scope: { sessionId: 'session-1' }, metadata: { source: 'test' }, }) @@ -185,13 +204,12 @@ describe('memory adapters', () => { expect(calls[0]?.[2]).toBe('I will keep updates short.') expect(calls[0]?.[3]).toMatchObject({ conversationId: 'session-1', - userId: 'user-1', metadata: { source: 'test' }, }) expect(result.uri).toBe('memory://neo4j-agent-memory/msg-1') }) - it('falls back to bridge-style snake_case message writes', async () => { + it('rejects clients that do not expose the official addMessage method', async () => { const calls: unknown[][] = [] const client = { short_term: { @@ -201,22 +219,17 @@ describe('memory adapters', () => { }, }, } - const adapter = createNeo4jAgentMemoryAdapter({ client }) - - const result = await adapter.write({ - kind: 'message', - role: 'user', - text: 'Remember this bridge message.', - scope: { sessionId: 'session-1', userId: 'user-1' }, - }) + const adapter = createNeo4jAgentMemoryAdapter({ client, transport: 'bridge' }) - expect(calls[0]?.[0]).toMatchObject({ - session_id: 'session-1', - role: 'user', - content: 'Remember this bridge message.', - user_identifier: 'user-1', - }) - expect(result.id).toBe('msg-bridge') + await expect( + adapter.write({ + kind: 'message', + role: 'user', + text: 'Remember this bridge message.', + scope: { sessionId: 'session-1' }, + }), + ).rejects.toThrow('missing method addMessage') + expect(calls).toEqual([]) }) it('uses narrow hosted SDK options for entity and preference writes', async () => { @@ -233,7 +246,7 @@ describe('memory adapters', () => { }, }, } - const adapter = createNeo4jAgentMemoryAdapter({ client }) + const adapter = createNeo4jAgentMemoryAdapter({ client, transport: 'bridge' }) await adapter.write({ kind: 'entity', @@ -241,14 +254,12 @@ describe('memory adapters', () => { entityType: 'PERSON', text: 'Alice Johnson is a software engineer.', metadata: { description: 'Software engineer at Acme Corp' }, - scope: { userId: 'user-1' }, }) await adapter.write({ kind: 'preference', category: 'writing', text: 'Prefers direct answers', metadata: { context: 'profile' }, - scope: { userId: 'user-1' }, }) expect(calls[0]).toEqual([ @@ -275,13 +286,12 @@ describe('memory adapters', () => { }, }, } - const adapter = createNeo4jAgentMemoryAdapter({ client }) + const adapter = createNeo4jAgentMemoryAdapter({ client, transport: 'bridge' }) const result = await adapter.write({ kind: 'preference', category: 'writing', text: 'prefers direct answers', - scope: { userId: 'user-1' }, }) expect(calls[0]?.[0]).toBe('writing') @@ -295,6 +305,55 @@ describe('memory adapters', () => { expect(result.sourceRecord?.metadata?.memoryKind).toBe('preference') }) + it('rejects direct scopes the Neo4j SDK cannot enforce before provider calls', async () => { + let providerCalls = 0 + const adapter = createNeo4jAgentMemoryAdapter({ + transport: 'bridge', + client: { + shortTerm: { + async searchMessages() { + providerCalls += 1 + return [] + }, + }, + longTerm: { + async searchEntities() { + providerCalls += 1 + return [] + }, + async addEntity() { + providerCalls += 1 + return { id: 'entity-1' } + }, + }, + reasoning: { + async recordStep() { + providerCalls += 1 + return { id: 'reasoning-1' } + }, + }, + }, + }) + + await expect( + adapter.search('company', { kinds: ['entity'], scope: { tenantId: 'tenant-1' } }), + ).rejects.toThrow('cannot enforce scope fields: tenantId') + await expect(adapter.search('anything', { scope: { sessionId: 'session-1' } })).rejects.toThrow( + 'cannot enforce scope fields: sessionId', + ) + await expect( + adapter.write({ kind: 'entity', text: 'Acme', scope: { sessionId: 'session-1' } }), + ).rejects.toThrow('cannot enforce scope fields: sessionId') + await expect( + adapter.write({ + kind: 'reasoning-trace', + text: 'Investigated the issue.', + scope: { sessionId: 'session-1', runId: 'run-1' }, + }), + ).rejects.toThrow('cannot enforce both sessionId and runId') + expect(providerCalls).toBe(0) + }) + it('preserves Neo4j SDK unsupported errors instead of masking them with fallbacks', async () => { const calls: string[] = [] const client = { @@ -309,7 +368,7 @@ describe('memory adapters', () => { }, }, } - const adapter = createNeo4jAgentMemoryAdapter({ client }) + const adapter = createNeo4jAgentMemoryAdapter({ client, transport: 'bridge' }) await expect( adapter.write({ @@ -321,15 +380,24 @@ describe('memory adapters', () => { expect(calls).toEqual(['addPreference']) }) - it('uses the configured adapter id for fallback memory URIs', async () => { + it('uses the configured adapter id for memory URIs', async () => { const client = { - async getContext() { - return 'Use the private project namespace.' + shortTerm: { + async searchMessages() { + return [{ id: 'message-1', role: 'user', content: 'Use the private project namespace.' }] + }, }, } - const adapter = createNeo4jAgentMemoryAdapter({ client, id: 'neo4j-private' }) + const adapter = createNeo4jAgentMemoryAdapter({ + client, + transport: 'rest', + id: 'neo4j-private', + }) - const context = await adapter.getContext('project namespace') + const context = await adapter.getContext('project namespace', { + scope: { sessionId: 'conversation-1' }, + kinds: ['message'], + }) expect(context.hits[0]?.uri).toMatch(/^memory:\/\/neo4j-private\//) expect(context.sourceRecords[0]?.uri).toBe(context.hits[0]?.uri) diff --git a/tests/memory-holdout.test.ts b/tests/memory-holdout.test.ts index c6f6756..6eecc6a 100644 --- a/tests/memory-holdout.test.ts +++ b/tests/memory-holdout.test.ts @@ -279,7 +279,12 @@ describe('retrieval holdout: adapter bypass paths still log the call', () => { }, }, } - const adapter = createNeo4jAgentMemoryAdapter({ client }) + const adapter = createNeo4jAgentMemoryAdapter({ + client, + transport: 'rest', + contextMode: 'native', + branchId: 'holdout-branch', + }) const context = await adapter.getContext('style', { scope: { sessionId: 'conv-1', tags: { taskId: 'task-3' } }, @@ -308,38 +313,6 @@ describe('retrieval holdout: adapter bypass paths still log the call', () => { expect(event.deliveredIds).toEqual(['obs-1', 'msg-1']) expect(event.queryHash).toBe(sha256('style').slice(0, 16)) }) - - it('emits a raw-string-context bypass event for string getContext results', async () => { - const { config, events } = collectingConfig({ epsilon: 1, watchlist: ['w-1'], rng: () => 0 }) - const client = { - async getContext() { - return 'Use the private project namespace.' - }, - } - const adapter = createNeo4jAgentMemoryAdapter({ client, id: 'neo4j-private' }) - - const context = await adapter.getContext('project namespace', { holdout: config }) - - expect(context.text).toBe('Use the private project namespace.') - expect(context.hits).toHaveLength(1) - expect(events).toHaveLength(1) - const event = events[0]! - expect(event.bypassReason).toBe('raw-string-context') - expect(event.holdoutEligible).toBe(false) - expect(event.droppedId).toBeNull() - expect(event.eligible).toEqual([ - { - id: context.hits[0]!.id, - rank: 1, - score: 1, - kind: 'fact', - contentHash: sha256('Use the private project namespace.').slice(0, 16), - }, - ]) - expect(event.deliveredIds).toEqual([context.hits[0]!.id]) - expect(event.sessionId).toBeUndefined() - expect(event.sessionIdHash).toBeUndefined() - }) }) describe('retrieval holdout: value-keyed session registry', () => { diff --git a/tests/memory-systems.test.ts b/tests/memory-systems.test.ts new file mode 100644 index 0000000..cabfb70 --- /dev/null +++ b/tests/memory-systems.test.ts @@ -0,0 +1,4876 @@ +import type { MemoryClient as Neo4jMemoryClient } from '@neo4j-labs/agent-memory' +import { + createRunCostLedger, + inMemoryCampaignStorage, + type SurfaceProposer, +} from '@tangle-network/agent-eval/campaign' +import type { MemoryClient } from 'mem0ai' +import type { Memory as OssMemory } from 'mem0ai/oss' +import { describe, expect, it } from 'vitest' +import { + type AgentMemoryAdapter, + type AgentMemoryHit, + type AgentMemoryScope, + agentMemorySequenceJudge, + buildAgentMemorySequenceScenarios, + buildAgentMemorySequencesFromBenchmarkCases, + createAgentMemoryBranch, + createGraphitiMemoryAdapter, + createMem0MemoryAdapter, + createNeo4jAgentMemoryAdapter, + forkAgentMemoryBranchSnapshot, + type GraphitiMcpClientLike, + graphitiMemoryAdapterIdentity, + type Mem0ClientLike, + mem0MemoryAdapterIdentity, + type RunAgentMemoryExperimentOptions, + type RunAgentMemoryImprovementOptions, + runAgentMemoryExperiment as runAgentMemoryExperimentRaw, + runAgentMemoryImprovement as runAgentMemoryImprovementRaw, + runBoundedMemoryLifecycle, +} from '../src/memory/index' +import { mergeRankedMemoryHits } from '../src/memory/rank' + +function withProcessLocalController< + T extends { + storage?: unknown + controllerMode?: 'process-local' + acquireRunLease?: unknown + }, +>(options: T): T { + if (!options.storage || options.controllerMode || options.acquireRunLease) return options + return { ...options, controllerMode: 'process-local' } +} + +function runAgentMemoryExperiment(options: RunAgentMemoryExperimentOptions) { + return runAgentMemoryExperimentRaw(withProcessLocalController(options)) +} + +function runAgentMemoryImprovement(options: RunAgentMemoryImprovementOptions) { + return runAgentMemoryImprovementRaw(withProcessLocalController(options)) +} + +describe('memory lifecycle', () => { + it('blocks later operations until timed-out work on the same adapter settles', async () => { + const resource = {} + const abortController = new AbortController() + let release: (() => void) | undefined + const pending = new Promise((resolve) => { + release = resolve + }) + let laterRuns = 0 + + await expect( + runBoundedMemoryLifecycle({ + operation: 'slow clear', + timeoutMs: 5, + resource, + abortController, + run: () => pending, + }), + ).rejects.toThrow('slow clear did not finish within 5ms') + expect(abortController.signal.aborted).toBe(true) + await expect( + runBoundedMemoryLifecycle({ + operation: 'unsafe close', + timeoutMs: 5, + resource, + run: () => { + laterRuns += 1 + }, + }), + ).rejects.toThrow("unsafe close cannot start because 'slow clear' is still running") + expect(laterRuns).toBe(0) + + release?.() + await pending + await new Promise((resolve) => setTimeout(resolve, 0)) + await runBoundedMemoryLifecycle({ + operation: 'settled close', + timeoutMs: 5, + resource, + run: () => { + laterRuns += 1 + }, + }) + expect(laterRuns).toBe(1) + }) + + it('keeps URI and text pairs distinct when newline-delimited identities collide', () => { + const first: AgentMemoryHit = { + id: 'first', + uri: 'memory://provider/a\nb', + kind: 'fact', + text: 'c', + } + const second: AgentMemoryHit = { + id: 'second', + uri: 'memory://provider/a', + kind: 'fact', + text: 'b\nc', + } + + expect(mergeRankedMemoryHits([[first], [second]]).map((hit) => hit.id)).toEqual([ + 'first', + 'second', + ]) + }) +}) + +describe('Mem0 adapter', () => { + it('is type-compatible with both current Mem0 clients', () => { + const hosted = null as unknown as MemoryClient + const oss = null as unknown as OssMemory + + expect(createMem0MemoryAdapter({ client: hosted, mode: 'hosted' }).id).toBe('mem0-hosted') + expect(createMem0MemoryAdapter({ client: oss, mode: 'oss' }).id).toBe('mem0-oss') + }) + + it('writes and searches with the same provider scope', async () => { + const calls: Array<{ method: string; options?: Record }> = [] + const client: Mem0ClientLike = { + async add(_messages, options) { + calls.push({ method: 'add', options }) + return [{ id: 'memory-1', event: 'ADD' }] + }, + async search(_query, options) { + calls.push({ method: 'search', options }) + return { + results: [ + { + id: 'memory-1', + memory: 'Use concise status updates.', + score: 0.91, + metadata: { + memoryKind: 'preference', + memoryTitle: 'Writing style', + eventId: 'event-1', + }, + }, + ], + } + }, + } + const adapter = createMem0MemoryAdapter({ client, mode: 'hosted', rerank: true }) + const scope = { + tenantId: 'tenant-1', + userId: 'user-1', + agentId: 'agent-1', + teamId: 'team-1', + namespace: 'branch-1/private', + sessionId: 'session-1', + tags: { task: 'support' }, + } + + await adapter.write({ + id: 'event-1', + kind: 'preference', + title: 'Writing style', + text: 'Use concise status updates.', + scope, + metadata: { run_id: 'attempted-override', memoryKind: 'entity' }, + }) + const hits = await adapter.search('status style', { scope, limit: 3 }) + + const add = calls[0]?.options + const search = calls[1]?.options + expect(add).toMatchObject({ + userId: 'user-1', + agentId: 'agent-1', + runId: 'branch-1/private', + infer: true, + metadata: { + memoryKind: 'preference', + tenant_id: 'tenant-1', + user_id: 'user-1', + agent_id: 'agent-1', + team_id: 'team-1', + run_id: 'branch-1/private', + session_id: 'session-1', + tag_task: 'support', + }, + }) + expect(search).toMatchObject({ + filters: { + user_id: 'user-1', + agent_id: 'agent-1', + run_id: 'branch-1/private', + tenant_id: 'tenant-1', + team_id: 'team-1', + session_id: 'session-1', + tag_task: 'support', + }, + topK: 3, + rerank: true, + }) + expect(hits[0]).toMatchObject({ + id: 'memory-1', + kind: 'preference', + text: 'Use concise status updates.', + score: 0.91, + normalizedScore: 0.91, + metadata: { eventId: 'event-1' }, + }) + }) + + it('pushes memory kinds into provider filters before top-k truncation', async () => { + const searchOptions: Record[] = [] + const adapter = createMem0MemoryAdapter({ + mode: 'hosted', + client: { + async add() { + return [] + }, + async search(_query, options) { + searchOptions.push(options ?? {}) + const filters = options?.filters as Record | undefined + if (filters?.memoryKind === 'preference') { + return { + results: [ + { + id: 'preference-1', + memory: 'Use concise updates.', + metadata: { memoryKind: 'preference' }, + }, + ], + } + } + return { + results: [ + { + id: 'fact-1', + memory: 'Higher-ranked unrelated fact.', + metadata: { memoryKind: 'fact' }, + }, + ], + } + }, + }, + }) + + const filtered = await adapter.search('updates', { + scope: { userId: 'user-1' }, + kinds: ['preference'], + limit: 1, + }) + const unfiltered = await adapter.search('updates', { + scope: { userId: 'user-1' }, + kinds: [], + limit: 1, + }) + + expect(filtered.map((hit) => hit.id)).toEqual(['preference-1']) + expect(unfiltered.map((hit) => hit.id)).toEqual(['fact-1']) + expect(searchOptions[0]).toMatchObject({ + filters: { user_id: 'user-1', memoryKind: 'preference' }, + topK: 1, + }) + expect(searchOptions[1]).toMatchObject({ filters: { user_id: 'user-1' }, topK: 1 }) + }) + + it('rejects legacy asynchronous hosted responses the current SDK cannot produce', async () => { + const adapter = createMem0MemoryAdapter({ + mode: 'hosted', + client: { + async add() { + return { status: 'PENDING', eventId: 'event-1' } + }, + async search() { + return { results: [] } + }, + }, + }) + + await expect( + adapter.write({ + kind: 'fact', + text: 'Remember this.', + scope: { userId: 'user-1' }, + }), + ).rejects.toThrow('mem0ai 3.x must return a memory array') + }) + + it('requires fresh attempt branches for hosted Mem0', async () => { + const client: Mem0ClientLike = { + async add() { + return [] + }, + async search() { + return { results: [] } + }, + } + const visible = createMem0MemoryAdapter({ mode: 'hosted', client }) + + expect(() => createAgentMemoryBranch({ adapter: visible, branchId: 'mem0-resumable' })).toThrow( + "must use lifetime='attempt'", + ) + const attempt = createAgentMemoryBranch({ + adapter: visible, + branchId: 'mem0-attempt', + lifetime: 'attempt', + }) + const snapshot = await attempt.snapshot() + await attempt.close() + expect(() => + createAgentMemoryBranch({ + adapter: visible, + branchId: 'mem0-attempt', + snapshot, + }), + ).toThrow('attempt snapshots cannot be resumed') + }) + + it('does not journal an empty provider result as an accepted write', async () => { + const adapter = createMem0MemoryAdapter({ + mode: 'hosted', + client: { + async add() { + return [] + }, + async search() { + return { results: [] } + }, + }, + }) + + await expect( + adapter.write({ + kind: 'fact', + text: 'No provider mutation', + scope: { userId: 'user-1' }, + }), + ).resolves.toMatchObject({ accepted: false }) + }) + + it('journals and replays Mem0 deletion mutations', async () => { + const parentCalls: string[] = [] + const childCalls: string[] = [] + const adapter = (calls: string[]) => + createMem0MemoryAdapter({ + mode: 'oss', + client: { + async add(messages) { + const text = messages[0]!.content + calls.push(text) + return [ + { + id: text.startsWith('Forget') ? 'memory-old' : 'memory-new', + metadata: { event: text.startsWith('Forget') ? 'DELETE' : 'ADD' }, + }, + ] + }, + async search() { + return { results: [] } + }, + }, + }) + const parent = createAgentMemoryBranch({ + adapter: adapter(parentCalls), + branchId: 'mem0-parent', + baseScope: { agentId: 'agent-1' }, + }) + + await parent.write({ kind: 'fact', text: 'Remember the old launch date.' }) + const deletion = await parent.write({ kind: 'fact', text: 'Forget the old launch date.' }) + const snapshot = await parent.snapshot() + await parent.fork({ branchId: 'mem0-child', adapter: adapter(childCalls) }) + + expect(deletion).toMatchObject({ accepted: true, metadata: { event: 'DELETE' } }) + expect(snapshot.journal.map((entry) => entry.input.text)).toEqual([ + 'Remember the old launch date.', + 'Forget the old launch date.', + ]) + expect(childCalls).toEqual(parentCalls) + }) + + it('includes the default scope in the Mem0 adapter identity', () => { + const base = { mode: 'hosted' as const, id: 'mem0', backendRef: 'mem0-account-a' } + expect(mem0MemoryAdapterIdentity({ ...base, defaultScope: { tenantId: 'tenant-a' } })).not.toBe( + mem0MemoryAdapterIdentity({ ...base, defaultScope: { tenantId: 'tenant-b' } }), + ) + expect(mem0MemoryAdapterIdentity({ ...base, ingestionTimeoutMs: 100 })).not.toBe( + mem0MemoryAdapterIdentity({ ...base, ingestionTimeoutMs: 200 }), + ) + }) + + it('passes OSS entity filters on add', async () => { + let addOptions: Record | undefined + const adapter = createMem0MemoryAdapter({ + mode: 'oss', + client: { + async add(_messages, options) { + addOptions = options + return { results: [{ id: 'oss-1', memory: 'Remembered' }] } + }, + async search() { + return { results: [] } + }, + }, + }) + + await adapter.write({ + kind: 'fact', + text: 'Remembered', + scope: { userId: 'user-1', agentId: 'agent-1', namespace: 'run-1' }, + }) + + expect(addOptions).toMatchObject({ + userId: 'user-1', + agentId: 'agent-1', + runId: 'run-1', + filters: { user_id: 'user-1', agent_id: 'agent-1', run_id: 'run-1' }, + }) + }) + + it('rejects direct OSS operations without a provider entity scope', async () => { + let providerCalls = 0 + const adapter = createMem0MemoryAdapter({ + mode: 'oss', + client: { + async add() { + providerCalls += 1 + return { results: [] } + }, + async search() { + providerCalls += 1 + return { results: [] } + }, + }, + }) + + await expect(adapter.write({ kind: 'fact', text: 'No owner.' })).rejects.toThrow( + 'requires scope.userId, scope.agentId, scope.runId, or scope.namespace', + ) + await expect(adapter.search('No owner.')).rejects.toThrow( + 'requires scope.userId, scope.agentId, scope.runId, or scope.namespace', + ) + expect(providerCalls).toBe(0) + }) + + it('rejects unscoped hosted operations before calling Mem0', async () => { + let providerCalls = 0 + const adapter = createMem0MemoryAdapter({ + mode: 'hosted', + appId: 'support-app', + client: { + async add() { + providerCalls += 1 + return [] + }, + async search() { + providerCalls += 1 + return { results: [] } + }, + async getAll() { + providerCalls += 1 + return { results: [] } + }, + async delete() { + providerCalls += 1 + return { message: 'deleted' } + }, + }, + }) + + await expect(adapter.write({ kind: 'fact', text: 'No owner.' })).rejects.toThrow( + 'Mem0 hosted write requires', + ) + await expect(adapter.search('No owner.')).rejects.toThrow('Mem0 hosted search requires') + await expect(adapter.clear?.()).rejects.toThrow('refusing an unscoped Mem0 clear') + expect(providerCalls).toBe(0) + }) + + it('deletes only memories matching a complete scoped filter', async () => { + const rows = new Set(['memory-1', 'memory-2']) + const getAllOptions: Record[] = [] + let deleteAllCalls = 0 + const adapter = createMem0MemoryAdapter({ + mode: 'hosted', + appId: 'support-app', + client: { + async add() { + return [] + }, + async search() { + return { results: [] } + }, + async getAll(options) { + getAllOptions.push(options ?? {}) + return { results: [...rows].map((id) => ({ id, memory: id })) } + }, + async delete(memoryId) { + rows.delete(memoryId) + return { message: 'deleted' } + }, + async deleteAll() { + deleteAllCalls += 1 + return { message: 'deleted all' } + }, + }, + }) + + await adapter.clear?.({ + tenantId: 'tenant-1', + userId: 'user-1', + namespace: 'physical-branch', + runId: 'logical-run', + sessionId: 'session-1', + tags: { visibility: 'private' }, + }) + + expect(rows.size).toBe(0) + expect(deleteAllCalls).toBe(0) + expect(getAllOptions[0]).toMatchObject({ + filters: { + app_id: 'support-app', + user_id: 'user-1', + run_id: 'physical-branch', + tenant_id: 'tenant-1', + logical_run_id: 'logical-run', + session_id: 'session-1', + tag_visibility: 'private', + }, + page: 1, + pageSize: 100, + showExpired: true, + }) + }) + + it('waits for hosted deletes to disappear without deleting the same memory twice', async () => { + let deletedAt: number | undefined + let getAllCalls = 0 + const deletedIds: string[] = [] + const adapter = createMem0MemoryAdapter({ + mode: 'hosted', + ingestionTimeoutMs: 50, + pollIntervalMs: 1, + client: { + async add() { + return [] + }, + async search() { + return { results: [] } + }, + async getAll() { + getAllCalls += 1 + const visible = deletedAt === undefined || Date.now() - deletedAt < 5 + return { results: visible ? [{ id: 'memory-1', memory: 'stale listing' }] : [] } + }, + async delete(memoryId) { + deletedIds.push(memoryId) + deletedAt = Date.now() + return { message: 'deletion accepted' } + }, + }, + }) + + await adapter.clear?.({ userId: 'user-1', namespace: 'branch-1' }) + + expect(deletedIds).toEqual(['memory-1']) + expect(getAllCalls).toBeGreaterThan(2) + }) + + it('waits for list and search indexes to drop deleted memories before reporting cleanup', async () => { + let deletedAt: number | undefined + let getAllCalls = 0 + let searchCalls = 0 + let deleteCalls = 0 + const adapter = createMem0MemoryAdapter({ + mode: 'hosted', + ingestionTimeoutMs: 50, + pollIntervalMs: 1, + client: { + async add() { + return [] + }, + async search() { + searchCalls += 1 + const visible = deletedAt === undefined || Date.now() - deletedAt < 8 + return { results: visible ? [{ id: 'memory-1', memory: 'stale listing' }] : [] } + }, + async getAll() { + getAllCalls += 1 + const visible = deletedAt === undefined || Date.now() - deletedAt < 5 + return { results: visible ? [{ id: 'memory-1', memory: 'stale listing' }] : [] } + }, + async delete() { + deleteCalls += 1 + deletedAt = Date.now() + return { message: 'deletion accepted' } + }, + }, + }) + + await adapter.clear?.({ userId: 'user-1', namespace: 'branch-1' }) + + expect(deleteCalls).toBe(1) + expect(getAllCalls).toBeGreaterThan(1) + expect(searchCalls).toBeGreaterThan(1) + }) + + it('waits for a fresh narrower id-less write when clearing a broader Mem0 scope', async () => { + const writtenAt = Date.now() + let deleted = false + let getAllCalls = 0 + let deleteCalls = 0 + const visible = () => !deleted && Date.now() - writtenAt >= 5 + const adapter = createMem0MemoryAdapter({ + mode: 'hosted', + ingestionTimeoutMs: 15, + pollIntervalMs: 1, + client: { + async add() { + return [{ event: 'ADD' }] + }, + async search() { + return { results: visible() ? [{ id: 'late-memory', memory: 'late fact' }] : [] } + }, + async getAll() { + getAllCalls += 1 + return { results: visible() ? [{ id: 'late-memory', memory: 'late fact' }] : [] } + }, + async delete(memoryId) { + expect(memoryId).toBe('late-memory') + deleteCalls += 1 + deleted = true + return { message: 'deleted' } + }, + }, + }) + + await adapter.write({ + kind: 'fact', + text: 'late fact', + scope: { userId: 'user-1', agentId: 'agent-1', namespace: 'branch-1' }, + }) + await adapter.clear?.({ userId: 'user-1' }) + + expect(deleteCalls).toBe(1) + expect(getAllCalls).toBeGreaterThan(1) + }) + + it('expires pending write probes after the configured visibility window', async () => { + let searchCalls = 0 + const adapter = createMem0MemoryAdapter({ + mode: 'hosted', + ingestionTimeoutMs: 1, + pollIntervalMs: 1, + client: { + async add() { + return [{ event: 'ADD' }] + }, + async search() { + searchCalls += 1 + return { results: [] } + }, + async getAll() { + return { results: [] } + }, + async delete() { + return { message: 'deleted' } + }, + }, + }) + + await adapter.write({ + kind: 'fact', + text: 'bounded pending fact', + scope: { userId: 'user-1' }, + }) + await new Promise((resolve) => setTimeout(resolve, 5)) + await adapter.clear?.({ userId: 'user-1' }) + + expect(searchCalls).toBe(0) + }) + + it('refuses to clear Mem0 without an exact identity scope', async () => { + let providerCalls = 0 + const adapter = createMem0MemoryAdapter({ + mode: 'hosted', + client: { + async add() { + return [] + }, + async search() { + providerCalls += 1 + return { results: [] } + }, + async getAll() { + providerCalls += 1 + return { results: [] } + }, + async delete() { + providerCalls += 1 + return { message: 'deleted' } + }, + }, + }) + + await expect(adapter.clear?.()).rejects.toThrow('refusing an unscoped Mem0 clear') + expect(providerCalls).toBe(0) + }) +}) + +describe('Graphiti adapter', () => { + it('uses the full default ingestion window before recovering an abandoned branch', () => { + const adapter = createGraphitiMemoryAdapter({ + client: { + async callTool() { + return { structuredContent: {} } + }, + }, + }) + + expect(adapter.branchIsolation).toEqual({ + mode: 'scoped', + processExitSafe: false, + recoveryDelayMs: 120_000, + }) + }) + + it('uses official MCP request shapes and rejoins fact provenance to episodes', async () => { + const requests: Array<{ name: string; arguments?: Record }> = [] + const episodes = new Map>() + const client: GraphitiMcpClientLike = { + async callTool(request) { + requests.push(request) + const args = request.arguments ?? {} + if (request.name === 'add_memory') { + episodes.set(String(args.uuid), { + uuid: args.uuid, + name: args.name, + content: args.episode_body, + source_description: args.source_description, + group_id: args.group_id, + }) + return { structuredContent: { message: 'queued' } } + } + if (request.name === 'get_episodes') { + return { + content: [{ type: 'text', text: JSON.stringify({ episodes: [...episodes.values()] }) }], + } + } + if (request.name === 'search_memory_facts') { + const episode = [...episodes.values()][0]! + return { + structuredContent: { + facts: [ + { + uuid: 'fact-1', + name: 'PREFERS', + fact: 'The user prefers concise status updates.', + group_id: episode.group_id, + episodes: [episode.uuid], + }, + ], + }, + } + } + if (request.name === 'search_nodes') { + return { structuredContent: { nodes: [] } } + } + throw new Error(`unexpected tool ${request.name}`) + }, + } + const adapter = createGraphitiMemoryAdapter({ + client, + search: ['facts'], + pollIntervalMs: 1, + ingestionTimeoutMs: 50, + }) + + const write = await adapter.write({ + id: 'event-1', + kind: 'message', + role: 'user', + text: 'I prefer concise status updates.', + scope: { userId: 'user-1', namespace: 'branch-1' }, + metadata: { actorId: 'user', timestamp: '2026-07-17T12:00:00Z' }, + }) + const hits = await adapter.search('status preference', { + scope: { userId: 'user-1', namespace: 'branch-1' }, + limit: 5, + }) + + expect(requests[0]).toMatchObject({ + name: 'add_memory', + arguments: { + episode_body: 'I prefer concise status updates.', + source: 'message', + reference_time: '2026-07-17T12:00:00Z', + }, + }) + expect(String(requests[0]?.arguments?.group_id)).toMatch(/^ak_[a-f0-9]{32}$/) + expect(String(requests[0]?.arguments?.uuid)).toMatch( + /^[a-f0-9]{8}-[a-f0-9]{4}-5[a-f0-9]{3}-a[a-f0-9]{3}-[a-f0-9]{12}$/, + ) + expect(write.metadata).toMatchObject({ consistency: 'visible', queued: false }) + expect(hits[0]).toMatchObject({ + id: 'fact-1', + kind: 'fact', + text: 'The user prefers concise status updates.', + metadata: { eventId: 'event-1', actorId: 'user' }, + }) + }) + + it('surfaces Graphiti tool errors', async () => { + const adapter = createGraphitiMemoryAdapter({ + client: { + async callTool() { + return { structuredContent: { error: 'database unavailable' } } + }, + }, + }) + + await expect(adapter.search('anything')).rejects.toThrow('database unavailable') + }) + + it('expands the episode scan when Graphiti truncates a long group', async () => { + const episodes = Array.from({ length: 150 }, (_, index) => ({ + uuid: `older-${index.toString().padStart(3, '0')}`, + })) + const limits: number[] = [] + const adapter = createGraphitiMemoryAdapter({ + pollIntervalMs: 1, + ingestionTimeoutMs: 50, + client: { + async callTool(request) { + if (request.name === 'add_memory') { + episodes.push({ uuid: String(request.arguments?.uuid) }) + return { structuredContent: { message: 'queued' } } + } + if (request.name === 'get_episodes') { + const limit = Number(request.arguments?.max_episodes) + limits.push(limit) + return { structuredContent: { episodes: episodes.slice(0, limit) } } + } + throw new Error(`unexpected tool ${request.name}`) + }, + }, + }) + + await adapter.write({ kind: 'fact', text: 'Visible beyond the first page.' }) + + expect(limits).toEqual([100, 200]) + }) + + it('keeps polling a full Graphiti group while the new episode becomes visible', async () => { + const older = Array.from({ length: 100 }, (_, index) => ({ uuid: `older-${index}` })) + let pendingUuid = '' + let visible = false + let polls = 0 + const adapter = createGraphitiMemoryAdapter({ + episodeScanLimit: 100, + pollIntervalMs: 1, + ingestionTimeoutMs: 50, + client: { + async callTool(request) { + if (request.name === 'add_memory') { + pendingUuid = String(request.arguments?.uuid) + setTimeout(() => { + visible = true + }, 5) + return { structuredContent: { message: 'queued' } } + } + if (request.name === 'get_episodes') { + polls += 1 + return { + structuredContent: { + episodes: visible ? [...older.slice(1), { uuid: pendingUuid }] : older, + }, + } + } + throw new Error(`unexpected tool ${request.name}`) + }, + }, + }) + + await expect( + adapter.write({ kind: 'fact', text: 'Eventually visible.' }), + ).resolves.toMatchObject({ accepted: true }) + expect(polls).toBeGreaterThan(1) + }) + + it('fails explicitly when visibility cannot be proved within the episode scan limit', async () => { + const episodes = Array.from({ length: 100 }, (_, index) => ({ uuid: `older-${index}` })) + const adapter = createGraphitiMemoryAdapter({ + episodeScanLimit: 100, + pollIntervalMs: 1, + ingestionTimeoutMs: 50, + client: { + async callTool(request) { + if (request.name === 'add_memory') { + episodes.push({ uuid: String(request.arguments?.uuid) }) + return { structuredContent: { message: 'queued' } } + } + if (request.name === 'get_episodes') { + return { structuredContent: { episodes: episodes.slice(0, 100) } } + } + throw new Error(`unexpected tool ${request.name}`) + }, + }, + }) + + await expect(adapter.write({ kind: 'fact', text: 'Outside the scan window.' })).rejects.toThrow( + 'episodeScanLimit=100', + ) + }) + + it('supports documented tool aliases and does not invent missing relevance scores', async () => { + const names: string[] = [] + const adapter = createGraphitiMemoryAdapter({ + search: ['facts'], + toolNames: { searchFacts: 'search_facts' }, + client: { + async callTool(request) { + names.push(request.name) + return { + structuredContent: { + facts: [{ uuid: 'fact-1', fact: 'An unscored fact', episodes: [] }], + }, + } + }, + }, + }) + + const hits = await adapter.search('fact', { minScore: 0.5 }) + + expect(names).toEqual(['search_facts']) + expect(hits).toEqual([]) + }) + + it('isolates run, session, and tag scopes into different Graphiti groups', async () => { + const groupIds: string[] = [] + const adapter = createGraphitiMemoryAdapter({ + search: ['facts'], + client: { + async callTool(request) { + if (request.name === 'search_memory_facts') { + groupIds.push(String((request.arguments?.group_ids as string[] | undefined)?.[0])) + return { structuredContent: { facts: [] } } + } + return { structuredContent: { episodes: [] } } + }, + }, + }) + + await adapter.search('query', { + scope: { namespace: 'same', runId: 'run-a', sessionId: 'session', tags: { arm: 'a' } }, + }) + await adapter.search('query', { + scope: { namespace: 'same', runId: 'run-b', sessionId: 'session', tags: { arm: 'b' } }, + }) + + expect(new Set(groupIds).size).toBe(2) + }) + + it('keeps caller-identified episode ids stable when groups execute in a different order', async () => { + const run = async (scopes: AgentMemoryScope[]) => { + const ids = new Map() + const adapter = createGraphitiMemoryAdapter({ + consistency: 'queued', + client: { + async callTool(request) { + if (request.name === 'add_memory') { + ids.set(String(request.arguments?.group_id), String(request.arguments?.uuid)) + } + return { structuredContent: { message: 'queued' } } + }, + }, + }) + for (const scope of scopes) { + await adapter.write({ id: 'source-1', kind: 'fact', text: 'same fact', scope }) + } + return ids + } + const first = await run([{ namespace: 'a' }, { namespace: 'b' }]) + const reversed = await run([{ namespace: 'b' }, { namespace: 'a' }]) + + expect(first).toEqual(reversed) + }) + + it('uses a new episode id when the content of one source event changes', async () => { + const episodeIds: string[] = [] + const adapter = createGraphitiMemoryAdapter({ + consistency: 'queued', + client: { + async callTool(request) { + if (request.name === 'add_memory') { + episodeIds.push(String(request.arguments?.uuid)) + } + return { structuredContent: { message: 'queued' } } + }, + }, + }) + + await adapter.write({ id: 'source-1', kind: 'fact', text: 'Launch is Friday.' }) + await adapter.write({ id: 'source-1', kind: 'fact', text: 'Launch is Monday.' }) + + expect(episodeIds).toHaveLength(2) + expect(episodeIds[0]).not.toBe(episodeIds[1]) + }) + + it('keeps repeated id-less Graphiti writes as distinct episodes', async () => { + const episodeIds: string[] = [] + const adapter = createGraphitiMemoryAdapter({ + consistency: 'queued', + client: { + async callTool(request) { + if (request.name === 'add_memory') { + episodeIds.push(String(request.arguments?.uuid)) + } + return { structuredContent: { message: 'queued' } } + }, + }, + }) + + await adapter.write({ kind: 'fact', text: 'The same observation.' }) + await adapter.write({ kind: 'fact', text: 'The same observation.' }) + + expect(episodeIds).toHaveLength(2) + expect(episodeIds[0]).not.toBe(episodeIds[1]) + }) + + it('keeps id-less Graphiti writes distinct across adapter restarts', async () => { + const episodeIds: string[] = [] + const client: GraphitiMcpClientLike = { + async callTool(request) { + if (request.name === 'add_memory') episodeIds.push(String(request.arguments?.uuid)) + return { structuredContent: { message: 'queued' } } + }, + } + + await createGraphitiMemoryAdapter({ client, consistency: 'queued' }).write({ + kind: 'fact', + text: 'The same observation.', + }) + await createGraphitiMemoryAdapter({ client, consistency: 'queued' }).write({ + kind: 'fact', + text: 'The same observation.', + }) + + expect(episodeIds).toHaveLength(2) + expect(episodeIds[0]).not.toBe(episodeIds[1]) + }) + + it('treats an empty Graphiti kind list as unfiltered search', async () => { + const names: string[] = [] + const adapter = createGraphitiMemoryAdapter({ + client: { + async callTool(request) { + names.push(request.name) + if (request.name === 'search_memory_facts') { + return { structuredContent: { facts: [] } } + } + if (request.name === 'search_nodes') return { structuredContent: { nodes: [] } } + throw new Error(`unexpected tool ${request.name}`) + }, + }, + }) + + await adapter.search('anything', { kinds: [] }) + + expect(names).toEqual(['search_memory_facts', 'search_nodes']) + }) + + it('fuses unscored Graphiti fact and node rankings before applying the limit', async () => { + const adapter = createGraphitiMemoryAdapter({ + client: { + async callTool(request) { + if (request.name === 'search_memory_facts') { + return { + structuredContent: { + facts: [ + { uuid: 'fact-1', fact: 'first fact' }, + { uuid: 'fact-2', fact: 'second fact' }, + ], + }, + } + } + if (request.name === 'search_nodes') { + return { + structuredContent: { + nodes: [ + { uuid: 'node-1', name: 'first node' }, + { uuid: 'node-2', name: 'second node' }, + ], + }, + } + } + throw new Error(`unexpected tool ${request.name}`) + }, + }, + }) + + const hits = await adapter.search('anything', { limit: 2 }) + + expect(hits.map((hit) => hit.id)).toEqual(['fact-1', 'node-1']) + }) + + it('uses Graphiti clear_graph for exact group cleanup', async () => { + const requests: Array<{ name: string; arguments?: Record }> = [] + const adapter = createGraphitiMemoryAdapter({ + client: { + async callTool(request) { + requests.push(request) + return { structuredContent: { message: 'Graph cleared successfully' } } + }, + }, + }) + + await adapter.clear?.({ userId: 'user-1', namespace: 'candidate-1' }) + + expect(requests).toHaveLength(1) + expect(requests[0]).toMatchObject({ + name: 'clear_graph', + arguments: { group_ids: [expect.stringMatching(/^ak_[a-f0-9]{32}$/)] }, + }) + }) + + it('rejects queued ingestion for resumable branch experiments', () => { + const adapter = createGraphitiMemoryAdapter({ + consistency: 'queued', + client: { + async callTool() { + return { structuredContent: { message: 'queued' } } + }, + }, + }) + + expect(() => createAgentMemoryBranch({ adapter, branchId: 'candidate-1' })).toThrow( + 'queued Graphiti writes can become visible after a process restart', + ) + }) + + it('includes the default scope in the Graphiti adapter identity', () => { + expect( + graphitiMemoryAdapterIdentity({ + id: 'graphiti', + backendRef: 'graphiti-cluster-a', + defaultScope: { namespace: 'one' }, + }), + ).not.toBe( + graphitiMemoryAdapterIdentity({ + id: 'graphiti', + backendRef: 'graphiti-cluster-a', + defaultScope: { namespace: 'two' }, + }), + ) + }) +}) + +describe('Neo4j Agent Memory adapter', () => { + it('is type-compatible with the current official client', () => { + const client = {} as unknown as Neo4jMemoryClient + expect( + createNeo4jAgentMemoryAdapter({ client, transport: 'rest', branchId: 'branch-1' }).id, + ).toBe('neo4j-agent-memory') + }) + + it('stores observations as searchable messages without retrying a successful void write', async () => { + let camelWrites = 0 + let snakeWrites = 0 + let role: unknown + let closes = 0 + const adapter = createNeo4jAgentMemoryAdapter({ + transport: 'rest', + branchId: 'branch-1', + client: { + shortTerm: { + async addMessage(_sessionId: unknown, inputRole: unknown) { + camelWrites += 1 + role = inputRole + return undefined + }, + async add_message() { + snakeWrites += 1 + return { id: 'duplicate' } + }, + async searchMessages() { + return [ + { + id: 'observation-1', + role: 'assistant', + content: 'The deployment completed.', + metadata: { agentKnowledgeKind: 'observation' }, + }, + ] + }, + }, + async close() { + closes += 1 + }, + }, + }) + + const write = await adapter.write({ + kind: 'observation', + role: 'tool', + text: 'The deployment completed.', + scope: { sessionId: 'session-1' }, + }) + const hits = await adapter.search('deployment', { + kinds: ['observation'], + scope: { sessionId: 'session-1' }, + }) + await adapter.close?.() + + expect(write.accepted).toBe(true) + expect(camelWrites).toBe(1) + expect(snakeWrites).toBe(0) + expect(role).toBe('assistant') + expect(hits).toMatchObject([{ id: 'observation-1', kind: 'observation' }]) + expect(closes).toBe(1) + }) + + it('fails clearly when REST is asked for bridge-only facts and preferences', async () => { + const adapter = createNeo4jAgentMemoryAdapter({ + transport: 'rest', + client: { longTerm: {} }, + }) + + await expect(adapter.write({ kind: 'fact', text: 'A fact' })).rejects.toThrow( + 'Neo4j REST cannot write fact', + ) + await expect(adapter.search('preference', { kinds: ['preference'] })).rejects.toThrow( + 'Neo4j rest cannot search memory kinds: preference', + ) + }) + + it('rejects an incompatible Neo4j client instead of reporting an empty search', async () => { + const adapter = createNeo4jAgentMemoryAdapter({ + client: { longTerm: {} }, + transport: 'rest', + }) + + await expect(adapter.search('company', { kinds: ['entity'] })).rejects.toThrow( + 'missing method searchEntities', + ) + }) + + it('rejects non-throwing Neo4j write errors', async () => { + const adapter = createNeo4jAgentMemoryAdapter({ + transport: 'rest', + client: { + shortTerm: { + async addMessage() { + return { success: false, error: { message: 'database unavailable' } } + }, + }, + }, + }) + + await expect( + adapter.write({ + kind: 'message', + text: 'Remember this.', + scope: { sessionId: 'session-1' }, + }), + ).rejects.toThrow('database unavailable') + }) + + it('maps hosted reasoning memories to recordStep', async () => { + let recorded: Record | undefined + const adapter = createNeo4jAgentMemoryAdapter({ + transport: 'rest', + client: { + reasoning: { + async recordStep(input: Record) { + recorded = input + return { id: 'step-1', ...input } + }, + }, + }, + }) + + const result = await adapter.write({ + kind: 'reasoning-trace', + text: 'Check the deployment status.', + title: 'status-check', + scope: { sessionId: 'conversation-1' }, + metadata: { action: 'query_status', result: 'healthy' }, + }) + + expect(recorded).toEqual({ + conversationId: 'conversation-1', + reasoning: 'Check the deployment status.', + actionTaken: 'query_status', + result: 'healthy', + }) + expect(result).toMatchObject({ accepted: true, id: 'step-1', kind: 'reasoning-trace' }) + }) + + it('uses bridge preference APIs only in bridge mode', async () => { + const adapter = createNeo4jAgentMemoryAdapter({ + transport: 'bridge', + client: { + longTerm: { + async addPreference(category: string, preference: string) { + return { id: 'preference-1', category, preference } + }, + async searchPreferences() { + return [{ id: 'preference-1', category: 'style', preference: 'Be concise.' }] + }, + }, + }, + }) + + await expect( + adapter.write({ kind: 'preference', category: 'style', text: 'Be concise.' }), + ).resolves.toMatchObject({ id: 'preference-1' }) + await expect(adapter.search('concise', { kinds: ['preference'] })).resolves.toMatchObject([ + { id: 'preference-1', kind: 'preference', text: 'Be concise.' }, + ]) + }) + + it('fuses unscored Neo4j memory-type rankings before applying the limit', async () => { + const adapter = createNeo4jAgentMemoryAdapter({ + transport: 'bridge', + client: { + shortTerm: { + async searchMessages() { + return [ + { id: 'message-1', role: 'user', content: 'first message' }, + { id: 'message-2', role: 'user', content: 'second message' }, + ] + }, + }, + longTerm: { + async searchEntities() { + return [ + { id: 'entity-1', name: 'first entity', type: 'custom' }, + { id: 'entity-2', name: 'second entity', type: 'custom' }, + ] + }, + async searchPreferences() { + return [] + }, + }, + reasoning: { + async getSimilarTraces() { + return [] + }, + }, + }, + }) + + const hits = await adapter.search('anything', { limit: 2 }) + + expect(hits.map((hit) => hit.id)).toEqual(['message-1', 'entity-1']) + }) +}) + +describe('memory branches', () => { + it('rejects two live handles for the same adapter branch', async () => { + const adapter = createScopedTestAdapter('duplicate-handle') + const first = createAgentMemoryBranch({ adapter, branchId: 'same-branch' }) + + expect(() => createAgentMemoryBranch({ adapter, branchId: 'same-branch' })).toThrow( + "memory branch 'same-branch' already has an open handle", + ) + + await first.close?.() + const resumed = createAgentMemoryBranch({ adapter, branchId: 'same-branch' }) + await resumed.close?.() + }) + + it('rejects adapters that do not declare how branches are isolated', () => { + const { branchIsolation: _branchIsolation, ...legacy } = createScopedTestAdapter('legacy') + + expect(() => + createAgentMemoryBranch({ + adapter: legacy, + branchId: 'candidate-a', + }), + ).toThrow(/adapter must declare branchIsolation/) + }) + + it('rejects Neo4j clients that were not isolated for the exact branch', () => { + const unscoped = createNeo4jAgentMemoryAdapter({ client: {}, transport: 'rest' }) + expect(() => createAgentMemoryBranch({ adapter: unscoped, branchId: 'candidate-a' })).toThrow( + 'create a separate MemoryClient namespace per branch', + ) + + const scoped = createNeo4jAgentMemoryAdapter({ + client: {}, + transport: 'rest', + branchId: 'candidate-a', + }) + expect( + createAgentMemoryBranch({ + adapter: scoped, + branchId: 'candidate-a', + policy: { read: ['shared'], write: 'shared' }, + }).branchId, + ).toBe('candidate-a') + expect(() => createAgentMemoryBranch({ adapter: scoped, branchId: 'candidate-a' })).toThrow( + 'only shared memory policy', + ) + expect(() => createAgentMemoryBranch({ adapter: scoped, branchId: 'candidate-b' })).toThrow( + "adapter instance belongs to branch 'candidate-a'", + ) + }) + + it('isolates branches and private agents while allowing team sharing', async () => { + const storage = createScopedTestAdapter('scoped') + const alpha = createAgentMemoryBranch({ + adapter: storage, + branchId: 'alpha', + policy: { read: ['private'], write: 'private' }, + baseScope: { tenantId: 'tenant', userId: 'user' }, + }) + const beta = createAgentMemoryBranch({ + adapter: storage, + branchId: 'beta', + policy: { read: ['private'], write: 'private' }, + baseScope: { tenantId: 'tenant', userId: 'user' }, + }) + + await alpha.write({ + kind: 'fact', + text: 'alpha private', + scope: { agentId: 'agent-a', teamId: 'team-1' }, + }) + await beta.write({ + kind: 'fact', + text: 'beta private', + scope: { agentId: 'agent-a', teamId: 'team-1' }, + }) + + expect((await alpha.search('private', { scope: { agentId: 'agent-a' } })).map(hitText)).toEqual( + ['alpha private'], + ) + expect((await alpha.search('private', { scope: { agentId: 'agent-b' } })).map(hitText)).toEqual( + [], + ) + expect((await beta.search('private', { scope: { agentId: 'agent-a' } })).map(hitText)).toEqual([ + 'beta private', + ]) + + const team = createAgentMemoryBranch({ + adapter: storage, + branchId: 'team-branch', + policy: { read: ['team'], write: 'team' }, + baseScope: { tenantId: 'tenant', userId: 'user' }, + }) + await team.write({ + kind: 'observation', + text: 'shared with the team', + scope: { agentId: 'agent-a', teamId: 'team-1' }, + }) + const teamHits = await team.search('team', { + scope: { agentId: 'agent-b', teamId: 'team-1' }, + }) + expect(teamHits.map(hitText)).toEqual(['shared with the team']) + }) + + it('removes every journal entry in a provider partition cleared through a narrower scope', async () => { + const branch = createAgentMemoryBranch({ + adapter: createScopedTestAdapter('team-clear'), + branchId: 'team-clear', + policy: { read: ['team'], write: 'team' }, + baseScope: { tenantId: 'tenant', teamId: 'team-1' }, + }) + await branch.write({ kind: 'fact', text: 'from a', scope: { agentId: 'agent-a' } }) + await branch.write({ kind: 'fact', text: 'from b', scope: { agentId: 'agent-b' } }) + + await branch.clear?.({ agentId: 'agent-a' }) + + expect((await branch.snapshot()).journal).toEqual([]) + await expect(branch.search('from b', { scope: { agentId: 'agent-b' } })).resolves.toEqual([]) + }) + + it('preserves provider order for unscored branch hits', async () => { + const hits: AgentMemoryHit[] = [ + { id: 'z-top', uri: 'memory://rank/z', kind: 'fact', text: 'provider first' }, + { id: 'a-lower', uri: 'memory://rank/a', kind: 'fact', text: 'provider second' }, + ] + const adapter: AgentMemoryAdapter = { + id: 'provider-ranked', + branchIsolation: { mode: 'scoped' }, + async search() { + return hits + }, + async getContext(query) { + return { query, text: '', hits: [], sourceRecords: [] } + }, + async write(input) { + return { accepted: true, id: 'write', uri: 'memory://rank/write', kind: input.kind } + }, + } + const branch = createAgentMemoryBranch({ + adapter, + branchId: 'provider-ranked', + baseScope: { agentId: 'agent-a' }, + }) + + const ranked = await branch.search('anything', { limit: 1 }) + + expect(ranked.map((hit) => hit.id)).toEqual(['z-top']) + }) + + it('fuses unscored visibility lists by provider rank', async () => { + const adapter: AgentMemoryAdapter = { + id: 'visibility-ranked', + branchIsolation: { mode: 'scoped' }, + async search(_query, options) { + return options?.scope?.tags?.memoryVisibility === 'private' + ? [ + { id: 'private-1', uri: 'memory://rank/private-1', kind: 'fact', text: 'p1' }, + { id: 'private-2', uri: 'memory://rank/private-2', kind: 'fact', text: 'p2' }, + ] + : [{ id: 'team-1', uri: 'memory://rank/team-1', kind: 'fact', text: 't1' }] + }, + async getContext(query) { + return { query, text: '', hits: [], sourceRecords: [] } + }, + async write(input) { + return { accepted: true, id: 'write', uri: 'memory://rank/write', kind: input.kind } + }, + } + const branch = createAgentMemoryBranch({ + adapter, + branchId: 'visibility-ranked', + policy: { read: ['private', 'team'], write: 'private' }, + baseScope: { agentId: 'agent-a', teamId: 'team-a' }, + }) + + const ranked = await branch.search('anything', { limit: 2 }) + + expect(ranked.map((hit) => hit.id)).toEqual(['private-1', 'team-1']) + }) + + it('uses rank fusion before incomparable scores from separate provider searches', async () => { + const adapter: AgentMemoryAdapter = { + id: 'visibility-scored', + branchIsolation: { mode: 'scoped' }, + async search(_query, options) { + return options?.scope?.tags?.memoryVisibility === 'private' + ? [ + { + id: 'private-first', + uri: 'memory://rank/private-first', + kind: 'fact', + text: 'private first', + score: 0.01, + }, + { + id: 'private-second', + uri: 'memory://rank/private-second', + kind: 'fact', + text: 'private second', + score: 1, + }, + ] + : [ + { + id: 'team-first', + uri: 'memory://rank/team-first', + kind: 'fact', + text: 'team first', + score: 0.02, + }, + ] + }, + async getContext(query) { + return { query, text: '', hits: [], sourceRecords: [] } + }, + async write(input) { + return { accepted: true, id: 'write', uri: 'memory://rank/write', kind: input.kind } + }, + } + const branch = createAgentMemoryBranch({ + adapter, + branchId: 'visibility-scored', + policy: { read: ['private', 'team'], write: 'private' }, + baseScope: { agentId: 'agent-a', teamId: 'team-a' }, + }) + + const ranked = await branch.search('anything', { limit: 2 }) + + expect(ranked.map((hit) => hit.id)).toEqual(['team-first', 'private-first']) + }) + + it('serializes writes per actor and permits independent actors in parallel', async () => { + let active = 0 + let maxActive = 0 + const starts: string[] = [] + const storage = createScopedTestAdapter('ordered', async (scope, text) => { + starts.push(`${scope.agentId}:${text}`) + active += 1 + maxActive = Math.max(maxActive, active) + await new Promise((resolve) => setTimeout(resolve, 10)) + active -= 1 + }) + const branch = createAgentMemoryBranch({ + adapter: storage, + branchId: 'parallel', + policy: { read: ['private'], write: 'private' }, + }) + + await Promise.all([ + branch.write({ kind: 'message', text: 'a1', scope: { agentId: 'a' } }), + branch.write({ kind: 'message', text: 'a2', scope: { agentId: 'a' } }), + branch.write({ kind: 'message', text: 'b1', scope: { agentId: 'b' } }), + ]) + + expect(maxActive).toBe(2) + expect(starts.indexOf('a:a1')).toBeLessThan(starts.indexOf('a:a2')) + expect((await branch.snapshot()).journal.map((entry) => entry.input.text)).toEqual([ + 'a1', + 'a2', + 'b1', + ]) + }) + + it('preserves one actor ordering across sessions and tags', async () => { + const completed: string[] = [] + const storage = createScopedTestAdapter('actor-order', async (_scope, text) => { + if (text === 'first') await new Promise((resolve) => setTimeout(resolve, 20)) + completed.push(text) + }) + const branch = createAgentMemoryBranch({ + adapter: storage, + branchId: 'actor-order', + baseScope: { tenantId: 'tenant', agentId: 'agent-a' }, + }) + + await Promise.all([ + branch.write({ + kind: 'fact', + text: 'first', + scope: { sessionId: 'session-1', tags: { task: 'one' } }, + }), + branch.write({ + kind: 'fact', + text: 'second', + scope: { sessionId: 'session-2', tags: { task: 'two' } }, + }), + ]) + + expect(completed).toEqual(['first', 'second']) + }) + + it('takes a point-in-time snapshot while later writes wait at the boundary', async () => { + let releaseFirst!: () => void + const firstMayFinish = new Promise((resolve) => { + releaseFirst = resolve + }) + let reportFirstStarted!: () => void + const firstStarted = new Promise((resolve) => { + reportFirstStarted = resolve + }) + const providerStarts: string[] = [] + const storage = createScopedTestAdapter('snapshot-boundary', async (_scope, text) => { + providerStarts.push(text) + if (text === 'first') { + reportFirstStarted() + await firstMayFinish + } + }) + const branch = createAgentMemoryBranch({ + adapter: storage, + branchId: 'snapshot-boundary', + baseScope: { agentId: 'worker' }, + }) + + const firstWrite = branch.write({ kind: 'fact', text: 'first' }) + await firstStarted + const snapshotPromise = branch.snapshot() + const secondWrite = branch.write({ kind: 'fact', text: 'second' }) + await Promise.resolve() + + expect(providerStarts).toEqual(['first']) + releaseFirst() + await firstWrite + const snapshot = await snapshotPromise + await secondWrite + + expect(snapshot.journal.map((entry) => entry.input.text)).toEqual(['first']) + expect(providerStarts).toEqual(['first', 'second']) + }) + + it('does not clear storage while a preceding read is still using it', async () => { + let finishRead!: () => void + const readMayFinish = new Promise((resolve) => { + finishRead = resolve + }) + let reportReadStarted!: () => void + const readStarted = new Promise((resolve) => { + reportReadStarted = resolve + }) + let clearStarted = false + const adapter: AgentMemoryAdapter = { + id: 'read-clear-boundary', + branchIsolation: { mode: 'scoped' }, + async search() { + reportReadStarted() + await readMayFinish + return [] + }, + async getContext(query) { + return { query, text: '', hits: [], sourceRecords: [] } + }, + async write(input) { + return { accepted: true, id: 'write', uri: 'memory://write', kind: input.kind } + }, + async clear() { + clearStarted = true + }, + } + const branch = createAgentMemoryBranch({ + adapter, + branchId: 'read-clear-boundary', + baseScope: { agentId: 'worker' }, + }) + + const read = branch.search('active read') + await readStarted + const clear = branch.clear?.() + await Promise.resolve() + expect(clearStarted).toBe(false) + + finishRead() + await read + await clear + expect(clearStarted).toBe(true) + }) + + it('clears a touched scope after the provider commits and then throws', async () => { + let dirty = false + const clearedScopes: AgentMemoryScope[] = [] + const adapter: AgentMemoryAdapter = { + id: 'commit-then-error', + branchIsolation: { mode: 'scoped' }, + async search() { + return [] + }, + async getContext(query) { + return { query, text: '', hits: [], sourceRecords: [] } + }, + async write() { + dirty = true + throw new Error('connection lost after commit') + }, + async clear(scope) { + clearedScopes.push(scope ?? {}) + dirty = false + }, + } + const branch = createAgentMemoryBranch({ + adapter, + branchId: 'commit-then-error', + baseScope: { tenantId: 'tenant' }, + }) + + await expect( + branch.write({ kind: 'fact', text: 'partial', scope: { agentId: 'worker' } }), + ).rejects.toThrow('connection lost after commit') + await branch.clear?.() + + expect(dirty).toBe(false) + expect(clearedScopes).toHaveLength(1) + expect(clearedScopes[0]).toMatchObject({ tenantId: 'tenant', agentId: 'worker' }) + }) + + it('rejects non-durable journal data before it can corrupt resume state', async () => { + let providerWrites = 0 + const adapter = createScopedTestAdapter('durable-journal', async () => { + providerWrites += 1 + }) + const branch = createAgentMemoryBranch({ + adapter, + branchId: 'durable-journal', + baseScope: { agentId: 'worker' }, + }) + const cyclic: Record = {} + cyclic.self = cyclic + + await expect( + branch.write({ kind: 'fact', text: 'bad metadata', metadata: cyclic }), + ).rejects.toThrow('must not contain cycles') + expect(providerWrites).toBe(0) + expect((await branch.snapshot()).journal).toEqual([]) + }) + + it('returns snapshots detached from live branch state', async () => { + const adapter = createScopedTestAdapter('detached-snapshot') + const branch = createAgentMemoryBranch({ + adapter, + branchId: 'detached-snapshot', + baseScope: { tenantId: 'tenant-original', agentId: 'agent-a' }, + }) + await branch.write({ + kind: 'fact', + text: 'original text', + metadata: { nested: { status: 'original' } }, + }) + + const exposed = await branch.snapshot() + exposed.baseScope.tenantId = 'tenant-mutated' + const nested = exposed.journal[0]!.input.metadata!.nested as Record + nested.status = 'mutated' + const current = await branch.snapshot() + + expect(current.baseScope.tenantId).toBe('tenant-original') + expect(current.journal[0]?.input.metadata).toEqual({ nested: { status: 'original' } }) + expect(await branch.search('original')).toHaveLength(1) + }) + + it('closes a shared adapter once after every branch releases it', async () => { + let closes = 0 + const base = createScopedTestAdapter('shared-lifecycle') + const adapter: AgentMemoryAdapter = { + ...base, + async close() { + closes += 1 + }, + } + const first = createAgentMemoryBranch({ + adapter, + branchId: 'first', + baseScope: { agentId: 'agent-1' }, + }) + const second = createAgentMemoryBranch({ + adapter, + branchId: 'second', + baseScope: { agentId: 'agent-2' }, + }) + + await first.close?.() + await first.close?.() + expect(closes).toBe(0) + await second.close?.() + await second.close?.() + + expect(closes).toBe(1) + await expect(first.search('after close')).rejects.toThrow('closed branch') + }) + + it('waits for a concurrent fork to finish before closing the parent', async () => { + let releaseSnapshot!: () => void + let releaseReplay!: () => void + let reportReplayStarted!: () => void + const snapshotBlocked = new Promise((resolve) => { + releaseSnapshot = resolve + }) + const replayBlocked = new Promise((resolve) => { + releaseReplay = resolve + }) + const replayStarted = new Promise((resolve) => { + reportReplayStarted = resolve + }) + const base = createScopedTestAdapter('fork-close-race') + let writes = 0 + let flushes = 0 + let closes = 0 + const adapter: AgentMemoryAdapter = { + ...base, + async write(input) { + writes += 1 + if (writes === 2) { + reportReplayStarted() + await replayBlocked + } + return base.write(input) + }, + async flush() { + flushes += 1 + if (flushes === 1) await snapshotBlocked + }, + async close() { + closes += 1 + }, + } + const parent = createAgentMemoryBranch({ + adapter, + branchId: 'fork-close-parent', + baseScope: { agentId: 'worker' }, + }) + await parent.write({ kind: 'fact', text: 'replay me' }) + + const fork = parent.fork({ branchId: 'fork-close-child' }) + const close = parent.close!() + let closeSettled = false + void close.finally(() => { + closeSettled = true + }) + releaseSnapshot() + await replayStarted + await Promise.resolve() + await Promise.resolve() + + expect(closeSettled).toBe(false) + expect(closes).toBe(0) + releaseReplay() + const child = await fork + await close + expect((await child.search('replay')).map(hitText)).toEqual(['replay me']) + expect(closes).toBe(0) + await child.close?.() + expect(closes).toBe(1) + }) + + it('forks by replaying accepted writes into another adapter and resumes exactly', async () => { + const parentStorage = createScopedTestAdapter('parent') + const childStorage = createScopedTestAdapter('child') + const parent = createAgentMemoryBranch({ + adapter: parentStorage, + branchId: 'parent', + policy: { read: ['private'], write: 'private' }, + baseScope: { tenantId: 'tenant', agentId: 'agent-a' }, + }) + await parent.write({ kind: 'fact', text: 'seed fact' }) + + await expect(parent.fork({ branchId: 'parent' })).rejects.toThrow( + 'child branchId must differ from its parent', + ) + expect((await parent.search('fact')).map(hitText)).toEqual(['seed fact']) + + const child = await parent.fork({ branchId: 'child', adapter: childStorage }) + await child.write({ kind: 'fact', text: 'child only' }) + + expect((await parent.search('fact')).map(hitText)).toEqual(['seed fact']) + expect((await child.search('fact')).map(hitText)).toEqual(['seed fact', 'child only']) + const snapshot = await child.snapshot() + expect(snapshot.journal.every((entry) => Boolean(entry.input.id))).toBe(true) + expect(new Set(snapshot.journal.map((entry) => entry.input.id)).size).toBe(2) + await child.close?.() + const resumed = createAgentMemoryBranch({ + adapter: childStorage, + branchId: 'child', + snapshot, + }) + snapshot.baseScope.tenantId = 'mutated-after-resume' + expect((await resumed.search('fact')).map(hitText)).toEqual(['seed fact', 'child only']) + expect((await resumed.snapshot()).baseScope.tenantId).toBe('tenant') + expect((await resumed.snapshot()).digest).toBe(snapshot.digest) + + await expect( + Promise.resolve().then(() => + createAgentMemoryBranch({ + adapter: childStorage, + branchId: 'child', + snapshot: { ...snapshot, digest: 'sha256:bad' }, + }), + ), + ).rejects.toThrow('digest mismatch') + }) + + it('replays an attempt snapshot into a fresh branch', async () => { + const adapter: AgentMemoryAdapter = { + ...createScopedTestAdapter('attempt-source'), + branchIsolation: { mode: 'scoped', processExitSafe: false, recoveryDelayMs: 1 }, + } + const source = createAgentMemoryBranch({ + adapter, + branchId: 'attempt-source', + lifetime: 'attempt', + baseScope: { agentId: 'worker' }, + }) + await source.write({ kind: 'fact', text: 'durable observation' }) + const snapshot = await source.snapshot() + const target = await forkAgentMemoryBranchSnapshot({ + snapshot, + adapter: createScopedTestAdapter('resumable-target'), + branchId: 'resumable-target', + lifetime: 'resumable', + }) + + expect((await target.search('observation')).map(hitText)).toEqual(['durable observation']) + expect(target.parentBranchId).toBe('attempt-source') + expect(target.lifetime).toBe('resumable') + }) +}) + +describe('agent memory experiments', () => { + it('versions memory scoring for resumable cache safety', () => { + expect(agentMemorySequenceJudge().judgeVersion).toBe('agent-knowledge:memory-sequence:v2') + }) + + it('requires an explicit controller policy for custom storage', async () => { + await expect( + runAgentMemoryExperimentRaw({ + experimentId: 'custom-storage-controller', + sequences: [ + { + id: 'history', + family: 'first-party', + steps: [{ id: 'probe', probes: [{ id: 'probe', query: 'x', referenceAnswer: 'x' }] }], + }, + ], + candidates: [ + { + id: 'memory', + ref: 'memory:v1', + createAdapter: () => createScopedTestAdapter('memory'), + }, + ], + runDir: '/runs/custom-storage-controller', + storage: inMemoryCampaignStorage(), + }), + ).rejects.toThrow("requires acquireRunLease or controllerMode='process-local'") + }) + + it('bounds provider cleanup and leaves the attempt available for recovery', async () => { + const storage = inMemoryCampaignStorage() + let clearCalls = 0 + const adapter = createScopedTestAdapter('bounded-cleanup') + const clear = adapter.clear! + adapter.clear = async (scope) => { + clearCalls += 1 + if (clearCalls === 1) return new Promise(() => {}) + await clear(scope) + } + const startedAt = Date.now() + + await expect( + runAgentMemoryExperiment({ + experimentId: 'bounded-cleanup', + sequences: [ + { + id: 'history', + family: 'first-party', + steps: [ + { + id: 'remember', + scope: { agentId: 'worker' }, + writes: [{ kind: 'fact', text: 'bounded fact' }], + probes: [{ id: 'probe', query: 'bounded', referenceAnswer: 'bounded fact' }], + }, + ], + }, + ], + candidates: [{ id: 'memory', ref: 'memory:v1', createAdapter: () => adapter }], + runDir: '/runs/bounded-cleanup', + storage, + cleanupTimeoutMs: 10, + }), + ).rejects.toThrow('memory experiment cleanup failed after dispatch') + + expect(Date.now() - startedAt).toBeLessThan(500) + expect( + storage.read('/runs/bounded-cleanup/memory-attempts.jsonl')?.trim().split('\n'), + ).toHaveLength(1) + }) + + it('does not charge provider cost when side-effect-free adapter construction fails', async () => { + const storage = inMemoryCampaignStorage() + const costLedger = createRunCostLedger({ + storage, + runDir: '/runs/create-adapter-cost', + costCeilingUsd: 1, + }) + + await expect( + runAgentMemoryExperiment({ + experimentId: 'create-adapter-cost', + sequences: [ + { + id: 'history', + family: 'first-party', + steps: [{ id: 'probe', probes: [{ id: 'probe', query: 'x', referenceAnswer: 'x' }] }], + }, + ], + candidates: [ + { + id: 'paid-memory', + ref: 'paid-memory:v1', + externalCostUsdPerSequence: 0.25, + createAdapter() { + throw new Error('provider setup rejected') + }, + }, + ], + runDir: '/runs/create-adapter-cost', + storage, + costLedger, + }), + ).rejects.toThrow('memory experiment cleanup failed after dispatch') + + expect(costLedger.summary().totalCostUsd).toBe(0) + }) + + it('charges a factory failure after dedicated provider provisioning starts', async () => { + const storage = inMemoryCampaignStorage() + const costLedger = createRunCostLedger({ + storage, + runDir: '/runs/create-adapter-provider-cost', + costCeilingUsd: 1, + }) + + await expect( + runAgentMemoryExperiment({ + experimentId: 'create-adapter-provider-cost', + sequences: [ + { + id: 'history', + family: 'first-party', + steps: [{ id: 'probe', probes: [{ id: 'probe', query: 'x', referenceAnswer: 'x' }] }], + }, + ], + candidates: [ + { + id: 'paid-memory', + ref: 'paid-memory:v1', + externalCostUsdPerSequence: 0.25, + createAdapter({ markExternalCall }) { + markExternalCall() + throw new Error('provider provisioning failed') + }, + }, + ], + runDir: '/runs/create-adapter-provider-cost', + storage, + costLedger, + }), + ).rejects.toThrow('memory experiment cleanup failed after dispatch') + + expect(costLedger.summary()).toMatchObject({ totalCalls: 1, totalCostUsd: 0.25 }) + }) + + it('records provider cleanup before a paid-call receipt can be interrupted', async () => { + const storage = inMemoryCampaignStorage() + const append = storage.append!.bind(storage) + let interruptReceipt = true + storage.append = (path, value, expectedBytes) => { + if ( + interruptReceipt && + path.endsWith('/cost-ledger.jsonl') && + value.includes('"status":"settled"') && + value.includes('agent-knowledge:memory-experiment') + ) { + interruptReceipt = false + throw new Error('simulated process exit before execute receipt') + } + return append(path, value, expectedBytes) + } + const purposes: string[] = [] + const sequence = { + id: 'history', + family: 'first-party' as const, + steps: [ + { + id: 'remember', + scope: { agentId: 'worker' }, + writes: [{ kind: 'fact' as const, text: 'durable fact' }], + probes: [{ id: 'recall', query: 'durable', referenceAnswer: 'durable fact' }], + }, + ], + } + const run = () => + runAgentMemoryExperiment({ + experimentId: 'execute-receipt-crash', + sequences: [sequence], + candidates: [ + { + id: 'memory', + ref: 'memory:v1', + externalCostUsdPerSequence: 0.1, + createAdapter({ purpose }) { + purposes.push(purpose) + return createScopedTestAdapter(`memory:${purpose}`) + }, + }, + ], + runDir: '/runs/execute-receipt-crash', + storage, + costCeiling: 1, + }) + + const interrupted = await run() + expect(interrupted.campaign.aggregates).toMatchObject({ + cellsFailed: 1, + cost: { accountingComplete: false, unresolvedCalls: 1 }, + }) + expect( + storage.read('/runs/execute-receipt-crash/memory-attempts.jsonl')?.trim().split('\n'), + ).toHaveLength(2) + + const result = await run() + expect(purposes).toEqual(['execute', 'execute']) + expect(result.rows[0]).toMatchObject({ cellsFailed: 0, totalCostUsd: 0.2 }) + }) + + it('accounts for parallel candidates against one shared dollar limit', async () => { + let releaseFirstCalls: (() => void) | undefined + const firstCallsReady = new Promise((resolve) => { + releaseFirstCalls = resolve + }) + let activeCalls = 0 + let firstCalls = 0 + let maxActiveCalls = 0 + const createAdapter = (id: string): AgentMemoryAdapter => { + const adapter = createScopedTestAdapter(id) + const clear = adapter.clear! + let entered = false + adapter.clear = async (scope) => { + if (!entered) { + entered = true + firstCalls += 1 + activeCalls += 1 + maxActiveCalls = Math.max(maxActiveCalls, activeCalls) + if (firstCalls === 2) releaseFirstCalls?.() + await firstCallsReady + activeCalls -= 1 + } + await clear(scope) + } + return adapter + } + + const result = await runAgentMemoryExperiment({ + experimentId: 'parallel-shared-cost', + sequences: [ + { + id: 'history', + family: 'first-party', + steps: [ + { + id: 'remember', + scope: { agentId: 'worker' }, + writes: [{ kind: 'fact', text: 'parallel fact' }], + probes: [ + { + id: 'recall', + query: 'parallel', + scope: { agentId: 'worker' }, + referenceAnswer: 'parallel fact', + }, + ], + }, + ], + }, + ], + candidates: ['first', 'second'].map((id) => ({ + id, + ref: `${id}:v1`, + externalCostUsdPerSequence: 0.1, + createAdapter: () => createAdapter(id), + })), + runDir: '/runs/parallel-shared-cost', + storage: inMemoryCampaignStorage(), + costCeiling: 0.2, + maxConcurrency: 2, + }) + + expect(maxActiveCalls).toBe(2) + expect(result.rows.map((row) => row.totalCostUsd).sort()).toEqual([0.1, 0.1]) + expect(result.campaign.aggregates.totalCostUsd).toBe(0.2) + }) + + it('runs sequential paid histories after exact external-cost receipts', async () => { + const storage = inMemoryCampaignStorage() + const costLedger = createRunCostLedger({ + storage, + runDir: '/runs/sequential-shared-cost', + costCeilingUsd: 0.2, + }) + const sequences = ['first', 'second'].map((id) => ({ + id, + family: 'first-party', + steps: [ + { + id: 'remember', + scope: { agentId: 'worker' }, + writes: [{ kind: 'fact' as const, text: `${id} fact` }], + probes: [ + { + id: 'recall', + query: id, + scope: { agentId: 'worker' }, + referenceAnswer: `${id} fact`, + }, + ], + }, + ], + })) + + const result = await runAgentMemoryExperiment({ + experimentId: 'sequential-shared-cost', + sequences, + candidates: [ + { + id: 'memory', + ref: 'memory:v1', + externalCostUsdPerSequence: 0.1, + createAdapter: ({ sequence }) => createScopedTestAdapter(sequence.id), + }, + ], + runDir: '/runs/sequential-shared-cost', + storage, + costLedger, + maxConcurrency: 1, + }) + + expect(result.rows[0]).toMatchObject({ cellsFailed: 0, totalCostUsd: 0.2 }) + expect(costLedger.summary()).toMatchObject({ + totalCalls: 2, + totalCostUsd: 0.2, + accountingComplete: true, + }) + }) + + it('reports the reserved cost of an interrupted prior cell attempt', async () => { + const storage = inMemoryCampaignStorage() + const runDir = '/runs/interrupted-cell-cost-reporting' + const sequence = { + id: 'history', + family: 'first-party' as const, + steps: [ + { + id: 'remember', + scope: { agentId: 'worker' }, + writes: [{ kind: 'fact' as const, text: 'resumed fact' }], + probes: [{ id: 'recall', query: 'resumed', referenceAnswer: 'resumed fact' }], + }, + ], + } + const candidate = { + id: 'memory', + ref: 'memory:v1', + externalCostUsdPerSequence: 0.1, + createAdapter: () => createScopedTestAdapter('memory'), + } + const scenarioId = buildAgentMemorySequenceScenarios([sequence], [candidate])[0]!.id + const abandonedLedger = createRunCostLedger({ storage, runDir, costCeilingUsd: 1 }) + const controller = new AbortController() + let releaseProvider!: () => void + let reportStarted!: () => void + const started = new Promise((resolve) => { + reportStarted = resolve + }) + const abandoned = abandonedLedger.runPaidCall({ + callId: 'prior-cell-attempt', + channel: 'agent', + phase: 'memory.experiment', + actor: 'agent-knowledge:memory-experiment:memory', + model: 'memory', + signal: controller.signal, + tags: { + runDir, + scenarioId, + cellId: 'prior-cell', + rep: '0', + runAttemptId: 'prior-attempt', + }, + maximumCharge: { externallyEnforcedMaximumUsd: 0.1 }, + async execute() { + reportStarted() + return await new Promise((resolve) => { + releaseProvider = () => resolve('late result') + }) + }, + receipt: () => ({ + model: 'memory', + inputTokens: 0, + outputTokens: 0, + actualCostUsd: 0.05, + }), + }) + await started + controller.abort(new Error('simulated process exit')) + await abandoned + + const resumedLedger = createRunCostLedger({ storage, runDir, costCeilingUsd: 1 }) + const result = await runAgentMemoryExperiment({ + experimentId: 'interrupted-cell-cost-reporting', + sequences: [sequence], + candidates: [candidate], + runDir, + storage, + costLedger: resumedLedger, + }) + + expect(result.rows[0]).toMatchObject({ candidateId: 'memory', totalCostUsd: 0.2 }) + expect(resumedLedger.summary()).toMatchObject({ totalCalls: 2, totalCostUsd: 0.2 }) + releaseProvider() + await abandonedLedger.waitForIdle() + }) + + it('reconciles interrupted paid calls from every parallel memory branch before resuming', async () => { + const storage = inMemoryCampaignStorage() + const runDir = '/runs/parallel-interrupted-cost-recovery' + const abandonedLedger = createRunCostLedger({ storage, runDir, costCeilingUsd: 1 }) + const controllers = [new AbortController(), new AbortController()] + const releases: Array<() => void> = [] + let started = 0 + let reportStarted: (() => void) | undefined + const bothStarted = new Promise((resolve) => { + reportStarted = resolve + }) + const abandonedCalls = ['one', 'two'].map((branch, index) => + abandonedLedger.runPaidCall({ + callId: `abandoned-${branch}`, + channel: 'agent', + phase: 'memory.train', + actor: `agent-knowledge:memory-experiment:${branch}`, + model: `provider-${branch}`, + signal: controllers[index]!.signal, + maximumCharge: { externallyEnforcedMaximumUsd: 0.1 }, + async execute() { + started += 1 + if (started === 2) reportStarted?.() + return await new Promise((resolve) => { + releases[index] = () => resolve('late provider result') + }) + }, + receipt: () => ({ + model: `provider-${branch}`, + inputTokens: 0, + outputTokens: 0, + actualCostUsd: 0.05, + }), + }), + ) + + await bothStarted + for (const controller of controllers) { + controller.abort(new Error('simulated process exit')) + } + await Promise.all(abandonedCalls) + expect(abandonedLedger.listPending().map((call) => call.state)).toEqual(['late', 'late']) + + const resumedLedger = createRunCostLedger({ storage, runDir, costCeilingUsd: 1 }) + expect(resumedLedger.listPending().map((call) => call.state)).toEqual([ + 'interrupted', + 'interrupted', + ]) + + const result = await runAgentMemoryExperiment({ + experimentId: 'parallel-interrupted-cost-recovery', + sequences: [ + { + id: 'history', + family: 'first-party', + steps: [ + { + id: 'remember', + scope: { agentId: 'worker' }, + writes: [{ kind: 'fact', text: 'resumed fact' }], + probes: [ + { + id: 'recall', + query: 'resumed', + referenceAnswer: 'resumed fact', + }, + ], + }, + ], + }, + ], + candidates: [ + { + id: 'resumed', + ref: 'resumed:v1', + createAdapter: () => createScopedTestAdapter('resumed'), + }, + ], + runDir, + storage, + costLedger: resumedLedger, + }) + + expect(result.rows[0]).toMatchObject({ candidateId: 'resumed', cellsFailed: 0 }) + expect(resumedLedger.summary()).toMatchObject({ + totalCalls: 2, + unresolvedCalls: 0, + totalCostUsd: 0.2, + accountingComplete: true, + }) + expect(resumedLedger.list().map((receipt) => receipt.error)).toEqual([ + expect.stringContaining('charged the reserved maximum'), + expect.stringContaining('charged the reserved maximum'), + ]) + + for (const release of releases) release() + await abandonedLedger.waitForIdle() + }) + + it('does not reuse cells after the candidate implementation reference changes', async () => { + const storage = inMemoryCampaignStorage() + const sequence = { + id: 'candidate-ref', + family: 'first-party' as const, + steps: [ + { + id: 'remember', + scope: { agentId: 'worker' }, + writes: [{ kind: 'fact' as const, text: 'remember me' }], + probes: [ + { + id: 'recall', + query: 'remember', + requiredFacts: [{ id: 'fact', anyOf: ['remember me'] }], + }, + ], + }, + ], + } + let creates = 0 + const run = (ref: string, visible: boolean) => + runAgentMemoryExperiment({ + experimentId: 'candidate-ref-cache', + sequences: [sequence], + candidates: [ + { + id: 'memory', + ref, + createAdapter() { + creates += 1 + const adapter = createScopedTestAdapter(`memory:${ref}`) + if (visible) return adapter + return { + ...adapter, + async search() { + return [] + }, + } + }, + }, + ], + runDir: '/runs/candidate-ref-cache', + storage, + }) + + const first = await run('memory:v1', true) + const cached = await run('memory:v1', false) + const changed = await run('memory:v2', false) + + expect(first.rows[0]?.scoreMean).toBe(1) + expect(cached.campaign.cells[0]?.cached).toBe(true) + expect(cached.rows[0]?.scoreMean).toBe(1) + expect(changed.campaign.cells[0]?.cached).toBe(false) + expect(changed.rows[0]?.scoreMean).toBe(0) + expect(creates).toBe(2) + }) + + it('runs existing ordered memory benchmark cases without reshaping the dataset', async () => { + const sequences = buildAgentMemorySequencesFromBenchmarkCases([ + { + id: 'launch-update', + family: 'longmemeval', + taskKind: 'memory-temporal', + split: 'holdout', + events: [ + { id: 'old', actorId: 'pm', text: 'Launch is April 3.' }, + { id: 'current', actorId: 'pm', text: 'Launch moved to April 17.' }, + ], + prompt: 'When is launch?', + requiredFacts: [{ id: 'current-date', anyOf: ['April 17'] }], + forbiddenFacts: [{ id: 'old-date', anyOf: ['April 3'], obsolete: true }], + expectedEventIds: ['current'], + expectedActorIds: ['pm'], + }, + ]) + const result = await runAgentMemoryExperiment({ + experimentId: 'benchmark-conversion', + sequences, + candidates: [ + { + id: 'literal-memory', + ref: 'literal-memory:v1', + createAdapter: () => createScopedTestAdapter('literal-memory'), + }, + ], + runDir: '/runs/benchmark-conversion', + storage: inMemoryCampaignStorage(), + }) + + expect(sequences[0]?.steps.map((step) => step.id)).toEqual([ + 'event:old', + 'event:current', + 'probe', + ]) + expect(result.rows[0]).toMatchObject({ + scoreMean: 0.75, + totalSequences: 1, + totalProbes: 1, + cellsFailed: 0, + }) + expect(result.rows[0]?.dimensions).toMatchObject({ memory_stale_safe: 0 }) + }) + + it('keeps each history ordered while comparing candidate branches in parallel', async () => { + const storage = inMemoryCampaignStorage() + const snapshots: string[] = [] + const stepOrder = new Map() + let active = 0 + let maxActive = 0 + let release: (() => void) | undefined + const twoActive = new Promise((resolve) => { + release = resolve + }) + const sequence = (id: string) => ({ + id, + family: 'first-party' as const, + split: 'holdout' as const, + steps: [ + { + id: 'research', + scope: { agentId: 'researcher', teamId: 'team-1' }, + writes: [ + { + id: `${id}-event`, + kind: 'fact' as const, + text: `${id} launch date is Friday`, + metadata: { eventId: `${id}-event`, actorId: 'researcher' }, + }, + ], + }, + { + id: 'delivery', + scope: { agentId: 'builder', teamId: 'team-1' }, + probes: [ + { + id: 'launch-date', + query: `${id} launch date`, + requiredFacts: [{ id: 'current', anyOf: [`${id} launch date is Friday`] }], + expectedEventIds: [`${id}-event`], + expectedActorIds: ['researcher'], + }, + ], + }, + ], + }) + + const result = await runAgentMemoryExperiment({ + experimentId: 'team-sharing-vs-private', + sequences: [sequence('alpha'), sequence('beta')], + candidates: [ + { + id: 'private', + ref: 'private:v1', + policy: { read: ['private'], write: 'private' }, + createAdapter: ({ branchId }) => createScopedTestAdapter(`private:${branchId}`), + }, + { + id: 'team', + ref: 'team:v1', + policy: { read: ['team'], write: 'team' }, + createAdapter: ({ branchId }) => createScopedTestAdapter(`team:${branchId}`), + }, + ], + runDir: '/runs/team-sharing-vs-private', + storage, + maxConcurrency: 4, + cleanupBranches: true, + executeStepRef: 'test-runtime/v1', + executeStep: async ({ memory, step }) => { + const order = stepOrder.get(memory.branchId) ?? [] + order.push(step.id) + stepOrder.set(memory.branchId, order) + if (step.id === 'research') { + active += 1 + maxActive = Math.max(maxActive, active) + if (active === 2) release?.() + await twoActive + active -= 1 + } + }, + onBranchSnapshot: ({ snapshot }) => { + snapshots.push(snapshot.digest) + }, + }) + + expect(maxActive).toBeGreaterThanOrEqual(2) + expect(result.rows[0]).toMatchObject({ + rank: 1, + candidateId: 'team', + scoreMean: 1, + passRate: 1, + totalSequences: 2, + totalCells: 2, + totalProbes: 2, + cellsFailed: 0, + }) + expect(result.rows[1]?.candidateId).toBe('private') + expect(result.rows[1]?.scoreMean).toBeLessThan(0.3) + expect(result.campaign.cells).toHaveLength(4) + expect(snapshots).toHaveLength(4) + expect([...stepOrder.values()].every((steps) => steps.join(',') === 'research,delivery')).toBe( + true, + ) + expect(storage.read(result.rankingJsonPath)).toContain('"candidateId": "team"') + expect(storage.read(result.rankingMarkdownPath)).toContain('| 1 | team |') + }) + + it('uses distinct external branch ids for distinct run directories', async () => { + const branchIds: string[] = [] + const sequence = { + id: 'branch-id', + family: 'first-party' as const, + steps: [ + { + id: 'probe', + scope: { agentId: 'worker' }, + probes: [{ id: 'state', query: 'state', referenceAnswer: 'state' }], + }, + ], + } + const run = (runDir: string) => + runAgentMemoryExperiment({ + experimentId: 'same-experiment', + sequences: [sequence], + candidates: [ + { + id: 'memory', + ref: 'memory:v1', + createAdapter({ branchId }) { + branchIds.push(branchId) + return createScopedTestAdapter(branchId) + }, + }, + ], + runDir, + storage: inMemoryCampaignStorage(), + }) + + await run('/runs/branch-id-a') + await run('/runs/branch-id-b') + + expect(branchIds).toHaveLength(2) + expect(branchIds[0]).not.toBe(branchIds[1]) + }) + + it('uses a fresh external branch id for each distributed execution attempt', async () => { + const branchIds: string[] = [] + const run = (runDir: string) => + runAgentMemoryExperiment({ + experimentId: 'distributed-experiment', + experimentRunId: 'distributed-run-17', + sequences: [ + { + id: 'shared-history', + family: 'first-party', + steps: [ + { + id: 'probe', + scope: { agentId: 'worker' }, + probes: [{ id: 'state', query: 'state', referenceAnswer: 'state' }], + }, + ], + }, + ], + candidates: [ + { + id: 'memory', + ref: 'memory:v1', + createAdapter({ branchId }) { + branchIds.push(branchId) + return createScopedTestAdapter(branchId) + }, + }, + ], + runDir, + storage: inMemoryCampaignStorage(), + }) + + await run('/worker-a/run') + await run('/worker-b/run') + + expect(branchIds).toHaveLength(2) + expect(branchIds[0]).not.toBe(branchIds[1]) + }) + + it('waits for timed-out provider work to finish cleanup before returning', async () => { + let releaseWrite!: () => void + const writeMayFinish = new Promise((resolve) => { + releaseWrite = resolve + }) + let reportWriteStarted!: () => void + const writeStarted = new Promise((resolve) => { + reportWriteStarted = resolve + }) + const rows: AgentMemoryHit[] = [] + let clears = 0 + const adapter: AgentMemoryAdapter = { + id: 'slow-provider', + branchIsolation: { mode: 'scoped' }, + async search() { + return [...rows] + }, + async getContext(query) { + return { query, text: rows.map(hitText).join('\n'), hits: [...rows], sourceRecords: [] } + }, + async write(input) { + reportWriteStarted() + await writeMayFinish + const hit = { + id: 'late-write', + uri: 'memory://slow-provider/late-write', + kind: input.kind, + text: input.text, + } + rows.push(hit) + return { accepted: true, id: hit.id, uri: hit.uri, kind: hit.kind } + }, + async clear() { + clears += 1 + rows.length = 0 + }, + } + const run = runAgentMemoryExperiment({ + experimentId: 'timeout-cleanup', + sequences: [ + { + id: 'slow-history', + family: 'first-party', + steps: [ + { + id: 'write', + scope: { agentId: 'worker' }, + writes: [{ kind: 'fact', text: 'late fact' }], + probes: [{ id: 'fact', query: 'fact', referenceAnswer: 'late fact' }], + }, + ], + }, + ], + candidates: [{ id: 'slow', ref: 'slow:v1', createAdapter: () => adapter }], + runDir: '/runs/timeout-cleanup', + storage: inMemoryCampaignStorage(), + dispatchTimeoutMs: 5, + }) + await writeStarted + let settled = false + void run.then( + () => { + settled = true + }, + () => { + settled = true + }, + ) + await new Promise((resolve) => setTimeout(resolve, 15)) + + expect(settled).toBe(false) + releaseWrite() + const result = await run + + expect(result.rows[0]).toMatchObject({ cellsFailed: 1 }) + expect(result.campaign.cells[0]?.error).toContain('dispatch exceeded 5ms') + expect(clears).toBe(1) + expect(rows).toEqual([]) + }) + + it('fails when cleanup after a timed-out provider write fails', async () => { + let releaseWrite!: () => void + const writeMayFinish = new Promise((resolve) => { + releaseWrite = resolve + }) + let reportWriteStarted!: () => void + const writeStarted = new Promise((resolve) => { + reportWriteStarted = resolve + }) + let clears = 0 + const adapter: AgentMemoryAdapter = { + id: 'cleanup-failure', + branchIsolation: { mode: 'scoped' }, + async search() { + return [] + }, + async getContext(query) { + return { query, text: '', hits: [], sourceRecords: [] } + }, + async write(input) { + reportWriteStarted() + await writeMayFinish + return { + accepted: true, + id: 'late-write', + uri: 'memory://cleanup-failure/late-write', + kind: input.kind, + } + }, + async clear() { + clears += 1 + throw new Error('provider cleanup unavailable') + }, + } + const run = runAgentMemoryExperiment({ + experimentId: 'timeout-cleanup-failure', + sequences: [ + { + id: 'slow-history', + family: 'first-party', + steps: [ + { + id: 'write', + scope: { agentId: 'worker' }, + writes: [{ kind: 'fact', text: 'late fact' }], + probes: [{ id: 'fact', query: 'fact', referenceAnswer: 'late fact' }], + }, + ], + }, + ], + candidates: [{ id: 'slow', ref: 'slow:v1', createAdapter: () => adapter }], + runDir: '/runs/timeout-cleanup-failure', + storage: inMemoryCampaignStorage(), + dispatchTimeoutMs: 5, + }) + await writeStarted + await new Promise((resolve) => setTimeout(resolve, 10)) + releaseWrite() + + await expect(run).rejects.toThrow('memory experiment cleanup failed after dispatch') + }) + + it('recovers an unfinished provider branch before retrying the history', async () => { + const storage = inMemoryCampaignStorage() + const rows = new Map() + const operations: string[] = [] + const branchIds: Record<'first' | 'recovery' | 'retry', string | undefined> = { + first: undefined, + recovery: undefined, + retry: undefined, + } + let firstExecution = true + const sequence = { + id: 'recover-history', + family: 'first-party' as const, + steps: [ + { + id: 'remember', + scope: { agentId: 'worker' }, + writes: [{ kind: 'fact' as const, text: 'sensitive provider fact' }], + probes: [ + { + id: 'recall', + query: 'provider fact', + referenceAnswer: 'sensitive provider fact', + }, + ], + }, + ], + } + const candidate = { + id: 'recoverable', + ref: 'recoverable:v1', + externalRecoveryCostUsdPerAttempt: 0.1, + createAdapter({ branchId, purpose }: { branchId: string; purpose: 'execute' | 'recovery' }) { + const executionLabel = + purpose === 'recovery' ? 'recovery' : firstExecution ? 'first' : 'retry' + branchIds[executionLabel] = branchId + operations.push(`create:${executionLabel}`) + let clearCalls = 0 + const adapter: AgentMemoryAdapter = { + id: 'recoverable-provider', + branchIsolation: { mode: 'scoped' }, + async search(_query, options) { + return [...(rows.get(options.scope?.namespace ?? '') ?? [])] + }, + async getContext(query, options) { + const hits = await adapter.search(query, options) + return { query, text: hits.map(hitText).join('\n'), hits, sourceRecords: [] } + }, + async write(input) { + operations.push(`write:${executionLabel}`) + const namespace = input.scope?.namespace ?? '' + const hit = { + id: input.id ?? `${executionLabel}-fact`, + uri: `memory://recoverable/${executionLabel}`, + kind: input.kind, + text: input.text, + } + rows.set(namespace, [...(rows.get(namespace) ?? []), hit]) + return { accepted: true, id: hit.id, uri: hit.uri, kind: hit.kind } + }, + async clear(scope) { + clearCalls += 1 + operations.push(`clear:${executionLabel}`) + if (executionLabel === 'first' && clearCalls === 1) { + throw new Error('provider cleanup unavailable') + } + rows.delete(scope?.namespace ?? '') + }, + } + if (purpose === 'execute') firstExecution = false + return adapter + }, + } + const run = () => + runAgentMemoryExperiment({ + experimentId: 'restart-recovery', + sequences: [sequence], + candidates: [candidate], + runDir: '/runs/restart-recovery', + storage, + costCeiling: 1, + }) + + await expect(run()).rejects.toThrow('memory experiment cleanup failed after dispatch') + expect(rows.size).toBe(1) + + const result = await run() + + expect(result.rows[0]).toMatchObject({ candidateId: 'recoverable', cellsFailed: 0 }) + expect(result.rows[0]?.totalCostUsd).toBe(0.1) + expect(result.campaign.aggregates.totalCostUsd).toBe(0.1) + expect(branchIds.recovery).toBe(branchIds.first) + expect(branchIds.retry).not.toBe(branchIds.first) + expect(operations.indexOf('clear:recovery')).toBeLessThan(operations.indexOf('write:retry')) + expect(rows.size).toBe(0) + const attemptEvents = storage + .read(result.attemptLogPath)! + .trim() + .split('\n') + .map((line) => JSON.parse(line) as { status: string; recovery: boolean }) + expect(attemptEvents.map(({ status, recovery }) => ({ status, recovery }))).toEqual([ + { status: 'started', recovery: false }, + { status: 'cleaned', recovery: true }, + { status: 'started', recovery: false }, + { status: 'cleaned', recovery: false }, + ]) + }) + + it('closes and disposes a recovery adapter that arrives after its factory timeout', async () => { + const storage = inMemoryCampaignStorage() + const runDir = '/runs/late-recovery-adapter' + storage.write( + `${runDir}/memory-attempts.jsonl`, + `${JSON.stringify({ + schema: 2, + status: 'started', + branchId: 'unfinished-branch', + candidateId: 'memory', + candidateRef: 'memory:v1', + sequenceId: 'history', + rep: 0, + seed: 42, + cleanupBranches: true, + externalCostUsdPerSequence: 0, + externalRecoveryCostUsdPerAttempt: 0, + recordedAt: '2026-01-01T00:00:00.000Z', + recovery: false, + })}\n`, + ) + const sequence = { + id: 'history', + family: 'first-party' as const, + steps: [ + { + id: 'remember', + scope: { agentId: 'worker' }, + writes: [{ kind: 'fact' as const, text: 'fact' }], + probes: [{ id: 'recall', query: 'fact', referenceAnswer: 'fact' }], + }, + ], + } + let resolveCreation!: (adapter: AgentMemoryAdapter) => void + const creation = new Promise((resolve) => { + resolveCreation = resolve + }) + let reportDisposed!: () => void + const disposed = new Promise((resolve) => { + reportDisposed = resolve + }) + let closeCalls = 0 + let disposeCalls = 0 + const lateAdapter = createScopedTestAdapter('memory-provider') + lateAdapter.close = async () => { + closeCalls += 1 + } + + await expect( + runAgentMemoryExperiment({ + experimentId: 'late-recovery-adapter', + sequences: [sequence], + candidates: [ + { + id: 'memory', + ref: 'memory:v1', + createAdapter: ({ purpose }) => + purpose === 'recovery' ? creation : createScopedTestAdapter('memory-provider'), + async disposeAdapter(adapter) { + expect(adapter).toBe(lateAdapter) + disposeCalls += 1 + reportDisposed() + }, + }, + ], + runDir, + storage, + cleanupTimeoutMs: 10, + }), + ).rejects.toThrow("memory: abandoned memory branch 'unfinished-branch' recovery failed") + + resolveCreation(lateAdapter) + await disposed + expect({ closeCalls, disposeCalls }).toEqual({ closeCalls: 1, disposeCalls: 1 }) + }) + + it('reconciles a crash after provider recovery but before its cost receipt', async () => { + const storage = inMemoryCampaignStorage() + const append = storage.append!.bind(storage) + let failRecoveryReceipt = false + storage.append = (path, value, expectedBytes) => { + if ( + failRecoveryReceipt && + path.endsWith('/cost-ledger.jsonl') && + value.includes('"status":"settled"') && + value.includes('agent-knowledge:memory-recovery') + ) { + failRecoveryReceipt = false + throw new Error('simulated process exit before recovery receipt') + } + return append(path, value, expectedBytes) + } + + let firstExecution = true + let recoveryClears = 0 + const candidate = { + id: 'crash-recoverable', + ref: 'crash-recoverable:v1', + externalCostUsdPerSequence: 0.1, + externalRecoveryCostUsdPerAttempt: 0.1, + createAdapter({ purpose }: { purpose: 'execute' | 'recovery' }) { + const adapter = createScopedTestAdapter(`crash-recoverable:${purpose}`) + const clear = adapter.clear! + let clearCalls = 0 + const failThisExecution = purpose === 'execute' && firstExecution + if (purpose === 'execute') firstExecution = false + adapter.clear = async (scope) => { + clearCalls += 1 + if (purpose === 'recovery') recoveryClears += 1 + if (failThisExecution && clearCalls === 1) { + throw new Error('leave the first branch active') + } + await clear(scope) + } + return adapter + }, + } + const run = () => + runAgentMemoryExperiment({ + experimentId: 'recovery-receipt-crash', + sequences: [ + { + id: 'history', + family: 'first-party', + steps: [ + { + id: 'remember', + scope: { agentId: 'worker' }, + writes: [{ kind: 'fact', text: 'durable recovery fact' }], + probes: [ + { + id: 'recall', + query: 'durable', + referenceAnswer: 'durable recovery fact', + }, + ], + }, + ], + }, + ], + candidates: [candidate], + runDir: '/runs/recovery-receipt-crash', + storage, + costCeiling: 1, + }) + + await expect(run()).rejects.toThrow('memory experiment cleanup failed after dispatch') + failRecoveryReceipt = true + await expect(run()).rejects.toThrow('failed to persist') + expect( + storage.read('/runs/recovery-receipt-crash/memory-attempts.jsonl')?.trim().split('\n'), + ).toHaveLength(2) + + const result = await run() + const costLedger = createRunCostLedger({ + storage, + runDir: '/runs/recovery-receipt-crash', + costCeilingUsd: 1, + }) + + expect(result.rows[0]).toMatchObject({ + candidateId: 'crash-recoverable', + cellsFailed: 0, + }) + expect(result.rows[0]?.totalCostUsd).toBeCloseTo(0.3) + expect(recoveryClears).toBe(1) + expect(costLedger.summary()).toMatchObject({ unresolvedCalls: 0, accountingComplete: true }) + expect(costLedger.summary().totalCostUsd).toBeCloseTo(0.3) + }) + + it('recovers independent abandoned branches in parallel before retrying them', async () => { + const storage = inMemoryCampaignStorage() + let phase: 'leave-active' | 'recover' = 'leave-active' + let activeRecoveries = 0 + let maxActiveRecoveries = 0 + let releaseRecoveries: (() => void) | undefined + const recoveriesStarted = new Promise((resolve) => { + releaseRecoveries = resolve + }) + const sequences = ['one', 'two'].map((id) => ({ + id, + family: 'first-party' as const, + steps: [ + { + id: 'remember', + scope: { agentId: 'worker' }, + writes: [{ kind: 'fact' as const, text: `${id} fact` }], + probes: [{ id: 'recall', query: id, referenceAnswer: `${id} fact` }], + }, + ], + })) + const run = () => + runAgentMemoryExperiment({ + experimentId: 'parallel-recovery', + sequences, + candidates: [ + { + id: 'memory', + ref: 'memory:v1', + async createAdapter({ purpose }) { + if (purpose === 'recovery') { + activeRecoveries += 1 + maxActiveRecoveries = Math.max(maxActiveRecoveries, activeRecoveries) + if (activeRecoveries === 2) releaseRecoveries?.() + await recoveriesStarted + activeRecoveries -= 1 + return null + } + if (phase === 'leave-active') return null + return createScopedTestAdapter('parallel-recovery') + }, + }, + ], + runDir: '/runs/parallel-recovery', + storage, + maxConcurrency: 2, + }) + + await expect(run()).rejects.toThrow('memory experiment cleanup failed after dispatch') + phase = 'recover' + const result = await run() + + expect(maxActiveRecoveries).toBe(2) + expect(result.rows[0]).toMatchObject({ candidateId: 'memory', cellsFailed: 0 }) + const recovered = storage + .read(result.attemptLogPath)! + .trim() + .split('\n') + .map((line) => JSON.parse(line) as { status: string; recovery: boolean }) + .filter((event) => event.status === 'cleaned' && event.recovery) + expect(recovered).toHaveLength(2) + }) + + it('closes an attempt whose provider was never created before retrying it', async () => { + const storage = inMemoryCampaignStorage() + const purposes: Array<'execute' | 'recovery'> = [] + let executionCalls = 0 + const sequence = { + id: 'provider-creation', + family: 'first-party' as const, + steps: [ + { + id: 'remember', + scope: { agentId: 'worker' }, + writes: [{ kind: 'fact' as const, text: 'created provider fact' }], + probes: [ + { + id: 'recall', + query: 'provider fact', + referenceAnswer: 'created provider fact', + }, + ], + }, + ], + } + const run = () => + runAgentMemoryExperiment({ + experimentId: 'provider-creation-recovery', + sequences: [sequence], + candidates: [ + { + id: 'sometimes-created', + ref: 'sometimes-created:v1', + externalRecoveryCostUsdPerAttempt: 0.1, + createAdapter({ purpose }) { + purposes.push(purpose) + if (purpose === 'recovery') return null + executionCalls += 1 + if (executionCalls === 1) return null + return createScopedTestAdapter('created-provider') + }, + }, + ], + runDir: '/runs/provider-creation-recovery', + storage, + costCeiling: 1, + }) + + await expect(run()).rejects.toThrow('memory experiment cleanup failed after dispatch') + const result = await run() + + expect(result.rows[0]).toMatchObject({ candidateId: 'sometimes-created', cellsFailed: 0 }) + expect(result.rows[0]?.totalCostUsd).toBe(0) + expect(purposes).toEqual(['execute', 'recovery', 'execute']) + const attemptEvents = storage + .read(result.attemptLogPath)! + .trim() + .split('\n') + .map((line) => JSON.parse(line) as { status: string; recovery: boolean }) + expect(attemptEvents.map(({ status, recovery }) => ({ status, recovery }))).toEqual([ + { status: 'started', recovery: false }, + { status: 'cleaned', recovery: true }, + { status: 'started', recovery: false }, + { status: 'cleaned', recovery: false }, + ]) + }) + + it('uses a retired candidate only to clean its unfinished branch', async () => { + const storage = inMemoryCampaignStorage() + const runDir = '/runs/retired-recovery-candidate' + const sequence = { + id: 'history', + family: 'first-party' as const, + steps: [ + { + id: 'remember', + scope: { agentId: 'worker' }, + writes: [{ kind: 'fact' as const, text: 'active fact' }], + probes: [{ id: 'recall', query: 'active', referenceAnswer: 'active fact' }], + }, + ], + } + storage.write( + `${runDir}/memory-attempts.jsonl`, + `${JSON.stringify({ + schema: 2, + status: 'started', + branchId: 'retired-branch', + candidateId: 'retired', + candidateRef: 'retired:v1', + sequenceId: sequence.id, + rep: 0, + seed: 1, + cleanupBranches: true, + externalCostUsdPerSequence: 0, + externalRecoveryCostUsdPerAttempt: 0.1, + recordedAt: '2026-01-01T00:00:00.000Z', + recovery: false, + })}\n`, + ) + const purposes: string[] = [] + let retiredClears = 0 + const result = await runAgentMemoryExperiment({ + experimentId: 'retired-recovery-candidate', + sequences: [sequence], + candidates: [ + { + id: 'active', + ref: 'active:v1', + createAdapter({ purpose }) { + purposes.push(`active:${purpose}`) + return createScopedTestAdapter('active') + }, + }, + ], + recoveryCandidates: [ + { + id: 'retired', + ref: 'retired:v1', + externalRecoveryCostUsdPerAttempt: 0.1, + createAdapter({ purpose }) { + purposes.push(`retired:${purpose}`) + const adapter = createScopedTestAdapter('retired') + adapter.clear = async () => { + retiredClears += 1 + } + return adapter + }, + }, + ], + runDir, + storage, + resumable: false, + costCeiling: 1, + }) + + expect(result.rows).toHaveLength(1) + expect(result.rows[0]?.candidateId).toBe('active') + expect(result.rows[0]?.totalCostUsd).toBe(0) + expect(result).toMatchObject({ totalCostUsd: 0.1, unrankedRecoveryCostUsd: 0.1 }) + expect(purposes).toEqual(['retired:recovery', 'active:execute']) + expect(retiredClears).toBeGreaterThan(0) + }) + + it('refuses to hide an unfinished branch when candidate cost settings change', async () => { + const storage = inMemoryCampaignStorage() + const runDir = '/runs/changed-recovery-costs' + const sequence = { + id: 'history', + family: 'first-party' as const, + steps: [{ id: 'probe', probes: [{ id: 'recall', query: 'fact', referenceAnswer: 'fact' }] }], + } + storage.write( + `${runDir}/memory-attempts.jsonl`, + `${JSON.stringify({ + schema: 2, + status: 'started', + branchId: 'unfinished-branch', + candidateId: 'memory', + candidateRef: 'memory:v1', + sequenceId: sequence.id, + rep: 0, + seed: 1, + cleanupBranches: true, + externalCostUsdPerSequence: 0, + externalRecoveryCostUsdPerAttempt: 0, + recordedAt: '2026-01-01T00:00:00.000Z', + recovery: false, + })}\n`, + ) + let adapterCreates = 0 + + await expect( + runAgentMemoryExperiment({ + experimentId: 'changed-recovery-costs', + sequences: [sequence], + candidates: [ + { + id: 'memory', + ref: 'memory:v1', + externalCostUsdPerSequence: 0.1, + createAdapter() { + adapterCreates += 1 + return createScopedTestAdapter('memory') + }, + }, + ], + runDir, + storage, + costCeiling: 1, + }), + ).rejects.toThrow('candidate cost settings changed') + + expect(adapterCreates).toBe(0) + expect(storage.read(`${runDir}/memory-attempts.jsonl`)?.trim().split('\n')).toHaveLength(1) + }) + + it('refuses a recovery backlog larger than maxRecoveryAttempts', async () => { + const storage = inMemoryCampaignStorage() + const runDir = '/runs/recovery-backlog-limit' + const sequence = { + id: 'history', + family: 'first-party' as const, + steps: [ + { + id: 'probe', + scope: { agentId: 'worker' }, + probes: [{ id: 'recall', query: 'fact', referenceAnswer: 'fact' }], + }, + ], + } + storage.write( + `${runDir}/memory-attempts.jsonl`, + `${[ + { + schema: 2, + status: 'started', + branchId: 'branch-1', + candidateId: 'memory', + candidateRef: 'memory:v1', + sequenceId: sequence.id, + rep: 0, + seed: 1, + cleanupBranches: true, + externalCostUsdPerSequence: 0, + externalRecoveryCostUsdPerAttempt: 0, + recordedAt: '2026-01-01T00:00:00.000Z', + recovery: false, + }, + { + schema: 2, + status: 'started', + branchId: 'branch-2', + candidateId: 'memory', + candidateRef: 'memory:v1', + sequenceId: sequence.id, + rep: 0, + seed: 2, + cleanupBranches: true, + externalCostUsdPerSequence: 0, + externalRecoveryCostUsdPerAttempt: 0, + recordedAt: '2026-01-01T00:00:00.000Z', + recovery: false, + }, + ] + .map((event) => JSON.stringify(event)) + .join('\n')}\n`, + ) + let providerCreates = 0 + + await expect( + runAgentMemoryExperiment({ + experimentId: 'recovery-backlog-limit', + sequences: [sequence], + candidates: [ + { + id: 'memory', + ref: 'memory:v1', + createAdapter() { + providerCreates += 1 + return createScopedTestAdapter('memory') + }, + }, + ], + runDir, + storage, + maxRecoveryAttempts: 1, + }), + ).rejects.toThrow('2 unfinished attempts; maxRecoveryAttempts is 1') + expect(providerCreates).toBe(0) + }) + + it('bounds repeated provider recovery across process restarts', async () => { + const storage = inMemoryCampaignStorage() + const runDir = '/runs/recovery-retry-limit' + const sequence = { + id: 'history', + family: 'first-party' as const, + steps: [{ id: 'probe', probes: [{ id: 'recall', query: 'fact', referenceAnswer: 'fact' }] }], + } + storage.write( + `${runDir}/memory-attempts.jsonl`, + `${JSON.stringify({ + schema: 2, + status: 'started', + branchId: 'unfinished-branch', + candidateId: 'memory', + candidateRef: 'memory:v1', + sequenceId: sequence.id, + rep: 0, + seed: 1, + cleanupBranches: true, + externalCostUsdPerSequence: 0, + externalRecoveryCostUsdPerAttempt: 0, + recordedAt: '2026-01-01T00:00:00.000Z', + recovery: false, + })}\n`, + ) + let recoveryCreates = 0 + const run = () => + runAgentMemoryExperiment({ + experimentId: 'recovery-retry-limit', + sequences: [sequence], + candidates: [ + { + id: 'memory', + ref: 'memory:v1', + createAdapter({ purpose }) { + if (purpose === 'recovery') recoveryCreates += 1 + throw new Error('provider recovery unavailable') + }, + }, + ], + runDir, + storage, + maxRecoveryRetriesPerAttempt: 2, + }) + + await expect(run()).rejects.toThrow( + "abandoned memory branch 'unfinished-branch' recovery failed", + ) + await expect(run()).rejects.toThrow( + "abandoned memory branch 'unfinished-branch' recovery failed", + ) + await expect(run()).rejects.toThrow('exhausted 2 recovery attempts') + expect(recoveryCreates).toBe(2) + expect( + storage.read(`${runDir}/memory-recovery-attempts.jsonl`)?.trim().split('\n'), + ).toHaveLength(2) + }) + + it('allows one controller per run while its history workers run in parallel', async () => { + const storage = inMemoryCampaignStorage() + let reportStarted: (() => void) | undefined + const started = new Promise((resolve) => { + reportStarted = resolve + }) + let releaseWorker: (() => void) | undefined + const continueWorker = new Promise((resolve) => { + releaseWorker = resolve + }) + const options: RunAgentMemoryExperimentOptions = { + experimentId: 'single-controller', + sequences: [ + { + id: 'one', + family: 'first-party', + steps: [ + { + id: 'work', + scope: { agentId: 'worker' }, + writes: [{ kind: 'fact', text: 'parallel worker fact' }], + probes: [ + { + id: 'recall', + query: 'worker fact', + referenceAnswer: 'parallel worker fact', + }, + ], + }, + ], + }, + ], + candidates: [ + { + id: 'memory', + ref: 'memory:v1', + createAdapter: () => createScopedTestAdapter('single-controller'), + }, + ], + runDir: '/runs/single-controller', + storage, + executeStepRef: 'blocking-step:v1', + async executeStep() { + reportStarted?.() + await continueWorker + }, + } + + const firstRun = runAgentMemoryExperiment(options) + await started + await expect( + runAgentMemoryExperiment({ ...options, storage: inMemoryCampaignStorage() }), + ).rejects.toThrow('active controller') + releaseWorker?.() + const result = await firstRun + + expect(result.rows[0]).toMatchObject({ candidateId: 'memory', cellsFailed: 0 }) + }) + + it('leaves a lost controller branch for the next owner and accepts duplicate cleanup receipts', async () => { + const storage = inMemoryCampaignStorage() + let firstControllerOwned = true + let expireFirstController = true + let controllerCount = 0 + let searches = 0 + let clears = 0 + let closes = 0 + let factVisible = false + const purposes: string[] = [] + const options: RunAgentMemoryExperimentOptions = { + experimentId: 'lost-controller', + sequences: [ + { + id: 'one', + family: 'first-party', + steps: [ + { + id: 'work', + scope: { agentId: 'worker' }, + writes: [{ kind: 'fact', text: 'must be cleaned' }], + probes: [{ id: 'probe', query: 'must', referenceAnswer: 'must be cleaned' }], + }, + ], + }, + ], + candidates: [ + { + id: 'memory', + ref: 'memory:v1', + createAdapter({ purpose }) { + purposes.push(purpose) + const adapter: AgentMemoryAdapter = { + id: 'lost-controller', + branchIsolation: { mode: 'scoped' }, + async search() { + searches += 1 + return factVisible + ? [ + { + id: 'write', + uri: 'memory://lost-controller/write', + kind: 'fact', + text: 'must be cleaned', + }, + ] + : [] + }, + async getContext(query, searchOptions) { + const hits = await adapter.search(query, searchOptions) + return { query, text: hits.map(hitText).join('\n'), hits, sourceRecords: [] } + }, + async write(input) { + factVisible = true + return { + accepted: true, + id: 'write', + uri: 'memory://lost-controller/write', + kind: input.kind, + } + }, + async clear() { + clears += 1 + factVisible = false + }, + async close() { + closes += 1 + }, + } + return adapter + }, + }, + ], + runDir: '/runs/lost-controller', + storage, + acquireRunLease: async () => { + controllerCount += 1 + const controller = controllerCount + return { + assertOwned() { + if (controller === 1 && !firstControllerOwned) { + throw new Error('controller ownership expired') + } + }, + release() {}, + } + }, + executeStepRef: 'expire-controller:v1', + async executeStep() { + if (expireFirstController) { + expireFirstController = false + firstControllerOwned = false + } + }, + } + const run = () => runAgentMemoryExperiment(options) + + await expect(run()).rejects.toThrow('controller ownership expired') + expect(searches).toBe(0) + expect(clears).toBe(0) + expect(closes).toBe(1) + expect(factVisible).toBe(true) + + const result = await run() + expect(result.rows[0]).toMatchObject({ candidateId: 'memory', cellsFailed: 0 }) + expect(purposes).toEqual(['execute', 'recovery', 'execute']) + expect(clears).toBe(2) + expect(closes).toBe(3) + expect(factVisible).toBe(false) + + const journal = storage.read(result.attemptLogPath)! + const lines = journal.trim().split('\n') + const last = lines.at(-1)! + expect(storage.append!(result.attemptLogPath, `${last}\n`, Buffer.byteLength(journal))).toBe( + Buffer.byteLength(journal) + Buffer.byteLength(`${last}\n`), + ) + const callsBeforeCachedRun = { + purposes: purposes.length, + searches, + clears, + closes, + } + const cached = await run() + expect(cached.campaign.aggregates.cellsCached).toBe(1) + expect({ purposes: purposes.length, searches, clears, closes }).toEqual(callsBeforeCachedRun) + }) + + it('preserves both the run failure and controller release failure', async () => { + const error = await runAgentMemoryExperiment({ + experimentId: 'run-and-release-failure', + sequences: [ + { + id: 'one', + family: 'first-party', + steps: [ + { + id: 'probe', + probes: [{ id: 'probe', query: 'state', referenceAnswer: 'state' }], + }, + ], + }, + ], + candidates: [ + { + id: 'unused', + ref: 'unused:v1', + createAdapter: () => createScopedTestAdapter('unused'), + }, + ], + runDir: '/runs/run-and-release-failure', + storage: inMemoryCampaignStorage(), + acquireRunLease: async () => ({ + assertOwned() { + throw new Error('run ownership failed') + }, + release() { + throw new Error('release failed') + }, + }), + }).catch((caught: unknown) => caught) + + expect(error).toBeInstanceOf(AggregateError) + expect((error as AggregateError).errors.map((item) => String(item))).toEqual([ + 'Error: run ownership failed', + 'Error: release failed', + ]) + }) + + it('uses the same random seed for every candidate on one history and repetition', async () => { + const seeds = new Map() + const sequence = { + id: 'paired-history', + family: 'first-party' as const, + steps: [ + { + id: 'probe', + scope: { agentId: 'worker' }, + probes: [{ id: 'state', query: 'state', referenceAnswer: 'state' }], + }, + ], + } + await runAgentMemoryExperiment({ + experimentId: 'paired-seeds', + sequences: [sequence], + candidates: ['a', 'b'].map((candidateId) => ({ + id: candidateId, + ref: `${candidateId}:v1`, + createAdapter: ({ sequence: candidateSequence, rep, seed }) => { + seeds.set(`${candidateId}:${candidateSequence.id}:${rep}`, seed) + return createScopedTestAdapter(`${candidateId}:${rep}`) + }, + })), + reps: 2, + runDir: '/runs/paired-seeds', + storage: inMemoryCampaignStorage(), + maxConcurrency: 4, + }) + + expect(seeds.get('a:paired-history:0')).toBe(seeds.get('b:paired-history:0')) + expect(seeds.get('a:paired-history:1')).toBe(seeds.get('b:paired-history:1')) + expect(seeds.get('a:paired-history:0')).not.toBe(seeds.get('a:paired-history:1')) + }) + + it('fails the experiment when accepted writes cannot be cleared after a failed step', async () => { + let clears = 0 + let closes = 0 + let disposals = 0 + const run = runAgentMemoryExperiment({ + experimentId: 'failed-step-cleanup', + sequences: [ + { + id: 'failure', + family: 'first-party', + steps: [ + { + id: 'write-then-fail', + scope: { agentId: 'worker' }, + writes: [{ kind: 'fact', text: 'partial state' }], + probes: [{ id: 'state', query: 'partial state', referenceAnswer: 'partial state' }], + }, + ], + }, + ], + candidates: [ + { + id: 'scoped', + ref: 'scoped:v1', + createAdapter() { + const base = createScopedTestAdapter('failure') + return { + ...base, + async clear(scope) { + clears += 1 + await base.clear?.(scope) + throw new Error('provider clear failed') + }, + async close() { + closes += 1 + }, + } + }, + async disposeAdapter() { + disposals += 1 + }, + }, + ], + runDir: '/runs/failed-step-cleanup', + storage: inMemoryCampaignStorage(), + cleanupBranches: true, + executeStepRef: 'failing-worker/v1', + async executeStep() { + throw new Error('worker failed') + }, + }) + + await expect(run).rejects.toThrow('memory experiment cleanup failed after dispatch') + expect(clears).toBe(1) + expect(closes).toBe(1) + expect(disposals).toBe(1) + }) + + it('cleans an unjournaled provider side effect and ranks failed histories below complete ones', async () => { + let dirtyRows = 0 + let clears = 0 + const result = await runAgentMemoryExperiment({ + experimentId: 'provider-side-effect-cleanup', + sequences: [ + { + id: 'one', + family: 'first-party', + steps: [ + { + id: 'remember', + scope: { agentId: 'worker' }, + writes: [{ kind: 'fact', text: 'Launch is Friday.' }], + probes: [{ id: 'launch', query: 'launch', referenceAnswer: 'Launch is Friday.' }], + }, + ], + }, + ], + candidates: [ + { + id: 'side-effect-then-error', + ref: 'side-effect-then-error:v1', + createAdapter: () => ({ + id: 'side-effect-then-error', + branchIsolation: { mode: 'scoped' }, + async search() { + return [] + }, + async getContext(query) { + return { query, text: '', hits: [], sourceRecords: [] } + }, + async write() { + dirtyRows += 1 + throw new Error('provider disconnected after commit') + }, + async clear() { + clears += 1 + dirtyRows = 0 + }, + }), + }, + { + id: 'complete', + ref: 'complete:v1', + createAdapter: () => createScopedTestAdapter('complete'), + }, + ], + runDir: '/runs/provider-side-effect-cleanup', + storage: inMemoryCampaignStorage(), + cleanupBranches: true, + maxConcurrency: 2, + }) + + expect(dirtyRows).toBe(0) + expect(clears).toBe(1) + expect(result.rows[0]).toMatchObject({ + candidateId: 'complete', + cellsFailed: 0, + scoreMean: 1, + }) + expect(result.rows[1]).toMatchObject({ + candidateId: 'side-effect-then-error', + cellsFailed: 1, + scoreMean: 0, + passRate: 0, + }) + }) + + it('closes and disposes an adapter when branch validation fails', async () => { + let closes = 0 + let disposals = 0 + const result = await runAgentMemoryExperiment({ + experimentId: 'invalid-branch-cleanup', + sequences: [ + { + id: 'one', + family: 'first-party', + steps: [ + { + id: 'probe', + scope: { agentId: 'worker' }, + probes: [{ id: 'state', query: 'state', referenceAnswer: 'state' }], + }, + ], + }, + ], + candidates: [ + { + id: 'legacy', + ref: 'legacy:v1', + createAdapter() { + const { branchIsolation: _branchIsolation, ...legacy } = + createScopedTestAdapter('legacy') + return { + ...legacy, + async close() { + closes += 1 + }, + } + }, + async disposeAdapter() { + disposals += 1 + }, + }, + ], + runDir: '/runs/invalid-branch-cleanup', + storage: inMemoryCampaignStorage(), + cleanupBranches: false, + }) + + expect(result.rows[0]).toMatchObject({ cellsFailed: 1 }) + expect(result.campaign.cells[0]?.error).toContain('adapter must declare branchIsolation') + expect(closes).toBe(1) + expect(disposals).toBe(1) + }) + + it('fails closed when cleanup is requested for an adapter without scoped clear', async () => { + let disposals = 0 + const result = await runAgentMemoryExperiment({ + experimentId: 'unsupported-cleanup', + sequences: [ + { + id: 'one', + family: 'first-party', + steps: [ + { + id: 'probe', + scope: { agentId: 'worker' }, + probes: [{ id: 'state', query: 'state', referenceAnswer: 'state' }], + }, + ], + }, + ], + candidates: [ + { + id: 'no-clear', + ref: 'no-clear:v1', + createAdapter() { + const { clear: _clear, ...adapter } = createScopedTestAdapter('no-clear') + return adapter + }, + async disposeAdapter() { + disposals += 1 + }, + }, + ], + runDir: '/runs/unsupported-cleanup', + storage: inMemoryCampaignStorage(), + cleanupBranches: true, + }) + + expect(result.rows[0]).toMatchObject({ cellsFailed: 1 }) + expect(result.campaign.cells[0]?.error).toContain('requires an adapter with scoped clear') + expect(disposals).toBe(1) + }) + + it('rejects dynamic writes whose scope cannot be cleaned after a crash', async () => { + let providerWrites = 0 + const result = await runAgentMemoryExperiment({ + experimentId: 'undeclared-dynamic-scope', + sequences: [ + { + id: 'one', + family: 'first-party', + steps: [ + { + id: 'agent-step', + scope: { agentId: 'declared' }, + probes: [{ id: 'state', query: 'state', referenceAnswer: 'state' }], + }, + ], + }, + ], + candidates: [ + { + id: 'scoped', + ref: 'scoped:v1', + createAdapter: () => + createScopedTestAdapter('scoped', async () => { + providerWrites += 1 + }), + }, + ], + runDir: '/runs/undeclared-dynamic-scope', + storage: inMemoryCampaignStorage(), + cleanupBranches: true, + executeStepRef: 'dynamic-scope-test/v1', + executeStep: async ({ memory }) => { + await memory.write({ + kind: 'fact', + text: 'must not reach the provider', + scope: { agentId: 'undeclared' }, + }) + }, + }) + + expect(result.rows[0]).toMatchObject({ cellsFailed: 1 }) + expect(result.campaign.cells[0]?.error).toContain( + 'write scope was not declared in the experiment sequence cleanupScopes', + ) + expect(providerWrites).toBe(0) + }) + + it('requires an external-state disposer when scoped cleanup is disabled', async () => { + let creates = 0 + await expect( + runAgentMemoryExperiment({ + experimentId: 'missing-disposer', + sequences: [ + { + id: 'one', + family: 'first-party', + steps: [ + { + id: 'probe', + scope: { agentId: 'worker' }, + probes: [{ id: 'state', query: 'state', referenceAnswer: 'state' }], + }, + ], + }, + ], + candidates: [ + { + id: 'persistent', + ref: 'persistent:v1', + createAdapter() { + creates += 1 + return createScopedTestAdapter('persistent') + }, + }, + ], + runDir: '/runs/missing-disposer', + storage: inMemoryCampaignStorage(), + cleanupBranches: false, + }), + ).rejects.toThrow('requires disposeAdapter') + + expect(creates).toBe(0) + }) + + it('rejects probes that cannot distinguish a useful memory system', async () => { + await expect( + runAgentMemoryExperiment({ + experimentId: 'no-target', + sequences: [ + { + id: 'one', + family: 'first-party', + steps: [ + { + id: 'probe', + scope: { agentId: 'worker' }, + probes: [{ id: 'state', query: 'state' }], + }, + ], + }, + ], + candidates: [ + { + id: 'memory', + ref: 'memory:v1', + createAdapter: () => createScopedTestAdapter('memory'), + }, + ], + runDir: '/runs/no-target', + storage: inMemoryCampaignStorage(), + }), + ).rejects.toThrow('has no measurable target') + }) +}) + +describe('agent memory improvement', () => { + it('fails fast when a JavaScript caller uses the removed onPromote option', async () => { + await expect( + runAgentMemoryImprovementRaw({ + onPromote() {}, + } as unknown as RunAgentMemoryImprovementOptions), + ).rejects.toThrow( + 'onPromote was removed; use activation.readCurrent and activation.compareAndSet', + ) + }) + + it('requires an explicit controller policy for custom improvement storage', async () => { + await expect( + runAgentMemoryImprovementRaw({ + experimentId: 'custom-improvement-storage', + trainSequences: [improvementSequence('train', 'train')], + holdoutSequences: [improvementSequence('holdout', 'holdout')], + seeds: [ + { + config: { mode: 'baseline' }, + track: 'baseline', + proposer: 'default', + }, + ], + createCandidate: () => ({ + ref: 'memory:v1', + createAdapter: () => createScopedTestAdapter('memory'), + }), + proposer: { kind: 'noop', propose: async () => [] }, + improvementRef: 'custom-improvement-storage:v1', + budget: { maxSteps: 1 }, + runDir: '/runs/custom-improvement-storage', + storage: inMemoryCampaignStorage(), + }), + ).rejects.toThrow("requires acquireRunLease or controllerMode='process-local'") + }) + + it('searches isolated configs and activates only a fresh holdout win', async () => { + type Config = { visibility: 'private' | 'team' | 'shared' } + const storage = inMemoryCampaignStorage() + const promoted: Config[] = [] + let activeConfig: Config = { visibility: 'private' } + const activationIds: string[] = [] + const contenderIds = new Set() + const activeContenderCalls = new Map() + let maxConcurrentConfigs = 0 + let reportContendersActive: (() => void) | undefined + const contendersActive = new Promise((resolve) => { + reportContendersActive = resolve + }) + let releaseContenders: (() => void) | undefined + const continueContenders = new Promise((resolve) => { + releaseContenders = resolve + }) + let proposalCalls = 0 + const proposer: SurfaceProposer = { + kind: 'team-sharing-proposer', + async propose() { + proposalCalls += 1 + return [ + { + surface: JSON.stringify({ visibility: 'team' }), + label: 'share within the team', + rationale: "the second agent needs the first agent's accepted fact", + }, + { + surface: JSON.stringify({ visibility: 'shared' }), + label: 'share globally', + rationale: 'compare a broader sharing policy under the same histories', + }, + ] + }, + } + + const options: RunAgentMemoryImprovementOptions = { + experimentId: 'improve-team-memory', + trainSequences: [ + improvementSequence('train-a', 'train'), + improvementSequence('train-b', 'train'), + ], + holdoutSequences: [ + improvementSequence('holdout-a', 'holdout'), + improvementSequence('holdout-b', 'holdout'), + ], + seeds: [ + { + config: { visibility: 'private' }, + track: 'baseline', + proposer: 'seed', + }, + ], + proposer, + improvementRef: 'team-memory-policy/v1', + budget: { maxSteps: 1 }, + populationSize: 2, + candidateConcurrency: 2, + sequenceConcurrency: 4, + runDir: '/runs/improve-team-memory', + storage, + significance: { minProductiveRuns: 2, resamples: 200, seed: 7 }, + createCandidate: ({ config, candidateId }) => { + if (config.visibility !== 'private') contenderIds.add(candidateId) + return { + ref: `visibility:${config.visibility}:v1`, + label: config.visibility, + policy: { read: [config.visibility], write: config.visibility }, + createAdapter: ({ branchId }) => createScopedTestAdapter(`${candidateId}:${branchId}`), + } + }, + executeStepRef: 'parallel-config-proof/v1', + executeStep: async ({ candidateId, step }) => { + if (step.id !== 'research' || !contenderIds.has(candidateId)) return + activeContenderCalls.set(candidateId, (activeContenderCalls.get(candidateId) ?? 0) + 1) + maxConcurrentConfigs = Math.max(maxConcurrentConfigs, activeContenderCalls.size) + if (activeContenderCalls.size === 2) reportContendersActive?.() + try { + await continueContenders + } finally { + const remaining = (activeContenderCalls.get(candidateId) ?? 1) - 1 + if (remaining === 0) activeContenderCalls.delete(candidateId) + else activeContenderCalls.set(candidateId, remaining) + } + }, + activation: { + ref: 'memory-policy/live:v1', + async readCurrent() { + return structuredClone(activeConfig) + }, + async compareAndSet({ activationId, expectedConfig, config }) { + expect(activeConfig).toEqual(expectedConfig) + activationIds.push(activationId) + activeConfig = structuredClone(config) + promoted.push(structuredClone(config)) + }, + }, + } + const firstRun = runAgentMemoryImprovement(options) + await contendersActive + await expect(runAgentMemoryImprovement(options)).rejects.toThrow('active controller') + releaseContenders?.() + const result = await firstRun + + expect(result.decision).toMatchObject({ + status: 'promote', + reasons: [], + baselineScore: 0.25, + winnerScore: 1, + lift: 0.75, + }) + expect(result.decision.significance).toMatchObject({ n: 2, significant: true }) + expect(result.winnerConfig).toEqual({ visibility: 'team' }) + expect(result.holdout?.campaign.cells).toHaveLength(4) + expect(maxConcurrentConfigs).toBe(2) + expect(promoted).toEqual([{ visibility: 'team' }]) + expect(result.activation).toMatchObject({ status: 'activated' }) + expect(storage.read(result.resultJsonPath)).toContain('"status": "promote"') + expect( + JSON.parse(storage.read('/runs/improve-team-memory/memory-improvement-manifest.json') ?? '{}') + .identity?.schema, + ).toBe(6) + + const resumed = await runAgentMemoryImprovement(options) + expect(proposalCalls).toBe(1) + expect(promoted).toEqual([{ visibility: 'team' }]) + expect(activationIds).toEqual([result.activation.id]) + expect(resumed.activation).toEqual({ + ...result.activation, + status: 'already-activated', + }) + expect(activeConfig).toEqual({ visibility: 'team' }) + await expect( + runAgentMemoryImprovement({ ...options, budget: { maxSteps: 2 } }), + ).rejects.toThrow('does not match its persisted inputs or implementationRef') + await expect(runAgentMemoryImprovement({ ...options, minHoldoutScore: 0.99 })).rejects.toThrow( + 'does not match its persisted inputs or implementationRef', + ) + await expect( + runAgentMemoryImprovement({ ...options, improvementRef: 'team-memory-policy/v2' }), + ).rejects.toThrow('does not match its persisted inputs or implementationRef') + }) + + it('recovers when the live config changes before the activation event is persisted', async () => { + type Config = { visibility: 'private' | 'team' } + const storage = inMemoryCampaignStorage() + const append = storage.append!.bind(storage) + let rejectActivatedEvent = true + storage.append = (path, value, expectedBytes) => { + if ( + rejectActivatedEvent && + path.includes('/activations/') && + value.includes('"status":"activated"') + ) { + rejectActivatedEvent = false + throw new Error('activation journal unavailable') + } + return append(path, value, expectedBytes) + } + let activeConfig: Config = { visibility: 'private' } + let compareAndSetCalls = 0 + const options: RunAgentMemoryImprovementOptions = { + experimentId: 'recover-memory-activation', + trainSequences: [ + improvementSequence('activation-train-a', 'train'), + improvementSequence('activation-train-b', 'train'), + ], + holdoutSequences: [ + improvementSequence('activation-holdout-a', 'holdout'), + improvementSequence('activation-holdout-b', 'holdout'), + ], + seeds: [ + { + config: { visibility: 'private' }, + track: 'baseline', + proposer: 'seed', + }, + ], + proposer: { + kind: 'team-sharing-proposer', + async propose() { + return [ + { + surface: JSON.stringify({ visibility: 'team' }), + label: 'share with team', + rationale: 'the second agent needs the accepted fact', + }, + ] + }, + }, + improvementRef: 'recover-memory-activation:v1', + budget: { maxSteps: 1 }, + runDir: '/runs/recover-memory-activation', + storage, + significance: { minProductiveRuns: 2, resamples: 200, seed: 11 }, + createCandidate: ({ config, candidateId }) => ({ + ref: `visibility:${config.visibility}:v1`, + policy: { read: [config.visibility], write: config.visibility }, + createAdapter: ({ branchId }) => createScopedTestAdapter(`${candidateId}:${branchId}`), + }), + activation: { + ref: 'memory-policy/live:v1', + async readCurrent() { + return structuredClone(activeConfig) + }, + async compareAndSet({ expectedConfig, config }) { + expect(activeConfig).toEqual(expectedConfig) + compareAndSetCalls += 1 + activeConfig = structuredClone(config) + }, + }, + } + + await expect(runAgentMemoryImprovement(options)).rejects.toThrow( + 'activation journal unavailable', + ) + expect(activeConfig).toEqual({ visibility: 'team' }) + expect(compareAndSetCalls).toBe(1) + + const recovered = await runAgentMemoryImprovement(options) + expect(recovered.activation.status).toBe('recovered') + expect(compareAndSetCalls).toBe(1) + const resumed = await runAgentMemoryImprovement(options) + expect(resumed.activation.status).toBe('already-activated') + expect(compareAndSetCalls).toBe(1) + }) + + it('routes independent tracks to their named proposers with track context', async () => { + type Config = { visibility: 'private' | 'team' } + const trackContexts: Array<{ + id?: string + operation?: string + vision?: string + generation: number + costPhase?: string + hasCostLedger: boolean + }> = [] + let governorCostPhase: string | undefined + let governorHasCostLedger = false + const trackProposer: SurfaceProposer = { + kind: 'team-memory-researcher', + async propose(context) { + trackContexts.push({ + id: context.track?.id, + operation: context.track?.operation, + vision: context.track?.vision, + generation: context.generation, + costPhase: context.costPhase, + hasCostLedger: context.costLedger !== undefined, + }) + return [JSON.stringify({ visibility: 'team' })] + }, + } + + await runAgentMemoryImprovement({ + experimentId: 'named-track-proposers', + trainSequences: [improvementSequence('track-train', 'train')], + holdoutSequences: [improvementSequence('track-holdout', 'holdout')], + seeds: [ + { config: { visibility: 'private' }, track: 'baseline', proposer: 'baseline' }, + { + config: { visibility: 'private' }, + track: 'sharing-research', + proposer: 'team-memory-researcher', + vision: 'test whether team memory transfers accepted facts', + }, + ], + proposer: { + kind: 'unexpected-fallback', + async propose() { + throw new Error('named track should not use the fallback proposer') + }, + }, + proposers: { 'team-memory-researcher': trackProposer }, + governor: { + decide(context) { + governorCostPhase = context.costPhase + governorHasCostLedger = context.costLedger !== undefined + return { op: 'extend', track: 'sharing-research' } + }, + }, + improvementRef: 'named-track-proposers/v1', + budget: { maxSteps: 1 }, + populationSize: 1, + runDir: '/runs/named-track-proposers', + storage: inMemoryCampaignStorage(), + createCandidate: ({ config, candidateId }) => ({ + ref: `visibility:${config.visibility}:v1`, + policy: { read: [config.visibility], write: config.visibility }, + createAdapter: ({ branchId }) => createScopedTestAdapter(`${candidateId}:${branchId}`), + }), + }) + + expect(trackContexts).toEqual([ + { + id: 'sharing-research', + operation: 'extend', + vision: 'test whether team memory transfers accepted facts', + generation: 1, + costPhase: 'memory.proposal.sharing-research', + hasCostLedger: true, + }, + ]) + expect(governorCostPhase).toBe('memory.governor') + expect(governorHasCostLedger).toBe(true) + }) + + it('holds a winner when holdout histories do not test a critical dimension', async () => { + type Config = { visibility: 'private' | 'team' } + const result = await runAgentMemoryImprovement({ + experimentId: 'missing-critical-dimension', + trainSequences: [improvementSequence('critical-train', 'train')], + holdoutSequences: [ + improvementSequence('critical-holdout-a', 'holdout', false), + improvementSequence('critical-holdout-b', 'holdout', false), + improvementSequence('critical-holdout-c', 'holdout', false), + ], + seeds: [{ config: { visibility: 'private' }, track: 'baseline', proposer: 'sharing' }], + proposer: { + kind: 'sharing', + async propose() { + return [JSON.stringify({ visibility: 'team' })] + }, + }, + improvementRef: 'missing-critical-dimension/v1', + budget: { maxSteps: 1 }, + populationSize: 1, + runDir: '/runs/missing-critical-dimension', + storage: inMemoryCampaignStorage(), + significance: { minProductiveRuns: 1, resamples: 100, seed: 9 }, + criticalDimensions: ['memory_stale_safe'], + createCandidate: ({ config, candidateId }) => ({ + ref: `visibility:${config.visibility}:v1`, + policy: { read: [config.visibility], write: config.visibility }, + createAdapter: ({ branchId }) => createScopedTestAdapter(`${candidateId}:${branchId}`), + }), + }) + + expect(result.decision.status).toBe('hold') + expect(result.decision.criticalDimensions).toEqual([ + expect.objectContaining({ + dimension: 'memory_stale_safe', + n: 0, + expectedN: 0, + measured: false, + }), + ]) + expect(result.decision.reasons).toContain( + 'critical dimension memory_stale_safe has no applicable holdout histories', + ) + }) + + it('stops before a proposer call would exceed the run-wide cost limit', async () => { + let proposerExecuted = false + + await expect( + runAgentMemoryImprovement({ + experimentId: 'proposer-cost-limit', + trainSequences: [improvementSequence('cost-train', 'train')], + holdoutSequences: [improvementSequence('cost-holdout', 'holdout')], + seeds: [ + { + config: { visibility: 'private' as const }, + track: 'baseline', + proposer: 'costed', + }, + ], + proposer: { + kind: 'costed', + async propose(context) { + if (!context.costLedger) throw new Error('missing run cost ledger') + const paid = await context.costLedger.runPaidCall({ + actor: 'memory-config-proposer', + channel: 'agent', + phase: context.costPhase, + model: 'fixture-model', + maximumCharge: { externallyEnforcedMaximumUsd: 0.06 }, + execute: async () => { + proposerExecuted = true + return JSON.stringify({ visibility: 'team' }) + }, + receipt: () => ({ + model: 'fixture-model', + inputTokens: 0, + outputTokens: 0, + usageUnknown: true, + actualCostUsd: 0.06, + }), + }) + if (!paid.succeeded) throw paid.error + return [paid.value] + }, + }, + improvementRef: 'proposer-cost-limit/v1', + budget: { maxSteps: 1 }, + maxTotalCostUsd: 0.05, + runDir: '/runs/proposer-cost-limit', + storage: inMemoryCampaignStorage(), + createCandidate: ({ config, candidateId }) => ({ + ref: `visibility:${config.visibility}:v1`, + policy: { read: [config.visibility], write: config.visibility }, + createAdapter: ({ branchId }) => createScopedTestAdapter(`${candidateId}:${branchId}`), + }), + }), + ).rejects.toThrow('would exceed ceiling 0.05') + expect(proposerExecuted).toBe(false) + }) + + it('charges an interrupted proposer reservation before resuming the search', async () => { + type Config = { visibility: 'private' | 'team' } + const storage = inMemoryCampaignStorage() + const runDir = '/runs/interrupted-proposer-recovery' + const append = storage.append!.bind(storage) + let failFirstProposerReceipt = true + storage.append = (path, value, expectedBytes) => { + if ( + failFirstProposerReceipt && + path.endsWith('/cost-ledger.jsonl') && + value.includes('"status":"settled"') && + value.includes('memory-config-proposer') + ) { + failFirstProposerReceipt = false + throw new Error('simulated process exit before proposer receipt') + } + return append(path, value, expectedBytes) + } + let proposerCalls = 0 + const options: RunAgentMemoryImprovementOptions = { + experimentId: 'interrupted-proposer-recovery', + trainSequences: [improvementSequence('proposer-train', 'train')], + holdoutSequences: [improvementSequence('proposer-holdout', 'holdout')], + seeds: [ + { + config: { visibility: 'private' }, + track: 'baseline', + proposer: 'costed', + }, + ], + proposer: { + kind: 'costed', + async propose(context) { + proposerCalls += 1 + if (!context.costLedger) throw new Error('missing run cost ledger') + const paid = await context.costLedger.runPaidCall({ + actor: 'memory-config-proposer', + channel: 'agent', + phase: context.costPhase, + model: 'fixture-model', + maximumCharge: { externallyEnforcedMaximumUsd: 0.1 }, + execute: async () => JSON.stringify({ visibility: 'team' }), + receipt: () => ({ + model: 'fixture-model', + inputTokens: 0, + outputTokens: 0, + actualCostUsd: 0.1, + }), + }) + if (!paid.succeeded) throw paid.error + return [paid.value] + }, + }, + improvementRef: 'interrupted-proposer-recovery/v1', + budget: { maxSteps: 1 }, + maxTotalCostUsd: 0.2, + runDir, + storage, + createCandidate: ({ config, candidateId }) => ({ + ref: `visibility:${config.visibility}:v1`, + policy: { read: [config.visibility], write: config.visibility }, + createAdapter: ({ branchId }) => createScopedTestAdapter(`${candidateId}:${branchId}`), + }), + } + + await expect(runAgentMemoryImprovement(options)).rejects.toThrow('failed to persist') + const interruptedLedger = createRunCostLedger({ + storage, + runDir, + costCeilingUsd: 0.2, + }) + expect(interruptedLedger.listPending()).toEqual([ + expect.objectContaining({ actor: 'memory-config-proposer', state: 'interrupted' }), + ]) + + const result = await runAgentMemoryImprovement(options) + const resumedLedger = createRunCostLedger({ storage, runDir, costCeilingUsd: 0.2 }) + + expect(proposerCalls).toBe(2) + expect(result.totalCostUsd).toBe(0.2) + expect(resumedLedger.summary()).toMatchObject({ + totalCalls: 2, + unresolvedCalls: 0, + totalCostUsd: 0.2, + accountingComplete: true, + }) + expect(resumedLedger.list()[0]).toMatchObject({ + actor: 'memory-config-proposer', + costUsd: 0.1, + error: expect.stringContaining('charged the reserved maximum'), + }) + }) + + it('rejects train and holdout histories with the same id', async () => { + const sequence = improvementSequence('duplicate', 'train') + await expect( + runAgentMemoryImprovement({ + experimentId: 'overlap', + trainSequences: [sequence], + holdoutSequences: [{ ...sequence, split: 'holdout' }], + seeds: [{ config: {}, track: 'baseline', proposer: 'seed' }], + proposer: { + kind: 'unused', + async propose() { + return [] + }, + }, + improvementRef: 'overlap-test/v1', + budget: { maxSteps: 1 }, + runDir: '/runs/overlap', + storage: inMemoryCampaignStorage(), + createCandidate: () => ({ + ref: 'unused:v1', + createAdapter: () => createScopedTestAdapter('unused'), + }), + }), + ).rejects.toThrow('train/holdout overlap: duplicate') + }) + + it('rejects a holdout history copied under a different id', async () => { + const train = improvementSequence('train-original', 'train') + await expect( + runAgentMemoryImprovement({ + experimentId: 'renamed-overlap', + trainSequences: [train], + holdoutSequences: [{ ...train, id: 'renamed-holdout', split: 'holdout' }], + seeds: [{ config: {}, track: 'baseline', proposer: 'seed' }], + proposer: { + kind: 'unused', + async propose() { + return [] + }, + }, + improvementRef: 'renamed-overlap/v1', + budget: { maxSteps: 0 }, + runDir: '/runs/renamed-overlap', + storage: inMemoryCampaignStorage(), + createCandidate: () => ({ + ref: 'unused:v1', + createAdapter: () => createScopedTestAdapter('unused'), + }), + }), + ).rejects.toThrow('histories duplicate content') + }) +}) + +function improvementSequence(id: string, split: 'train' | 'holdout', includeStaleTarget = true) { + return { + id, + family: 'first-party' as const, + split, + steps: [ + { + id: 'research', + scope: { agentId: 'researcher', teamId: 'team-1' }, + writes: [ + { + id: `${id}-event`, + kind: 'fact' as const, + text: `${id} launch date is Friday`, + metadata: { eventId: `${id}-event`, actorId: 'researcher' }, + }, + ], + }, + { + id: 'delivery', + scope: { agentId: 'builder', teamId: 'team-1' }, + probes: [ + { + id: 'launch-date', + query: `${id} launch date`, + requiredFacts: [{ id: 'current', anyOf: [`${id} launch date is Friday`] }], + ...(includeStaleTarget + ? { + forbiddenFacts: [ + { + id: 'stale', + anyOf: [`${id} launch date is Thursday`], + obsolete: true, + }, + ], + } + : {}), + expectedEventIds: [`${id}-event`], + expectedActorIds: ['researcher'], + }, + ], + }, + ], + } +} + +function hitText(hit: AgentMemoryHit): string { + return hit.text +} + +function createScopedTestAdapter( + id: string, + beforeWrite?: (scope: AgentMemoryScope, text: string) => Promise, +): AgentMemoryAdapter { + const rows: Array<{ scope: AgentMemoryScope; hit: AgentMemoryHit }> = [] + let sequence = 0 + return { + id, + branchIsolation: { mode: 'scoped' }, + async search(_query, options = {}) { + return rows.filter((row) => sameScope(row.scope, options.scope)).map((row) => row.hit) + }, + async getContext(query, options = {}) { + const hits = await this.search(query, options) + return { query, hits, text: hits.map(hitText).join('\n'), sourceRecords: [] } + }, + async write(input) { + const scope = input.scope ?? {} + await beforeWrite?.(scope, input.text) + sequence += 1 + const memoryId = input.id ?? `${id}-${sequence}` + const hit: AgentMemoryHit = { + id: memoryId, + uri: `memory://${id}/${memoryId}`, + kind: input.kind, + text: input.text, + metadata: input.metadata, + } + rows.push({ scope, hit }) + return { accepted: true, id: memoryId, uri: hit.uri, kind: input.kind } + }, + async clear(scope) { + for (let index = rows.length - 1; index >= 0; index -= 1) { + if (sameScope(rows[index]!.scope, scope)) rows.splice(index, 1) + } + }, + } +} + +function sameScope(left: AgentMemoryScope, right: AgentMemoryScope = {}): boolean { + for (const key of [ + 'tenantId', + 'userId', + 'agentId', + 'teamId', + 'runId', + 'sessionId', + 'namespace', + ] as const) { + if (right[key] !== undefined && left[key] !== right[key]) return false + } + for (const [key, value] of Object.entries(right.tags ?? {})) { + if (left.tags?.[key] !== value) return false + } + return true +}