diff --git a/docs/architecture/openai-agents-sessions.md b/docs/architecture/openai-agents-sessions.md index d96f44f582..164a146f47 100644 --- a/docs/architecture/openai-agents-sessions.md +++ b/docs/architecture/openai-agents-sessions.md @@ -4,12 +4,23 @@ Loom episodic memories. Existing tools can separately retain facts beyond a conversation. The framework packages remain optional dependencies. -Each item is one `memory.set` call with its native JSON, a timestamp-and-random -key, the end user and a session derived from the user and native session id. -Point writes bypass SDK coalescing so concurrent repeated items remain separate. -Episodic rows preserve repeated items without semantic extraction or duplicate -merging. Python and TypeScript use separate derived-session prefixes because -their native item formats differ. +Each nonempty addition encodes every native item before discovery or mutation, +checks that the server advertises `memory.append_batch`, and sends one request. +The server requires the Postgres memory authority. It prepares every episodic row +and commits the whole addition only when every key is fresh under the generation +and head locks. A refused item or existing key refuses the batch. There is no +point-write or ordinary-bulk fallback; an empty addition sends nothing. The SDKs +and server come from the same checkout, as the +[installation instructions](../features/integrations/frameworks.md#openai-agents-sdk) +require. + +Each item retains its native JSON, timestamp-and-random key, end user and session +derived from the user and native session id. Episodic rows preserve repeated +values as distinct occurrences without semantic extraction or duplicate merging. +Python and TypeScript use separate derived-session prefixes because their native +item formats differ. A committed batch can report `PUBLICATION_PENDING`; resolve +an uncertain outcome with the original request's idempotency key, without a fresh +addition or compensating deletion. Reads page through `memory.list`, deduplicate overlapping keys and hydrate missing or truncated values with `memory.get`. Items are returned in key order, with a @@ -19,8 +30,8 @@ snapshots the validated user at construction. Its native image and file byte payloads use JSON arrays in storage and return as `Uint8Array`; ordinary JSON and media information remain unchanged. Deletion carries the user and derived session: `pop_item` deletes the latest item, and `clear_session` deletes the listed -history. Concurrent appends or clears are individual server operations, not a -transaction across the conversation. +history. Each addition is atomic; concurrent history paging, pops and clears do +not form a transaction across the conversation. If an item disappears before hydration, the read fails instead of returning an incomplete native item. Keys use writer clocks; concurrent writers do not share a server sequence. @@ -36,9 +47,11 @@ conversation. Native history reads and additions run as automatic work under all retained contributors. A marked reply protects every contributor before its value returns. A complete first marked history reply may still return its items; subsequent -pagination, hydration or item writes are suppressed. A marked item write stops a -batch before the next item. Explicit pop and clear operations remain usable after -protection and keep observing replies under the retained contributors. +pagination, hydration or batch writes are suppressed. A marked discovery response +prevents the append request. A marked append acknowledgement protects the retained +contributors after the whole batch may already have committed; it does not imply +rollback. Explicit pop and clear operations remain usable after protection and +keep observing replies under the retained contributors. Model-selected tools keep their explicit operation semantics. ## Source entry points diff --git a/docs/architecture/storage-authority.md b/docs/architecture/storage-authority.md index 04231f404b..276c5fb638 100644 --- a/docs/architecture/storage-authority.md +++ b/docs/architecture/storage-authority.md @@ -114,6 +114,34 @@ stops serving and background writers, waits out bounded sends, and states when t guarantee and optional settings apply. Both builds serve the same schema. An older writer can still reproduce the unchecked expiry race during that transition. +## Atomic episodic appends + +`memory.append_batch` appends 1–1,000 episodic occurrences under one identity tuple. +Every key must be distinct and absent, including from tombstoned heads. Equal values +remain separate rows. The handler requires the Postgres authority before memory or +provider mutation, then reuses bulk row preparation, blacklist checks, sensitivity, +provenance and policy admission. It runs no embedding, identity deduplication, entity +resolution or semantic supersession. Any refused entry aborts the entire batch. + +The commit checks absence again while holding the authority's generation and head +locks. Heads, revision rows, hash-only audit rows, delivery obligations and the bound +idempotency result commit together. A competing writer that creates one key refuses +the complete append. Ordinary `memory.bulk_set` keeps its per-entry verdicts and +identity deduplication; its partial selection is unsuitable for native history. + +Publication follows the commit. `PUBLICATION_PENDING` means the batch is committed and +its remaining delivery belongs to the delivery worker. A replay of that request's +idempotency key returns the committed rows, without appending again. A lost commit +acknowledgement remains an uncertain outcome until that same request is resolved; +clients must not compensate with deletes or invent replacement keys. + +The operation uses the existing episodic row format and erasure coverage, with no +schema migration. Older servers do not register it and refuse rather than accepting +an ignored atomicity flag. The ClickHouse authority refuses it; the operation catalogue +advertises availability, not the deployment's storage readiness. Existing history +reads, pops and clears keep their row format and behavior; this append operation does +not give them cross-process linearizability. + ## The tables The schema is `schemas/pg_migrations/` ([migrations](#migrations)). Every tenant primary diff --git a/docs/features/integrations/frameworks.md b/docs/features/integrations/frameworks.md index d72fba6d94..e5b3a4b88d 100644 --- a/docs/features/integrations/frameworks.md +++ b/docs/features/integrations/frameworks.md @@ -306,8 +306,16 @@ line per hit. With a `SyncAgent` the tool is sync; with an `Agent` it is async, ## OpenAI Agents SDK `loom_tools` returns Loom's agent verbs as the OpenAI Agents SDK's `FunctionTool`s, so the -model keeps and reads long-term memory in Loom during a run. Install with -`pip install 'clickhouse-loom[openai-agents]'` (`openai-agents` 0.24). +model keeps and reads long-term memory in Loom during a run. For the Python examples, +install the SDK from the root of the Loom checkout that supplies this guide: + +```bash +pip install './sdk/python[openai-agents]' +``` + +The extra installs `openai-agents` 0.24. Native sessions also require a server built +from that same checkout with the Postgres memory authority; see +[running the server from source](../../development/local-development.md#running-the-server-from-source). ```python from agents import Agent as OpenAIAgent, Runner @@ -363,13 +371,29 @@ result = await Runner.run(assistant, "Book my usual seat to Lisbon", session=ses | `Session` | Loom | | --------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | `get_items` | `memory.list` of the session's items, sorted by writer-generated key; `limit` (or `session_settings.limit`) keeps the latest after scanning the history; an item the list cuts at 2,048 characters, or sends without a value, is read whole with `memory.get` | -| `add_items` | one `memory.set` per item, an `episodic` memory under `/-`, the item's JSON its value | +| `add_items` | One `memory.append_batch` per call, each item an `episodic` row under `/-` with the item's JSON as its value | | `pop_item` | the latest item, read as `get_items(limit=1)` reads it, deleted with `memory.delete` and returned | | `clear_session` | `memory.delete` of every item of the session | +Python and TypeScript additions require a server advertising +[`memory.append_batch`](../../reference/operations/memory.md#memoryappend_batch) and using the +Postgres memory authority. A nonempty call encodes every item before sending one request +of at most 1,000 items. Equal values remain distinct occurrences. A refused entry or an +existing key refuses the whole batch before any history is appended. An empty call sends +nothing. Discovery failures, an older replica's missing operation and storage refusals +propagate without a point-write or bulk-write fallback. Existing history remains readable +on older servers. This changes the server requirement for additions in both SDKs; their supported +OpenAI Agents SDK dependency ranges are unchanged. + +A committed batch whose publication is pending reports `PUBLICATION_PENDING` with +`committed=true`; it is not a rollback. Preserve the original request's idempotency key +when resolving an uncertain outcome. Do not retry as a fresh addition or delete earlier +items to compensate. These append guarantees do not make concurrent history paging, +`pop_item` or `clear_session` atomic. + An `episodic` memory is not embedded, so an item is never merged with or judged against -another row, and a repeated item stays two items (one `memory.set` per item, bypassing SDK -coalescing: a batch would fold a value identical to a live row). A read fails if an item +another row. Repeated values remain separate occurrences in both SDKs' batches. +A read fails if an item disappears before its full value can be fetched. Items are stored under a Loom session derived from the user and the session id (`oai-py-` and a digest of the pair in Python, `oai-ts-` in TypeScript). Keys follow writer clocks; independent writers do not share a server @@ -388,6 +412,14 @@ established beyond its history, give the model `loom_remember`, or send the turn | `key_prefix` | `openai-agents` | the items' key prefix | | `session_settings` | the SDK's | `limit`: the default number of latest items `get_items` returns | +Build and install the TypeScript SDK from that same checkout using the +[SDK source instructions](../../../sdk/typescript/README.md). In your application, +install its supported native peer: + +```bash +npm install '@openai/agents@>=0.22.0 <0.23.0' +``` + In TypeScript, `loomTools(agent, scope)` from `@clickhouse/loom/integrations/openai-agents` returns the same tools built with `tool()` from `@openai/agents`, an optional peer dependency: diff --git a/docs/reference/http-api.md b/docs/reference/http-api.md index 7230e8a512..5320b581d5 100644 --- a/docs/reference/http-api.md +++ b/docs/reference/http-api.md @@ -159,13 +159,14 @@ curl -sX POST "$LOOM/v1/memory.set" \ blank or `"-"` org is the token's own. On a single-user server, a call with no token that names no `org`, or a blank or `"-"` one, runs in the install's org (`LOOM_SELF_SIGNUP_ORG`), the one MCP and the dashboard use; a code or tracker op - keeps its plane org. A call with no token to `memory.set`, `memory.bulk_set`, - `memory.get`, `memory.delete`, `memory.list`, `memory.history`, `memory.search`, + keeps its plane org. A call with no token to `memory.set`, `memory.append_batch`, + `memory.bulk_set`, `memory.get`, `memory.delete`, `memory.list`, `memory.history`, + `memory.search`, `memory.search_trace`, `memory.ask`, `memory.update`, `search.query` or `knowledge.search` may also leave out `namespace` and `agent`: it runs in the - install's namespace (`LOOM_DEFAULT_NAMESPACE`). With no agent, `memory.set` and - `memory.bulk_set` write as agent `http`; `memory.get` and `memory.delete` try the - key under agent `http`, then under any agent of the same user; `memory.list` reads + install's namespace (`LOOM_DEFAULT_NAMESPACE`). With no agent, `memory.set`, + `memory.append_batch` and `memory.bulk_set` write as agent `http`; `memory.get` and + `memory.delete` try the key under agent `http`, then under any agent of the same user; `memory.list` reads every agent's rows of the call's user and session (`-` when it names none) and `memory.history` every agent's, user's and session's versions of the key, as on MCP; a `memory.history` that names a `user_id` or `session_id` names its agent diff --git a/docs/reference/operations.md b/docs/reference/operations.md index da9c8cdd75..8eab7721bb 100644 --- a/docs/reference/operations.md +++ b/docs/reference/operations.md @@ -123,7 +123,7 @@ python scripts/codegen/docs_reference.py --check # exit 1 when a region is stal -375 operations in 43 groups. Every op is served at `POST /v1/` and as the MCP tool `loom_` (dots become underscores). +376 operations in 43 groups. Every op is served at `POST /v1/` and as the MCP tool `loom_` (dots become underscores). Columns: **Role** is the minimum role (`viewer` < `writer` < `admin` < `org_admin` < `platform_admin`); **Kind** is `read` (idempotent, viewer, unaudited, writes nothing — MCP `readOnlyHint`), `idempotent` (safe to retry) or `write`; **Listed under `all`** says whether an MCP connection on the `all` toolset lists the op in `tools/list` (`code.*` ops only with `LOOM_MCP_CODE_TOOLS=true`); an op marked `no` is still callable by name or through `loom_op`, unless it is hidden from MCP; **Audit** marks ops that leave an `audit_log` row. @@ -476,6 +476,7 @@ Fields for every `memory` op: [operations/memory.md](operations/memory.md). | --- | --- | --- | --- | --- | --- | | [`memory.agents_list`](operations/memory.md#memoryagents_list) | Distinct agent names that have ever written into (org, namespace). agent + user_id are forced sentinels — we aggregate ACROSS them to discover which agents exist. | viewer | read | yes | | | [`memory.aggregate`](operations/memory.md#memoryaggregate) | Templated SQL aggregation over typed records in loom.memory. | viewer | read | no | | +| [`memory.append_batch`](operations/memory.md#memoryappend_batch) | Append distinct episodic occurrences atomically on the Postgres authority. | writer | idempotent | no | | | [`memory.apply`](operations/memory.md#memoryapply) | One scope grammar for every write that can touch more than one row. | writer | write | yes | yes | | [`memory.ask`](operations/memory.md#memoryask) | Ask a question and get a grounded answer synthesized from the caller's recalled memories (or, with allow_synthesis=False, just the memories). | viewer | read | yes | | | [`memory.assign_category`](operations/memory.md#memoryassign_category) | Link a memory to one of the namespace's registered categories. | writer | idempotent | no | yes | diff --git a/docs/reference/operations/memory.md b/docs/reference/operations/memory.md index 2b31b27e5a..d0f9c7b6a3 100644 --- a/docs/reference/operations/memory.md +++ b/docs/reference/operations/memory.md @@ -52,6 +52,38 @@ Templated SQL aggregation over typed records in loom.memory. | `rows` | `list[_AggregateRow]` | One row per group (a single row when `group_by` is empty). | | `sql_executed` | `str` | the exact SQL run, for debugging without trace logs | +## `memory.append_batch` + +Append distinct episodic occurrences atomically on the Postgres authority. + +`POST /v1/memory.append_batch` · MCP `loom_memory_append_batch` · role `writer` · idempotent · not listed under the MCP `all` toolset + +| Input field | Type | Default | Meaning | +| --- | --- | --- | --- | +| `org` | `str` | required | The tenant (first element of the identity tuple). | +| `namespace` | `str` | required | The namespace the op reads or writes. | +| `agent` | `str` | required | The writing or reading agent; `'-'` or `'*'` means any agent where the op allows it. | +| `user_id` | `str` | `'-'` | The end user the row belongs to; `'-'` for none. | +| `session_id` | `str` | `'-'` | The session or conversation the row belongs to; `'-'` for none. | +| `run_id` | `str` | `'-'` | The agent run (task scope) the call belongs to; `'-'` for none. | +| `entries` | `list[_AppendBatchEntry]` | required | | + +| Output field | Type | Meaning | +| --- | --- | --- | +| `written` | `list[_BulkSetResult]` | One result per entry, in order: key, version (0 when not inserted) and verdict. | +| `skipped_pinned` | `list[str]` | Keys whose live row is pinned: skipped (result version=0) rather than overwritten, since overwriting would also unpin (entries default pinned=False). | +| `skipped_locked` | `list[str]` | Keys whose live row is locked read_only: skipped (result version=0), where memory.set would answer 423 LOCKED. | +| `skipped_blacklisted` | `list[str]` | Keys whose entry the blacklist refused or could not check in the request's budget: not written (result version=0), where memory.set would answer 422 BLACKLISTED. | +| `merged` | `list[str]` | Conflict verdicts, the same four memory.set produces: canonical restatement of an existing row → not inserted, existing row reinforced; close to a pinned peer → not inserted; same fact in other words → inserted as status=duplicate… | +| `value_updates_linked` | `int` | Older rows marked superseded by a value update in this batch. | +| `rejected_pinned` | `list[str]` | Keys not inserted because they are too close to a pinned row. | +| `duplicates` | `list[str]` | Keys inserted as `status=duplicate`: the same fact as a live row in other words. | +| `conflicted` | `list[str]` | Keys inserted and cross-linked as `status=conflicted` with the rows they contradict. | +| `unchecked` | `int` | Entries past bulk_conflict_check_max_entries, written without a check. | +| `retyped` | `list[str]` | Entries declared procedural whose value is not a procedure body, stored as semantic tagged retyped_from:procedural. | +| `identity_duplicates` | `dict[str, str]` | Entries not written because a live row in the same identity already carries the same content: entry key -> the keeper's key. | +| `stage_ms` | `dict[str, float]` | Wall time per phase of the write (existing_lookup, entity_resolution, prepare, embed, verdicts, insert, value_updates, projections, total) — Diagnostic; readers tolerate unknown keys. | + ## `memory.apply` One scope grammar for every write that can touch more than one row. diff --git a/docs/reference/status.md b/docs/reference/status.md index 8071866f40..fc832313ad 100644 --- a/docs/reference/status.md +++ b/docs/reference/status.md @@ -25,7 +25,7 @@ serves is in the [MCP reference](./mcp.md#toolsets). -375 operations, 261 of them experimental. +376 operations, 262 of them experimental. | Group | Stable | Experimental | | --- | --- | --- | @@ -53,7 +53,7 @@ serves is in the [MCP reference](./mcp.md#toolsets). | `insights` | 0 | [`insights.catalog`](operations/insights.md#insightscatalog), [`insights.query`](operations/insights.md#insightsquery) | | `knowledge` | 5 | [`knowledge.bulk_delete`](operations/knowledge.md#knowledgebulk_delete), [`knowledge.dataset_approve`](operations/knowledge.md#knowledgedataset_approve), [`knowledge.dataset_get`](operations/knowledge.md#knowledgedataset_get), [`knowledge.dedup_list`](operations/knowledge.md#knowledgededup_list), [`knowledge.dedup_resolve`](operations/knowledge.md#knowledgededup_resolve), [`knowledge.delete`](operations/knowledge.md#knowledgedelete), [`knowledge.formats`](operations/knowledge.md#knowledgeformats), [`knowledge.gaps.draft`](operations/knowledge.md#knowledgegapsdraft), [`knowledge.gaps.list`](operations/knowledge.md#knowledgegapslist), [`knowledge.ingest`](operations/knowledge.md#knowledgeingest), [`knowledge.ingest_cancel`](operations/knowledge.md#knowledgeingest_cancel), [`knowledge.ingest_status`](operations/knowledge.md#knowledgeingest_status), [`knowledge.inspect`](operations/knowledge.md#knowledgeinspect), [`knowledge.move`](operations/knowledge.md#knowledgemove), [`knowledge.outline`](operations/knowledge.md#knowledgeoutline), [`knowledge.query`](operations/knowledge.md#knowledgequery), [`knowledge.query_history.add`](operations/knowledge.md#knowledgequery_historyadd), [`knowledge.query_history.list`](operations/knowledge.md#knowledgequery_historylist), [`knowledge.query_history.remove`](operations/knowledge.md#knowledgequery_historyremove), [`knowledge.reads`](operations/knowledge.md#knowledgereads), [`knowledge.restore`](operations/knowledge.md#knowledgerestore), [`knowledge.source.add`](operations/knowledge.md#knowledgesourceadd), [`knowledge.source.check`](operations/knowledge.md#knowledgesourcecheck), [`knowledge.source.inspect`](operations/knowledge.md#knowledgesourceinspect), [`knowledge.source.list`](operations/knowledge.md#knowledgesourcelist), [`knowledge.source.parse_links`](operations/knowledge.md#knowledgesourceparse_links), [`knowledge.source.push_configure`](operations/knowledge.md#knowledgesourcepush_configure), [`knowledge.source.push_confirm`](operations/knowledge.md#knowledgesourcepush_confirm), [`knowledge.source.push_status`](operations/knowledge.md#knowledgesourcepush_status), [`knowledge.source.remove`](operations/knowledge.md#knowledgesourceremove), [`knowledge.source.rotate`](operations/knowledge.md#knowledgesourcerotate), [`knowledge.source.sync_now`](operations/knowledge.md#knowledgesourcesync_now), [`knowledge.source.update`](operations/knowledge.md#knowledgesourceupdate), [`knowledge.synthesize`](operations/knowledge.md#knowledgesynthesize), [`knowledge.update_metadata`](operations/knowledge.md#knowledgeupdate_metadata), [`knowledge.upsert`](operations/knowledge.md#knowledgeupsert) | | `learnings` | 0 | [`learnings.list`](operations/learnings.md#learningslist), [`learnings.quality`](operations/learnings.md#learningsquality), [`learnings.review`](operations/learnings.md#learningsreview) | -| `memory` | 38 | [`memory.aggregate`](operations/memory.md#memoryaggregate), [`memory.apply`](operations/memory.md#memoryapply), [`memory.assign_category`](operations/memory.md#memoryassign_category), [`memory.auto_retirements`](operations/memory.md#memoryauto_retirements), [`memory.backlinks`](operations/memory.md#memorybacklinks), [`memory.copy_to`](operations/memory.md#memorycopy_to), [`memory.cross_promote`](operations/memory.md#memorycross_promote), [`memory.dedup_collapse`](operations/memory.md#memorydedup_collapse), [`memory.definitions.list`](operations/memory.md#memorydefinitionslist), [`memory.delete_version`](operations/memory.md#memorydelete_version), [`memory.demonstrate`](operations/memory.md#memorydemonstrate), [`memory.demote`](operations/memory.md#memorydemote), [`memory.exposure_summary`](operations/memory.md#memoryexposure_summary), [`memory.extraction_retry_mine`](operations/memory.md#memoryextraction_retry_mine), [`memory.extraction_status_mine`](operations/memory.md#memoryextraction_status_mine), [`memory.history`](operations/memory.md#memoryhistory), [`memory.import`](operations/memory.md#memoryimport), [`memory.import_curated`](operations/memory.md#memoryimport_curated), [`memory.inspect`](operations/memory.md#memoryinspect), [`memory.keys`](operations/memory.md#memorykeys), [`memory.lineage`](operations/memory.md#memorylineage), [`memory.lock`](operations/memory.md#memorylock), [`memory.outcome`](operations/memory.md#memoryoutcome), [`memory.outline`](operations/memory.md#memoryoutline), [`memory.owner_report`](operations/memory.md#memoryowner_report), [`memory.pin`](operations/memory.md#memorypin), [`memory.promote`](operations/memory.md#memorypromote), [`memory.proposal_decide`](operations/memory.md#memoryproposal_decide), [`memory.proposals`](operations/memory.md#memoryproposals), [`memory.record.conflicts`](operations/memory.md#memoryrecordconflicts), [`memory.redact`](operations/memory.md#memoryredact), [`memory.reindex`](operations/memory.md#memoryreindex), [`memory.reject`](operations/memory.md#memoryreject), [`memory.related`](operations/memory.md#memoryrelated), [`memory.stack`](operations/memory.md#memorystack), [`memory.statuses`](operations/memory.md#memorystatuses), [`memory.sweep_expired`](operations/memory.md#memorysweep_expired), [`memory.timeline`](operations/memory.md#memorytimeline), [`memory.touch`](operations/memory.md#memorytouch), [`memory.unlock`](operations/memory.md#memoryunlock), [`memory.unpin`](operations/memory.md#memoryunpin) | +| `memory` | 38 | [`memory.aggregate`](operations/memory.md#memoryaggregate), [`memory.append_batch`](operations/memory.md#memoryappend_batch), [`memory.apply`](operations/memory.md#memoryapply), [`memory.assign_category`](operations/memory.md#memoryassign_category), [`memory.auto_retirements`](operations/memory.md#memoryauto_retirements), [`memory.backlinks`](operations/memory.md#memorybacklinks), [`memory.copy_to`](operations/memory.md#memorycopy_to), [`memory.cross_promote`](operations/memory.md#memorycross_promote), [`memory.dedup_collapse`](operations/memory.md#memorydedup_collapse), [`memory.definitions.list`](operations/memory.md#memorydefinitionslist), [`memory.delete_version`](operations/memory.md#memorydelete_version), [`memory.demonstrate`](operations/memory.md#memorydemonstrate), [`memory.demote`](operations/memory.md#memorydemote), [`memory.exposure_summary`](operations/memory.md#memoryexposure_summary), [`memory.extraction_retry_mine`](operations/memory.md#memoryextraction_retry_mine), [`memory.extraction_status_mine`](operations/memory.md#memoryextraction_status_mine), [`memory.history`](operations/memory.md#memoryhistory), [`memory.import`](operations/memory.md#memoryimport), [`memory.import_curated`](operations/memory.md#memoryimport_curated), [`memory.inspect`](operations/memory.md#memoryinspect), [`memory.keys`](operations/memory.md#memorykeys), [`memory.lineage`](operations/memory.md#memorylineage), [`memory.lock`](operations/memory.md#memorylock), [`memory.outcome`](operations/memory.md#memoryoutcome), [`memory.outline`](operations/memory.md#memoryoutline), [`memory.owner_report`](operations/memory.md#memoryowner_report), [`memory.pin`](operations/memory.md#memorypin), [`memory.promote`](operations/memory.md#memorypromote), [`memory.proposal_decide`](operations/memory.md#memoryproposal_decide), [`memory.proposals`](operations/memory.md#memoryproposals), [`memory.record.conflicts`](operations/memory.md#memoryrecordconflicts), [`memory.redact`](operations/memory.md#memoryredact), [`memory.reindex`](operations/memory.md#memoryreindex), [`memory.reject`](operations/memory.md#memoryreject), [`memory.related`](operations/memory.md#memoryrelated), [`memory.stack`](operations/memory.md#memorystack), [`memory.statuses`](operations/memory.md#memorystatuses), [`memory.sweep_expired`](operations/memory.md#memorysweep_expired), [`memory.timeline`](operations/memory.md#memorytimeline), [`memory.touch`](operations/memory.md#memorytouch), [`memory.unlock`](operations/memory.md#memoryunlock), [`memory.unpin`](operations/memory.md#memoryunpin) | | `memory_schema` | 1 | [`memory_schema.approve`](operations/memory_schema.md#memory_schemaapprove), [`memory_schema.list`](operations/memory_schema.md#memory_schemalist), [`memory_schema.propose`](operations/memory_schema.md#memory_schemapropose) | | `mentor` | 1 | [`mentor.guide`](operations/mentor.md#mentorguide) | | `namespace` | 9 | [`namespace.cancel_request`](operations/namespace.md#namespacecancel_request), [`namespace.delete`](operations/namespace.md#namespacedelete), [`namespace.guide_get`](operations/namespace.md#namespaceguide_get), [`namespace.guide_history`](operations/namespace.md#namespaceguide_history), [`namespace.guide_set`](operations/namespace.md#namespaceguide_set), [`namespace.inventory`](operations/namespace.md#namespaceinventory), [`namespace.join`](operations/namespace.md#namespacejoin), [`namespace.leave`](operations/namespace.md#namespaceleave), [`namespace.list`](operations/namespace.md#namespacelist), [`namespace.list_categories`](operations/namespace.md#namespacelist_categories), [`namespace.list_ontology`](operations/namespace.md#namespacelist_ontology), [`namespace.mark_seen`](operations/namespace.md#namespacemark_seen), [`namespace.register_category`](operations/namespace.md#namespaceregister_category), [`namespace.restore`](operations/namespace.md#namespacerestore), [`namespace.rollback`](operations/namespace.md#namespacerollback), [`namespace.search_categories`](operations/namespace.md#namespacesearch_categories), [`namespace.snapshot`](operations/namespace.md#namespacesnapshot), [`namespace.tune`](operations/namespace.md#namespacetune) | diff --git a/sdk/python/loom/client.py b/sdk/python/loom/client.py index 530df58b3e..6c63be462f 100644 --- a/sdk/python/loom/client.py +++ b/sdk/python/loom/client.py @@ -1719,6 +1719,15 @@ async def aggregate( await self._agent.call("memory.aggregate", body, **options), ) + async def append_batch( + self, body: _ops.MemoryAppendBatchBody | None = None, /, **options: Any + ) -> _ops.MemoryBulkSetOutput: + """`memory.append_batch`.""" + return cast( + "_ops.MemoryBulkSetOutput", + await self._agent.call("memory.append_batch", body, **options), + ) + async def apply( self, body: _ops.MemoryApplyBody | None = None, /, **options: Any ) -> _ops.MemoryApplyOutput: diff --git a/sdk/python/loom/integrations/openai_agents.py b/sdk/python/loom/integrations/openai_agents.py index 9a88a58019..d284eab660 100644 --- a/sdk/python/loom/integrations/openai_agents.py +++ b/sdk/python/loom/integrations/openai_agents.py @@ -21,8 +21,11 @@ The tools are ``loom.tools.AgentTools`` (the same names, schemas and results as ``agent.tools()``), bound to one user, session or run. Identity and -isolation come from the Loom agent. Install with -``pip install 'clickhouse-loom[openai-agents]'``. +isolation come from the Loom agent. From the root of the matching Loom +checkout, install with ``pip install './sdk/python[openai-agents]'``. +Native sessions require a server from the same checkout, using the Postgres +memory authority and advertising ``memory.append_batch``. See the native +session guide in ``docs/features/integrations/frameworks.md``. """ from __future__ import annotations @@ -119,6 +122,10 @@ class LoomSession: TypeScript sessions have a separate history because their item formats differ. The mapping is in ``docs/features/integrations/frameworks.md`` (OpenAI Agents SDK). + Additions require a server serving ``memory.append_batch`` on the Postgres + memory authority. Each nonempty addition is one batch of at most 1,000 items; + an unsupported server refuses instead of writing part of the history. + Bind the application's existing protection family before construction. The session retains constructor and callback contributors across client rebinding; native user/session ids only route history. Native reads and additions are @@ -205,20 +212,26 @@ async def get_items(self, limit: int | None = None) -> list[TResponseInputItem]: async def add_items(self, items: list[TResponseInputItem]) -> None: global _last_stamp with self._protection.automatic(self._agent.transport.session_protection): - for item in items: + if not items: + return + if len(items) > 1000: + raise ValueError("A session addition takes at most 1,000 items") + # Encode the entire batch before dispatch: a malformed later item writes nothing. + values = [json.dumps(item) for item in items] + if not (await self._agent.capabilities()).has("memory.append_batch"): + raise LoomError("Session additions require memory.append_batch on the server") + entries = [] + for value in values: # The clock or one past this process's last stamp, never less: keys keep order. _last_stamp = max(time.time_ns(), _last_stamp + 1) - # One memory.set per item: a batch folds a value identical to a live row. - await self._agent.call( - "memory.set", + entries.append( { "memory_key": f"{self._prefix}{_last_stamp:020d}-{secrets.token_hex(4)}", - "value": json.dumps(item), - "memory_type": "episodic", - }, - **self._scope(), + "value": value, + } ) - require_automatic_allowed(self._agent.transport.session_protection) + await self._agent.call("memory.append_batch", {"entries": entries}, **self._scope()) + require_automatic_allowed(self._agent.transport.session_protection) async def pop_item(self) -> TResponseInputItem | None: with self._protection.observing(self._agent.transport.session_protection): diff --git a/sdk/python/loom/ops.py b/sdk/python/loom/ops.py index fd1913dc09..92ffb26dbb 100644 --- a/sdk/python/loom/ops.py +++ b/sdk/python/loom/ops.py @@ -212,6 +212,7 @@ class OpSpec: "learnings.review", "memory.agents_list", "memory.aggregate", + "memory.append_batch", "memory.apply", "memory.ask", "memory.assign_category", @@ -590,6 +591,7 @@ class OpSpec: "learnings.review", "memory.agents_list", "memory.aggregate", + "memory.append_batch", "memory.apply", "memory.ask", "memory.assign_category", @@ -2871,6 +2873,18 @@ class OpSpec: False, True, ), + "memory.append_batch": OpSpec( + "memory.append_batch", + "MemoryAppendBatchInput", + "MemoryAppendBatchBody", + "MemoryBulkSetOutput", + True, + "writer", + "experimental", + 300, + False, + False, + ), "memory.apply": OpSpec( "memory.apply", "MemoryApplyInput", @@ -12400,6 +12414,73 @@ class AggregateRow(TypedDict, total=False): metric: Required[float] +class MemoryAppendBatchInput(TypedDict, total=False): + "Append distinct episodic occurrences atomically on the Postgres authority." + + org: Required[str] + namespace: Required[str] + agent: Required[str] + user_id: NotRequired[str] + session_id: NotRequired[str] + run_id: NotRequired[str] + entries: Required[list["AppendBatchEntry"]] + + +class AppendBatchEntry(TypedDict, total=False): + memory_key: Required[str] + value: Required[Union[dict[str, Any], list[Any], str, int, float, bool]] + source_trace: NotRequired[str] + source_class: NotRequired[Optional[str]] + + +class MemoryAppendBatchBody(TypedDict, total=False): + "Append distinct episodic occurrences atomically on the Postgres authority." + + org: NotRequired[str] + namespace: NotRequired[str] + agent: NotRequired[str] + user_id: NotRequired[str] + session_id: NotRequired[str] + run_id: NotRequired[str] + entries: Required[list["AppendBatchEntry"]] + + +class MemoryBulkSetOutput(TypedDict, total=False): + written: Required[list["BulkSetResult"]] + skipped_pinned: NotRequired[list[str]] + skipped_locked: NotRequired[list[str]] + skipped_blacklisted: NotRequired[list[str]] + merged: NotRequired[list[str]] + value_updates_linked: NotRequired[int] + rejected_pinned: NotRequired[list[str]] + duplicates: NotRequired[list[str]] + conflicted: NotRequired[list[str]] + unchecked: NotRequired[int] + retyped: NotRequired[list[str]] + identity_duplicates: NotRequired[dict[str, str]] + stage_ms: NotRequired[dict[str, float]] + + +class BulkSetResult(TypedDict, total=False): + memory_key: Required[str] + version: Required[int] + memory_id: NotRequired[str] + conflict_verdict: NotRequired[str] + peers: NotRequired[list["MemorySetPeer"]] + next: NotRequired[str] + pinned_blocker_id: NotRequired[str] + + +class MemorySetPeer(TypedDict, total=False): + "A row the conflict check compared the write against." + + memory_id: NotRequired[str] + memory_key: NotRequired[str] + excerpt: NotRequired[str] + valid_at: NotRequired[Optional[str]] + similarity: NotRequired[float] + + class MemoryApplyInput(TypedDict, total=False): "One scope grammar for every write that can touch more than one row." @@ -13105,42 +13186,6 @@ class MemoryBulkSetBody(TypedDict, total=False): detect_updates: NotRequired[Optional[bool]] -class MemoryBulkSetOutput(TypedDict, total=False): - written: Required[list["BulkSetResult"]] - skipped_pinned: NotRequired[list[str]] - skipped_locked: NotRequired[list[str]] - skipped_blacklisted: NotRequired[list[str]] - merged: NotRequired[list[str]] - value_updates_linked: NotRequired[int] - rejected_pinned: NotRequired[list[str]] - duplicates: NotRequired[list[str]] - conflicted: NotRequired[list[str]] - unchecked: NotRequired[int] - retyped: NotRequired[list[str]] - identity_duplicates: NotRequired[dict[str, str]] - stage_ms: NotRequired[dict[str, float]] - - -class BulkSetResult(TypedDict, total=False): - memory_key: Required[str] - version: Required[int] - memory_id: NotRequired[str] - conflict_verdict: NotRequired[str] - peers: NotRequired[list["MemorySetPeer"]] - next: NotRequired[str] - pinned_blocker_id: NotRequired[str] - - -class MemorySetPeer(TypedDict, total=False): - "A row the conflict check compared the write against." - - memory_id: NotRequired[str] - memory_key: NotRequired[str] - excerpt: NotRequired[str] - valid_at: NotRequired[Optional[str]] - similarity: NotRequired[float] - - class MemoryClearInput(TypedDict, total=False): "Bulk-tombstone memories in one scope." diff --git a/sdk/python/loom/run.py b/sdk/python/loom/run.py index 56d7b73904..3c9a0406a4 100644 --- a/sdk/python/loom/run.py +++ b/sdk/python/loom/run.py @@ -64,6 +64,7 @@ def credit_answer(result: Mapping[str, Any]) -> None: "/v1/memory.pin", "/v1/memory.update", "/v1/memory.set_from_messages", + "/v1/memory.append_batch", "/v1/memory.bulk_set", "/v1/memory.link", "/v1/observe.record", diff --git a/sdk/python/tests/test_openai_agents.py b/sdk/python/tests/test_openai_agents.py index 93541d8ca3..68e21a355a 100644 --- a/sdk/python/tests/test_openai_agents.py +++ b/sdk/python/tests/test_openai_agents.py @@ -18,6 +18,7 @@ from agents import FunctionTool from agents.tool_context import ToolContext from loom import Agent +from loom.capabilities import Capabilities from loom.errors import LoomError from loom.integrations.openai_agents import LoomSession, loom_tools from loom.session_protection import AutomaticCaptureSuppressed, SessionProtection @@ -218,11 +219,14 @@ def __init__(self, rows: _Rows) -> None: self.memory = rows self.transport = SimpleNamespace(session_protection=None) + async def capabilities(self) -> Capabilities: + return Capabilities.from_response({"ops": [{"name": "memory.append_batch"}]}, None) + async def call(self, op: str, body: dict[str, Any], **scope: Any) -> dict[str, Any]: - assert op == "memory.set" - return await self.memory.set( - body["memory_key"], body["value"], body["memory_type"], **scope - ) + assert op == "memory.append_batch" + for entry in body["entries"]: + await self.memory.set(entry["memory_key"], entry["value"], "episodic", **scope) + return {} def _session(rows: _Rows, session_id: str = "s1", **kw: Any) -> Any: @@ -417,12 +421,14 @@ async def get(self, key: str, **kw: Any) -> None: asyncio.run(getattr(session, method)()) -def test_concurrent_repeats_use_point_writes_with_default_sdk_coalescing() -> None: +def test_concurrent_repeats_use_atomic_batches_with_default_sdk_coalescing() -> None: from loom.integrations.openai_agents import LoomSession posts: list[tuple[str, dict[str, Any]]] = [] async def post(path: str, body: dict[str, Any], *args: Any, **kwargs: Any) -> dict[str, Any]: + if path == "/v1/ops.capabilities": + return {"ops": [{"name": "memory.append_batch"}]} posts.append((path, dict(body))) return {} @@ -437,9 +443,10 @@ async def append() -> None: assert agent.coalesce_sets asyncio.run(append()) - assert [path for path, _ in posts] == ["/v1/memory.set"] * 4 - assert len({body["memory_key"] for _, body in posts}) == 4 - assert {body["value"] for _, body in posts} == {json.dumps(_msg("user", "again"))} + assert [path for path, _ in posts] == ["/v1/memory.append_batch"] * 4 + assert all(len(body["entries"]) == 1 for _, body in posts) + assert len({body["entries"][0]["memory_key"] for _, body in posts}) == 4 + assert {body["entries"][0]["value"] for _, body in posts} == {json.dumps(_msg("user", "again"))} def test_a_nested_key_prefix_is_not_read_as_this_sessions_items() -> None: @@ -508,6 +515,7 @@ def _protected_session_agent( retry={"max_attempts": 1}, ) agent.transport._client._transport = httpx.MockTransport(respond) + agent._caps = Capabilities.from_response({"ops": [{"name": "memory.append_batch"}]}, None) return agent @@ -611,11 +619,13 @@ async def go() -> None: asyncio.run(go()) -def test_marked_native_history_write_stops_before_the_next_item(tmp_path: Path) -> None: +def test_marked_native_batch_response_protects_the_whole_callback(tmp_path: Path) -> None: family = SessionProtection.create(base=tmp_path / "family") writes: list[dict[str, Any]] = [] def respond(request: httpx.Request) -> httpx.Response: + if request.url.path == "/v1/ops.capabilities": + return httpx.Response(200, json={"ops": [{"name": "memory.append_batch"}]}) writes.append(json.loads(request.content)) return httpx.Response(200, json={"mirrored": True}) @@ -625,7 +635,11 @@ async def go() -> None: session = LoomSession(agent, "native-history") with pytest.raises(AutomaticCaptureSuppressed): await session.add_items([_msg("user", "first"), _msg("assistant", "second")]) - assert len(writes) == 1 and json.loads(writes[0]["value"]) == _msg("user", "first") + assert len(writes) == 1 + assert [json.loads(entry["value"]) for entry in writes[0]["entries"]] == [ + _msg("user", "first"), + _msg("assistant", "second"), + ] assert family.protected finally: await agent.transport.close() @@ -719,3 +733,129 @@ async def go() -> None: await agent.transport.close() asyncio.run(go()) + + +# The transport tests keep serialization, discovery and mutation failures separate. +def test_a_later_unencodable_session_item_dispatches_nothing(tmp_path: Path) -> None: + posts: list[str] = [] + + def respond(request: httpx.Request) -> httpx.Response: + posts.append(request.url.path) + return httpx.Response(200, json={}) + + async def go() -> None: + family = SessionProtection.create(base=tmp_path / "family") + agent = _protected_session_agent(family, respond) + try: + with pytest.raises(TypeError): + await LoomSession(agent, "s").add_items( + [_msg("user", "first"), {"role": "user", "content": object()}] # type: ignore[list-item] + ) + assert posts == [] + finally: + await agent.transport.close() + + asyncio.run(go()) + + +def test_an_empty_session_addition_needs_no_discovery_or_mutation() -> None: + class EmptyOnly(_Memory): + async def capabilities(self) -> Capabilities: + raise AssertionError("empty additions need no discovery") + + async def call(self, *_a: Any, **_kw: Any) -> dict[str, Any]: + raise AssertionError("empty additions need no mutation") + + asyncio.run(LoomSession(EmptyOnly(_Rows()), "s").add_items([])) # type: ignore[arg-type] + + +@pytest.mark.parametrize("failure", ["discovery", "absent", "old_replica", "refused", "committed"]) +def test_session_batch_failures_never_fall_back_or_change_keys( + tmp_path: Path, failure: str +) -> None: + posts: list[tuple[str, dict[str, Any]]] = [] + + def respond(request: httpx.Request) -> httpx.Response: + posts.append((request.url.path, json.loads(request.content))) + if request.url.path == "/v1/ops.capabilities": + if failure == "discovery": + return httpx.Response(503, json={"detail": {"code": "BACKEND_UNAVAILABLE"}}) + return httpx.Response(200, json={"ops": []}) + assert request.url.path == "/v1/memory.append_batch" + if failure == "old_replica": + return httpx.Response(404, json={"detail": "Not Found"}) + code = "PUBLICATION_PENDING" if failure == "committed" else "VALIDATION_FAILED" + return httpx.Response( + 503 if failure == "committed" else 422, + json={ + "detail": { + "code": code, + "message": "batch refused", + "committed": failure == "committed", + } + }, + ) + + async def go() -> None: + family = SessionProtection.create(base=tmp_path / "family") + agent = _protected_session_agent(family, respond) + if failure in {"discovery", "absent"}: + agent._caps = None + try: + with pytest.raises(LoomError) as raised: + await LoomSession(agent, "s").add_items( + [_msg("user", "repeated"), _msg("user", "repeated")] + ) + assert not family.protected + assert len(posts) == 1 + if failure in {"discovery", "absent"}: + assert posts[0][0] == "/v1/ops.capabilities" + else: + entries = posts[0][1]["entries"] + assert len(entries) == 2 + assert entries[0]["memory_key"] != entries[1]["memory_key"] + assert entries[0]["value"] == entries[1]["value"] + if failure == "committed": + assert raised.value.code == "PUBLICATION_PENDING" + assert raised.value.detail["committed"] is True + finally: + await agent.transport.close() + + asyncio.run(go()) + + +def test_a_marked_discovery_reply_stops_before_sending_history(tmp_path: Path) -> None: + posts: list[str] = [] + family = SessionProtection.create(base=tmp_path / "family") + + def respond(request: httpx.Request) -> httpx.Response: + posts.append(request.url.path) + return httpx.Response( + 200, + json={ + "mirrored": True, + "ops": [{"name": "memory.append_batch"}], + }, + ) + + async def go() -> None: + agent = _protected_session_agent(family, respond) + agent._caps = None + try: + with pytest.raises(AutomaticCaptureSuppressed): + await LoomSession(agent, "s").add_items([_msg("user", "private")]) + assert posts == ["/v1/ops.capabilities"] and family.protected + finally: + await agent.transport.close() + + asyncio.run(go()) + + +def test_an_oversized_session_addition_is_not_split_or_dispatched() -> None: + class TooLarge(_Memory): + async def capabilities(self) -> Capabilities: + raise AssertionError("oversized additions need no discovery") + + session = LoomSession(TooLarge(_Rows()), "s") # type: ignore[arg-type] + with pytest.raises(ValueError, match="at most 1,000"): + asyncio.run(session.add_items([_msg("user", "again")] * 1001)) diff --git a/sdk/typescript/src/generated/ops.ts b/sdk/typescript/src/generated/ops.ts index ff302c5c8a..2de96f92ad 100644 --- a/sdk/typescript/src/generated/ops.ts +++ b/sdk/typescript/src/generated/ops.ts @@ -4777,6 +4777,57 @@ export interface MemoryAggregateOutput { sql_executed?: string; } +export interface AppendBatchEntry { + memory_key: string; + value: Record | unknown[] | string | number | boolean; + source_trace?: string; + source_class?: string | null; +} + +export interface MemoryAppendBatchInput { + org: string; + namespace: string; + agent: string; + user_id?: string; + session_id?: string; + run_id?: string; + entries: AppendBatchEntry[]; +} + +export interface BulkSetResult { + memory_key: string; + version: number; + memory_id?: string; + conflict_verdict?: string; + peers?: MemorySetPeer[]; + next?: string; + pinned_blocker_id?: string; +} + +export interface MemorySetPeer { + memory_id?: string; + memory_key?: string; + excerpt?: string; + valid_at?: string | null; + similarity?: number; +} + +export interface MemoryBulkSetOutput { + written: BulkSetResult[]; + skipped_pinned?: string[]; + skipped_locked?: string[]; + skipped_blacklisted?: string[]; + merged?: string[]; + value_updates_linked?: number; + rejected_pinned?: string[]; + duplicates?: string[]; + conflicted?: string[]; + unchecked?: number; + retyped?: string[]; + identity_duplicates?: Record; + stage_ms?: Record; +} + export interface ApplyTarget { keys?: string[] | null; filter?: ApplyFilter | null; @@ -5286,40 +5337,6 @@ export interface MemoryBulkSetInput { detect_updates?: boolean | null; } -export interface BulkSetResult { - memory_key: string; - version: number; - memory_id?: string; - conflict_verdict?: string; - peers?: MemorySetPeer[]; - next?: string; - pinned_blocker_id?: string; -} - -export interface MemorySetPeer { - memory_id?: string; - memory_key?: string; - excerpt?: string; - valid_at?: string | null; - similarity?: number; -} - -export interface MemoryBulkSetOutput { - written: BulkSetResult[]; - skipped_pinned?: string[]; - skipped_locked?: string[]; - skipped_blacklisted?: string[]; - merged?: string[]; - value_updates_linked?: number; - rejected_pinned?: string[]; - duplicates?: string[]; - conflicted?: string[]; - unchecked?: number; - retyped?: string[]; - identity_duplicates?: Record; - stage_ms?: Record; -} - export interface MemoryClearInput { org: string; namespace: string; @@ -10645,6 +10662,7 @@ export type OpName = | "learnings.review" | "memory.agents_list" | "memory.aggregate" + | "memory.append_batch" | "memory.apply" | "memory.ask" | "memory.assign_category" @@ -11023,6 +11041,7 @@ export interface OpMap { "learnings.review": { input: LearningsReviewInput; output: LearningsReviewOutput }; "memory.agents_list": { input: MemoryAgentsListInput; output: MemoryAgentsListOutput }; "memory.aggregate": { input: MemoryAggregateInput; output: MemoryAggregateOutput }; + "memory.append_batch": { input: MemoryAppendBatchInput; output: MemoryBulkSetOutput }; "memory.apply": { input: MemoryApplyInput; output: MemoryApplyOutput }; "memory.ask": { input: MemoryAskInput; output: MemoryAskOutput }; "memory.assign_category": { input: MemoryAssignCategoryInput; output: MemoryAssignCategoryOutput }; @@ -11401,6 +11420,7 @@ export const OPS: readonly OpSpec[] = [ { name: "learnings.review", group: "learnings", method: "review", idempotent: true, role: "admin", maturity: "experimental", budgetP99Ms: 150, audit: true, readOnly: false }, { name: "memory.agents_list", group: "memory", method: "agentsList", idempotent: true, role: "viewer", maturity: "stable", budgetP99Ms: 80, audit: false, readOnly: true }, { name: "memory.aggregate", group: "memory", method: "aggregate", idempotent: true, role: "viewer", maturity: "experimental", budgetP99Ms: 250, audit: false, readOnly: true }, + { name: "memory.append_batch", group: "memory", method: "appendBatch", idempotent: true, role: "writer", maturity: "experimental", budgetP99Ms: 300, audit: false, readOnly: false }, { name: "memory.apply", group: "memory", method: "apply", idempotent: false, role: "writer", maturity: "experimental", budgetP99Ms: 3000, audit: true, readOnly: false }, { name: "memory.ask", group: "memory", method: "ask", idempotent: true, role: "viewer", maturity: "stable", budgetP99Ms: 8000, audit: false, readOnly: true }, { name: "memory.assign_category", group: "memory", method: "assignCategory", idempotent: true, role: "writer", maturity: "experimental", budgetP99Ms: 60, audit: true, readOnly: false }, @@ -11609,4 +11629,4 @@ export const OP_BY_NAME: Readonly> = Object.fromEntries( OPS.map((o) => [o.name, o]), ) as Readonly>; -export const GENERATED_OP_COUNT = 375; +export const GENERATED_OP_COUNT = 376; diff --git a/sdk/typescript/src/integrations/openai-agents.test.ts b/sdk/typescript/src/integrations/openai-agents.test.ts index 706329ffa0..8db690cf42 100644 --- a/sdk/typescript/src/integrations/openai-agents.test.ts +++ b/sdk/typescript/src/integrations/openai-agents.test.ts @@ -65,11 +65,16 @@ describe("loomTools (OpenAI Agents SDK adapter)", () => { /** A Loom server keeping memory rows by (user, session, key), listed as the server lists * them: the newest second first, by key within a second, values cut to 2,048 characters. */ +const appendCapabilities = { + ops: [{ name: "memory.append_batch", required_role: "writer", request_fields: ["entries"] }], +}; + function rowsHarness(unserved = 0, cap = 100) { const rows = new Map(); const hidden = new Set(); const disappear = new Set(); const clock = { second: 0 }; + const behavior = { serveAppend: true, discoveryStatus: 200, appendStatus: 200, appendCode: "", blockedContent: "" }; const calls: Array<{ op: string; body: Record }> = []; const fetch = async (url: string, init: RequestInit) => { const op = url.split("/v1/")[1]; @@ -77,6 +82,38 @@ function rowsHarness(unserved = 0, cap = 100) { calls.push({ op, body }); const id = (key: string) => JSON.stringify([body.user_id, body.session_id, key]); let reply: unknown = {}; + if (op === "ops.capabilities") { + return new Response(JSON.stringify(behavior.serveAppend ? appendCapabilities : { ops: [] }), { + status: behavior.discoveryStatus, + }); + } + if (op === "memory.append_batch") { + const entries = body.entries as Array<{ memory_key: string; value: string }>; + const staged = entries.map((entry) => ({ + key: entry.memory_key, + value: JSON.parse(entry.value), + user: body.user_id, + session: body.session_id, + second: clock.second, + })); + if ( + (behavior.blockedContent && staged.some((row) => row.value.content === behavior.blockedContent)) || + (behavior.appendStatus !== 200 && behavior.appendCode !== "PUBLICATION_PENDING") + ) { + return new Response(JSON.stringify({ detail: { code: behavior.appendCode, message: "batch refused" } }), { + status: behavior.appendStatus === 200 ? 422 : behavior.appendStatus, + }); + } + for (const row of staged) rows.set(id(row.key), row); + return new Response( + JSON.stringify( + behavior.appendCode === "PUBLICATION_PENDING" + ? { detail: { code: "PUBLICATION_PENDING", message: "committed; publication pending", committed: true } } + : { written: staged.map((row) => ({ memory_key: row.key, version: 1 })) }, + ), + { status: behavior.appendStatus }, + ); + } if (op === "memory.set") { rows.set(id(body.memory_key), { key: body.memory_key, @@ -119,6 +156,7 @@ function rowsHarness(unserved = 0, cap = 100) { rows, calls, clock, + behavior, hidden, disappear, }; @@ -157,7 +195,7 @@ function protectedHistoryHarness( const op = url.split("/v1/")[1]; const body = JSON.parse(init.body as string) as Record; calls.push({ op, body }); - return new Response(JSON.stringify(reply(op, body)), { + return new Response(JSON.stringify(op === "ops.capabilities" ? appendCapabilities : reply(op, body)), { headers: { "content-type": "application/json" }, }); }, @@ -231,14 +269,33 @@ describe("LoomSession conversation protection", () => { expect(second.state.value).toBe("protected"); }); - it("stops the next native history write after the first marked acknowledgement", async () => { + it("observes a marked discovery response before any append", async () => { + const family = historyFamily(); + const calls: string[] = []; + const client = new LoomClient({ + baseUrl: "http://inert.invalid", + sessionProtection: family.root, + fetch: async (url) => { + calls.push(url.split("/v1/")[1]); + return new Response(JSON.stringify({ ...appendCapabilities, mirrored: true })); + }, + logLevel: "off", + }); + const session = new LoomSession(new Agent("o", "n", "bot", client), "s1"); + await expect(session.addItems([msg("user", "private")])).rejects.toBeInstanceOf(AutomaticCaptureSuppressed); + expect(calls).toEqual(["ops.capabilities"]); + expect(family.state.value).toBe("protected"); + }); + + it("refuses a marked batch acknowledgement without starting another write", async () => { const family = historyFamily(); const { agent, calls } = protectedHistoryHarness(family.root, () => ({ mirrored: true })); const session = new LoomSession(agent, "s1"); await expect(session.addItems([msg("user", "first"), msg("assistant", "second")])).rejects.toBeInstanceOf( AutomaticCaptureSuppressed, ); - expect(calls.map((call) => call.op)).toEqual(["memory.set"]); + expect(calls.map((call) => call.op)).toEqual(["ops.capabilities", "memory.append_batch"]); + expect(calls[1].body.entries).toHaveLength(2); expect(family.state.value).toBe("protected"); }); @@ -270,17 +327,22 @@ describe("LoomSession conversation protection", () => { expect(family.state.value).toBe("protected"); }); - it("rechecks the retained native history family after credentials mark and rebind it", async () => { - const first = historyFamily(); - const current = protectedHistoryHarness(historyFamily().root); - const original = protectedHistoryHarness(first.root, undefined, async () => { - await first.root.mark(); - Object.defineProperty(original.agent, "client", { value: current.client }); - return "token"; - }); - await expect(new LoomSession(original.agent, "s1").getItems()).rejects.toBeInstanceOf(AutomaticCaptureSuppressed); - expect([...original.calls, ...current.calls]).toEqual([]); - }); + it.each(["getItems", "addItems"] as const)( + "rechecks the retained native %s family after credentials rebind it", + async (method) => { + const first = historyFamily(); + const current = protectedHistoryHarness(historyFamily().root); + const original = protectedHistoryHarness(first.root, undefined, async () => { + await first.root.mark(); + Object.defineProperty(original.agent, "client", { value: current.client }); + return "token"; + }); + const session = new LoomSession(original.agent, "s1"); + const call = method === "getItems" ? session.getItems() : session.addItems([msg("user", "private")]); + await expect(call).rejects.toBeInstanceOf(AutomaticCaptureSuppressed); + expect([...original.calls, ...current.calls]).toEqual([]); + }, + ); it.each(["popItem", "clearSession"] as const)( "keeps explicit native %s usable and observes the retained family", @@ -310,6 +372,83 @@ describe("LoomSession conversation protection", () => { }); describe("LoomSession (OpenAI Agents SDK session)", () => { + it("does not discover or mutate for an empty append", async () => { + const { agent, calls } = rowsHarness(); + await new LoomSession(agent, "s1").addItems([]); + expect(calls).toEqual([]); + }); + + it("encodes every item before discovery so a malformed later item writes nothing", async () => { + const { agent, calls, rows } = rowsHarness(); + const cyclic = { role: "user", content: "bad", providerData: {} }; + cyclic.providerData = cyclic; + await expect( + new LoomSession(agent, "s1").addItems([msg("user", "valid"), cyclic as AgentInputItem]), + ).rejects.toThrow(); + expect(calls).toEqual([]); + expect(rows.size).toBe(0); + }); + + it("refuses an oversized logical append without chunking or discovery", async () => { + const { agent, calls, rows } = rowsHarness(); + await expect( + new LoomSession(agent, "s1").addItems(Array.from({ length: 1001 }, () => msg("user", "same"))), + ).rejects.toThrow("at most 1000"); + expect(calls).toEqual([]); + expect(rows.size).toBe(0); + }); + + it("preserves existing history when the second entry makes the batch refuse", async () => { + const { agent, calls, rows, behavior } = rowsHarness(); + const session = new LoomSession(agent, "s1"); + await session.addItems([msg("user", "old")]); + const before = [...rows.values()]; + calls.length = 0; + behavior.blockedContent = "blocked"; + await expect(session.addItems([msg("user", "valid"), msg("user", "blocked")])).rejects.toThrow("batch refused"); + expect([...rows.values()]).toEqual(before); + expect(calls.map((call) => call.op)).toEqual(["ops.capabilities", "memory.append_batch"]); + expect(await session.getItems()).toEqual([msg("user", "old")]); + }); + + it("refuses a missing capability without attempting a point or ordinary bulk write", async () => { + const { agent, calls, rows, behavior } = rowsHarness(); + behavior.serveAppend = false; + await expect(new LoomSession(agent, "s1").addItems([msg("user", "new")])).rejects.toThrow("memory.append_batch"); + expect(calls.map((call) => call.op)).toEqual(["ops.capabilities"]); + expect(rows.size).toBe(0); + }); + + it("propagates failed discovery without writing history", async () => { + const { agent, calls, rows, behavior } = rowsHarness(); + behavior.discoveryStatus = 403; + await expect(new LoomSession(agent, "s1").addItems([msg("user", "new")])).rejects.toMatchObject({ status: 403 }); + expect(calls.map((call) => call.op)).toEqual(["ops.capabilities"]); + expect(rows.size).toBe(0); + }); + + it.each([404, 422])("does not downgrade when the append target refuses with %s", async (status) => { + const { agent, calls, rows, behavior } = rowsHarness(); + behavior.appendStatus = status; + await expect(new LoomSession(agent, "s1").addItems([msg("user", "new")])).rejects.toMatchObject({ status }); + expect(calls.map((call) => call.op)).toEqual(["ops.capabilities", "memory.append_batch"]); + expect(rows.size).toBe(0); + }); + + it("surfaces committed publication-pending without resending or deleting the append", async () => { + const { agent, calls, rows, behavior } = rowsHarness(); + behavior.appendStatus = 503; + behavior.appendCode = "PUBLICATION_PENDING"; + const session = new LoomSession(agent, "s1"); + await expect(session.addItems([msg("user", "a"), msg("user", "b")])).rejects.toMatchObject({ + code: "PUBLICATION_PENDING", + detail: { committed: true }, + }); + expect(rows.size).toBe(2); + expect(calls.map((call) => call.op)).toEqual(["ops.capabilities", "memory.append_batch"]); + expect(await session.getItems()).toEqual([msg("user", "a"), msg("user", "b")]); + }); + it.each(["before", "after"])( "keeps the validated user when caller options change %s the first write", async (when) => { @@ -319,7 +458,7 @@ describe("LoomSession (OpenAI Agents SDK session)", () => { if (when === "after") await session.addItems([msg("user", "first")]); options.userId = when === "before" ? "-" : "bob"; await session.addItems([msg("user", "second")]); - expect(calls.filter((call) => call.op === "memory.set").map((call) => call.body.user_id)).toEqual( + expect(calls.filter((call) => call.op === "memory.append_batch").map((call) => call.body.user_id)).toEqual( when === "before" ? ["alice"] : ["alice", "alice"], ); expect((await session.getItems()).length).toBe(when === "before" ? 1 : 2); @@ -404,17 +543,27 @@ describe("LoomSession (OpenAI Agents SDK session)", () => { msg("assistant", "again"), ]); expect(await session.getItems(2)).toEqual([msg("user", "hi"), msg("assistant", "again")]); - const sets = calls.filter((c) => c.op === "memory.set").map((c) => c.body); - expect(new Set(sets.map((b) => b.memory_type))).toEqual(new Set(["episodic"])); - expect(new Set(sets.map((b) => b.user_id))).toEqual(new Set(["alice"])); - expect(String(sets[0].session_id)).toMatch(/^oai-ts-[0-9a-f]{20}$/); + const appends = calls.filter((c) => c.op === "memory.append_batch").map((c) => c.body); + expect(appends.map((b) => (b.entries as unknown[]).length)).toEqual([3, 1]); + const entries = appends.flatMap((b) => b.entries as Array<{ memory_key: string; value: string }>); + expect(new Set(entries.map((entry) => entry.memory_key)).size).toBe(4); + expect(entries.map((entry) => JSON.parse(entry.value))).toEqual([ + msg("user", "hi"), + msg("assistant", "hello"), + msg("user", "hi"), + msg("assistant", "again"), + ]); + expect(new Set(appends.map((b) => b.user_id))).toEqual(new Set(["alice"])); + expect(String(appends[0].session_id)).toMatch(/^oai-ts-[0-9a-f]{20}$/); }); it("separates native TypeScript history from Python history", async () => { const { agent, calls } = rowsHarness(); await new LoomSession(agent, "s1", { userId: "alice" }).addItems([msg("user", "x")]); // The digest matches Python, but its oai-py- session stores a different item format. - expect(calls[0].body.session_id).toBe("oai-ts-65872b142b2cfb23bddc"); + expect(calls.find((call) => call.op === "memory.append_batch")?.body.session_id).toBe( + "oai-ts-65872b142b2cfb23bddc", + ); }); it("keeps native tool history separate from Python tool results", async () => { @@ -428,7 +577,11 @@ describe("LoomSession (OpenAI Agents SDK session)", () => { await agent.call( "memory.set", { - memory_key: String(calls[0].body.memory_key), + memory_key: String( + ( + calls.filter((call) => call.op === "memory.append_batch")[0].body.entries as Array<{ memory_key: string }> + )[0].memory_key, + ), value: JSON.stringify({ type: "function_call_output", call_id: "c1", output: "other" }), memory_type: "episodic", }, @@ -508,7 +661,12 @@ describe("LoomSession (OpenAI Agents SDK session)", () => { const { agent, calls, disappear } = rowsHarness(); const session = new LoomSession(agent, "s1", { userId: "alice" }); await session.addItems([msg("assistant", "x".repeat(5000))]); - disappear.add(String(calls[0].body.memory_key)); + disappear.add( + String( + (calls.filter((call) => call.op === "memory.append_batch")[0].body.entries as Array<{ memory_key: string }>)[0] + .memory_key, + ), + ); await expect(session[method]()).rejects.toThrow("unavailable"); }); @@ -532,7 +690,9 @@ describe("LoomSession (OpenAI Agents SDK session)", () => { it("keeps non-ASCII user ids distinct", async () => { const { agent, calls } = rowsHarness(); await new LoomSession(agent, "s1", { userId: "jos\u00e9" }).addItems([msg("user", "x")]); - expect(calls[0].body.session_id).toBe("oai-ts-c10f2e9c4ca65c3f8b98"); + expect(calls.find((call) => call.op === "memory.append_batch")?.body.session_id).toBe( + "oai-ts-c10f2e9c4ca65c3f8b98", + ); }); it("trims long slash prefixes without scanning each suffix", async () => { diff --git a/sdk/typescript/src/integrations/openai-agents.ts b/sdk/typescript/src/integrations/openai-agents.ts index 0e1363fa8c..633f5f00dc 100644 --- a/sdk/typescript/src/integrations/openai-agents.ts +++ b/sdk/typescript/src/integrations/openai-agents.ts @@ -18,6 +18,7 @@ import { type AgentInputItem, type FunctionTool, type Session, tool } from "@openai/agents"; import type { Agent, Scope } from "../agent.js"; +import { Capabilities } from "../capabilities.js"; import { LoomError } from "../errors.js"; import { boundProtection, @@ -211,23 +212,32 @@ export class LoomSession implements Session { async addItems(items: AgentInputItem[]): Promise { this.rememberProducers(); - const scope = await this.scope(); - for (const entry of items) { + if (!items.length) return; + if (items.length > 1000) throw new LoomError("LoomSession appends at most 1000 items at once"); + // Encode the whole call before discovery or mutation; a bad later item writes nothing. + const entries = items.map((entry) => { + const value = JSON.stringify( + mapToolBytes(entry, (data) => (data instanceof Uint8Array ? Array.from(data) : data)), + ); + if (value === undefined) throw new LoomError("Session item cannot be serialized"); const clock = BigInt(Date.now()) * 1_000_000n; lastStamp = clock > lastStamp ? clock : lastStamp + 1n; - // One memory.set per item: a batch folds a value identical to a live row. - await this.agent.call( - "memory.set", - { - memory_key: `${this.prefix}${lastStamp.toString().padStart(20, "0")}-${randomHex()}`, - value: JSON.stringify(mapToolBytes(entry, (data) => (data instanceof Uint8Array ? Array.from(data) : data))), - memory_type: "episodic", - }, - scope, - this.control(true), - ); - await this.requireAllowed(); + return { + memory_key: `${this.prefix}${lastStamp.toString().padStart(20, "0")}-${randomHex()}`, + value, + }; + }); + await this.requireAllowed(); + const scope = await this.scope(); + const served = await this.agent.call("ops.capabilities", {}, scope, this.control(true)); + const caps = new Capabilities(served.ops ?? [], served.build_commit ?? "", served.schema_version ?? "", undefined); + await this.requireAllowed(); + if (!caps.has("memory.append_batch")) { + throw new LoomError("LoomSession needs a server serving memory.append_batch with the Postgres memory authority"); } + // Keep one payload and the transport's idempotency key across retries; never fall back to point writes. + await this.agent.call("memory.append_batch", { entries }, scope, this.control(true)); + await this.requireAllowed(); } async popItem(): Promise { diff --git a/server/core/ch.py b/server/core/ch.py index 15170e721f..151e46a7b3 100644 --- a/server/core/ch.py +++ b/server/core/ch.py @@ -187,10 +187,12 @@ class _CoordinationSession: channel: AsyncClient | None = None checked: bool = False closed: bool = False + inflight: int = 0 def check(self, client: CHClient | None = None) -> None: if ( self.closed + or self.owner._disconnecting or (client is not None and client is not self.owner) or self.owner.generation != self.generation or self.owner.client is not self.driver @@ -200,6 +202,22 @@ def check(self, client: CHClient | None = None) -> None: raise CoordinationSessionLost() +@dataclass +class _CoordinationTransport: + owner_driver: AsyncClient + generation: int + configuration: dict[str, Any] + channel: AsyncClient + + def current(self, client: CHClient) -> bool: + return ( + not client._disconnecting + and client._client is self.owner_driver + and client.generation == self.generation + and client.__dict__.get("_connection_config") == self.configuration + ) + + _coordination_session: ContextVar[_CoordinationSession | None] = ContextVar( "ch_coordination_session", default=None ) @@ -876,6 +894,8 @@ class CHClient: _slots: asyncio.Semaphore | None _stream_slots: asyncio.Semaphore _reconnect_lock: asyncio.Lock + _coordination_idle: _CoordinationTransport | None + _disconnecting: bool def __init__(self) -> None: self._client: AsyncClient | None = None @@ -897,6 +917,8 @@ def _ensure_state(self) -> None: limit = int(settings.ch_pool_maxsize) d["_slots"] = asyncio.Semaphore(limit) if limit > 0 else None d.setdefault("_stream_slots", asyncio.Semaphore(STREAM_SLOTS)) + d.setdefault("_coordination_idle", None) + d.setdefault("_disconnecting", False) if "_reconnect_lock" not in d: d["_reconnect_lock"] = asyncio.Lock() @@ -906,7 +928,12 @@ async def coordination_session(self) -> AsyncIterator[None]: Every step still attests the backend; a checked session is never recreated. """ - if _coordination_session.get() is not None or _analyst_var.get() is not None: + self._ensure_state() + if ( + _coordination_session.get() is not None + or _analyst_var.get() is not None + or self._disconnecting + ): raise CoordinationSessionLost() session = _CoordinationSession(self, self.client, self.generation, uuid.uuid4().hex) configuration = dict(self._connection_config) @@ -918,6 +945,12 @@ async def coordination_session(self) -> AsyncIterator[None]: async def open_channel() -> None: # Own it before the guarded call's final generation/identity check. + idle, self._coordination_idle = self._coordination_idle, None + if idle is not None: + if idle.current(self): + session.channel = idle.channel + return + idle.channel._force_close() session.channel = await _make_driver( configuration, connector_limit=1, coordination=True ) @@ -928,22 +961,34 @@ async def open_channel() -> None: yield failed = False finally: + reusable = not failed and not session.closed and session.inflight == 0 session.closed = True _coordination_session.reset(token) try: - if session.channel is not None: - if failed: - # A delivered proof timeout cannot bound a fresh cleanup await. - session.channel._force_close() - else: - try: - await session.channel.close() - except BaseException: - session.channel._force_close() - raise + await self._return_coordination_transport(session, configuration, reusable) finally: self._stream_slots.release() + async def _return_coordination_transport( + self, session: _CoordinationSession, configuration: dict[str, Any], reusable: bool + ) -> None: + if session.channel is None: + return + transport = _CoordinationTransport( + session.driver, session.generation, configuration, session.channel + ) + if not reusable or not transport.current(self): + # A failed or still-running proof must never lend its channel to another owner. + session.channel._force_close() + elif self._coordination_idle is None: + self._coordination_idle = transport + else: + try: + await session.channel.close() + except BaseException: + session.channel._force_close() + raise + @property def connected(self) -> bool: return self._client is not None @@ -958,20 +1003,31 @@ async def connect(self) -> None: await _turn_off_unsafe_condition_cache(client) self._client = client self._connection_config = configuration + self._disconnecting = False await _probe_vector_state(self) async def disconnect(self) -> None: self._ensure_state() - retired = list(self._retired.values()) - self._retired.clear() - for old in retired: - try: - await old.close() - except Exception: # noqa: BLE001 — closing is best effort - pass - if self._client is not None: - await self._client.close() - self._client = None + self._disconnecting = True + idle, self._coordination_idle = self._coordination_idle, None + try: + retired = list(self._retired.values()) + self._retired.clear() + for old in retired: + try: + await old.close() + except Exception: # noqa: BLE001 — closing is best effort + pass + if self._client is not None: + await self._client.close() + self._client = None + finally: + if idle is not None: + try: + await idle.channel.close() + except BaseException: + idle.channel._force_close() + raise @property def client(self) -> AsyncClient: @@ -1218,6 +1274,8 @@ async def _with_retry( # type: ignore[no-untyped-def] self._ensure_state() session = _coordination_session.get() took = False + if session is not None: + session.inflight += 1 try: if session is not None: session.check(self) @@ -1233,6 +1291,8 @@ async def _with_retry( # type: ignore[no-untyped-def] session.closed = True raise finally: + if session is not None: + session.inflight -= 1 if took: self._release_slot() diff --git a/server/core/idempotency.py b/server/core/idempotency.py index c3522d7f69..911472b831 100644 --- a/server/core/idempotency.py +++ b/server/core/idempotency.py @@ -70,7 +70,7 @@ class CacheResult: _bound_op: ContextVar[str] = ContextVar("loom_idempotency_op", default="") # The operations whose answer `_committed_hit` rebuilds from their revisions. Another # operation's commit (`profile.set` writes through memory.set) leaves its key running. -COMMITTED_REPLAY_OPS = frozenset({"memory.set", "memory.bulk_set"}) +COMMITTED_REPLAY_OPS = frozenset({"memory.set", "memory.bulk_set", "memory.append_batch"}) def bound_key() -> tuple[str, str, int] | None: @@ -404,8 +404,9 @@ async def _committed_hit(conn: Any, row: Any) -> CacheHit: """What a request committed before its answer was lost, in its op's answer shape (only those of COMMITTED_REPLAY_OPS mark their keys committed): each revision's identity, the first one's at the top level for memory.set, one `written` result - per row for memory.bulk_set, marked `idempotent_replay: committed`. Their - ClickHouse delivery is the delivery worker's, as for any commit.""" + per row for memory.bulk_set and memory.append_batch, marked + `idempotent_replay: committed`. Their ClickHouse delivery is the delivery worker's, + as for any commit.""" refs = list(row["revisions"] or []) revs = await conn.fetch( "SELECT r.namespace, r.seq, r.memory_id::text AS memory_id, r.version, " @@ -432,7 +433,7 @@ async def _committed_hit(conn: Any, row: Any) -> CacheHit: for r in revs ] body: dict[str, Any] - if row["operation"] == "memory.bulk_set": # its answer's own shape: one result per row + if row["operation"] in {"memory.bulk_set", "memory.append_batch"}: body = { "written": [ {k: r[k] for k in ("memory_key", "version", "memory_id")} for r in revisions diff --git a/server/identity/app/me_routes.py b/server/identity/app/me_routes.py index e8d7c99957..a0dd99b0c5 100644 --- a/server/identity/app/me_routes.py +++ b/server/identity/app/me_routes.py @@ -66,6 +66,7 @@ async def me_set_default_namespace( _SHARING_WRITE_OPS = [ "memory.set", + "memory.append_batch", "memory.bulk_set", "memory.update_by_id", "memory.pin", diff --git a/server/identity/app/namespace_audit.py b/server/identity/app/namespace_audit.py index 54c46c0e1a..e845a37180 100644 --- a/server/identity/app/namespace_audit.py +++ b/server/identity/app/namespace_audit.py @@ -105,8 +105,8 @@ _LEDGER_GETS = "('memory.get', 'memory.bulk_get')" _LEDGER_SEARCHES = "('memory.search', 'memory.ask')" _LEDGER_SETS = ( - "('memory.set', 'memory.bulk_set', 'memory.update', 'memory.update_by_id', " - "'memory.set_from_messages')" + "('memory.set', 'memory.append_batch', 'memory.bulk_set', 'memory.update', " + "'memory.update_by_id', 'memory.set_from_messages')" ) _NOT_STRUCTURED = "memory_type NOT IN {structured:Array(String)}" diff --git a/server/identity/app/namespace_summary.py b/server/identity/app/namespace_summary.py index 02fee99460..1d5d6e5efe 100644 --- a/server/identity/app/namespace_summary.py +++ b/server/identity/app/namespace_summary.py @@ -94,7 +94,7 @@ async def _activity() -> dict[str, Any] | None: ( """ SELECT - countIf(operation IN ('memory.set', 'memory.bulk_set', + countIf(operation IN ('memory.set', 'memory.append_batch', 'memory.bulk_set', 'memory.set_from_messages', 'memory.update_by_id', 'memory.touch')) AS writes, diff --git a/server/identity/domain/read_scope.py b/server/identity/domain/read_scope.py index 3c491853ec..81ad71df69 100644 --- a/server/identity/domain/read_scope.py +++ b/server/identity/domain/read_scope.py @@ -455,6 +455,7 @@ def pin_read_filters[M: BaseModel](input: M, auth: Any) -> tuple[M, bool]: "another user's reads as not found", "memory.set": "a write", "memory.pin": "a write", + "memory.append_batch": "a write: fresh episodic keys in the pinned scope", "memory.bulk_set": "a write", "memory.import": ( "a write: the items land as the pinned user and a key's state is read inside the " diff --git a/server/insights/app/op_mix.py b/server/insights/app/op_mix.py index 1174827a06..2ce5c93207 100644 --- a/server/insights/app/op_mix.py +++ b/server/insights/app/op_mix.py @@ -25,6 +25,7 @@ "memory.import_evidence", "memory.set_from_messages", "memory.bulk_set", + "memory.append_batch", "memory.record.write", "memory.import", "memory.import_curated", diff --git a/server/memory/api/models_bulk.py b/server/memory/api/models_bulk.py index f3e1ebbdfa..a62db13bde 100644 --- a/server/memory/api/models_bulk.py +++ b/server/memory/api/models_bulk.py @@ -22,6 +22,29 @@ MAX_BULK_ENTRIES = 1000 # not a setting: a wire bound the request schema and SDKs publish +class _AppendBatchEntry(BaseModel): + memory_key: str + value: dict[str, Any] | list[Any] | str | int | float | bool + source_trace: str = "" + # A caller may downgrade the source to derived, external or imported. + source_class: str | None = None + + @field_validator("source_trace") + @classmethod + def _source_trace(cls, v: str) -> str: + return caller_trace(v) + + +class MemoryAppendBatchInput(_Identity): + """Append distinct episodic occurrences atomically on the Postgres authority. + + Every key must be fresh. Equal values remain separate occurrences. A refused + entry refuses the entire batch; this operation never splits the request. + """ + + entries: list[_AppendBatchEntry] = Field(..., min_length=1, max_length=MAX_BULK_ENTRIES) + + class _BulkSetEntry(BaseModel): memory_key: str memory_type: MemoryType = "semantic" diff --git a/server/memory/api/ops_bulk.py b/server/memory/api/ops_bulk.py index 47aac31b8e..9645901323 100644 --- a/server/memory/api/ops_bulk.py +++ b/server/memory/api/ops_bulk.py @@ -3,6 +3,7 @@ from __future__ import annotations from server.memory.api.models_bulk import ( + MemoryAppendBatchInput, MemoryBulkGetInput, MemoryBulkGetOutput, MemoryBulkSetInput, @@ -18,7 +19,7 @@ MemorySetFromMessagesInput, MemorySetFromMessagesOutput, ) -from server.memory.app.bulk_set import handle_memory_bulk_set +from server.memory.app.bulk_set import handle_memory_append_batch, handle_memory_bulk_set from server.memory.app.curation import handle_memory_import_curated from server.memory.app.extract.handler import ( handle_memory_extraction_retry_mine, @@ -33,6 +34,19 @@ # Operation declarations # ============================================================ +memory_append_batch = Operation( + name="memory.append_batch", + maturity="experimental", + session_content=True, + input_model=MemoryAppendBatchInput, + output_model=MemoryBulkSetOutput, + handler=handle_memory_append_batch, + idempotent=True, + required_role=Role.WRITER, + audit=False, # Each committed revision carries its hash-only audit row. +) + + memory_bulk_set = Operation( name="memory.bulk_set", session_content=True, diff --git a/server/memory/api/ops_memory.py b/server/memory/api/ops_memory.py index 41c1d9821a..dda7144ac4 100644 --- a/server/memory/api/ops_memory.py +++ b/server/memory/api/ops_memory.py @@ -19,6 +19,7 @@ ) from server.memory.api.ops_ask import memory_ask from server.memory.api.ops_bulk import ( + memory_append_batch, memory_bulk_get, memory_bulk_set, memory_extraction_retry_mine, @@ -124,6 +125,7 @@ memory_demote, memory_definitions_list, memory_bulk_set, + memory_append_batch, memory_set_from_messages, memory_bulk_get, memory_extraction_status, diff --git a/server/memory/app/bulk_rows.py b/server/memory/app/bulk_rows.py index f932fd508c..1785f48672 100644 --- a/server/memory/app/bulk_rows.py +++ b/server/memory/app/bulk_rows.py @@ -222,6 +222,8 @@ class _BulkWriteState: # Keys of a step built without a vector whose labels the replay changed: its row lands # with none, so `_vectors_to_retire` retires the key's older vector when that row wins. vectorless_keys: set[str] = field(default_factory=set) + # A distinct occurrence batch may neither skip an entry nor revise an existing key. + strict_append: bool = False # Columns a bulk version row sets from the live row under its key before the rest of diff --git a/server/memory/app/bulk_set.py b/server/memory/app/bulk_set.py index d29387dbad..5b5a14a7ca 100644 --- a/server/memory/app/bulk_set.py +++ b/server/memory/app/bulk_set.py @@ -17,6 +17,7 @@ from server.core.ch import get_client, sequential_reads from server.core.datetimes import naive_utc_now from server.core.error_summary import error_summary +from server.core.errors import ValidationError from server.core.hashing import hash_text from server.core.hashing import sha256_value as _content_sha256 from server.core.vectors import codes_state @@ -25,8 +26,10 @@ from server.identity.domain.read_scope import scope_and from server.llm import customer from server.memory.api.models_bulk import ( + MemoryAppendBatchInput, MemoryBulkSetInput, MemoryBulkSetOutput, + _BulkSetEntry, ) from server.memory.app import after_write, blacklist_guard, pg_admission, webhooks from server.memory.app.bulk_rows import ( @@ -520,7 +523,9 @@ async def _bulk_insert(ch: Any, state: _BulkWriteState) -> None: after it fails: a retry of the same values writes nothing and would announce nothing.""" - if state.memory_rows and not pg.enabled(): + if state.strict_append: + _require_append_authority() + if not state.strict_append and state.memory_rows and not pg.enabled(): # Under Postgres the commit orders the batch after a delete (`write_fence`). first = state.memory_rows[0] lifted = await lift_above_deletes( @@ -535,10 +540,14 @@ async def _bulk_insert(ch: Any, state: _BulkWriteState) -> None: r for r in state.embedding_rows if str(r.get("memory_key")) not in retire ] async with fenced_once_committed( - lambda: _high_winners(state), op="memory.bulk_set", ch=ch + lambda: _high_winners(state), + op="memory.append_batch" if state.strict_append else "memory.bulk_set", + ch=ch, ) as answer: committed = ( - await _commit_batch(state, retire) if pg.enabled() and state.memory_rows else None + await _commit_batch(state, retire) + if (state.strict_append or pg.enabled()) and state.memory_rows + else None ) if committed is not None: answer.mark_committed() @@ -654,6 +663,8 @@ async def _commit_batch( without its type's vector, a key the batch embedded earlier, a failed lookup); they owe OLD_VECTORS. The commit adds any row over an embedded head itself.""" + if state.strict_append: + _require_append_authority() # The publication clears a revision's search document only for the version it was # built from, so only that version owes one. embedded = {(identity_key(r), int(r["version"])) for r in state.embedding_rows} @@ -665,16 +676,17 @@ async def _commit_batch( | (pg_memory.OLD_VECTORS if _owes_retirement(r, retire) else 0) for r in state.memory_rows ] + recheck = _fresh_append_head if state.strict_append else heads_unchanged(read) entries = await _reach_of(state) if not entries: # No verdict to keep: only the heads and the policy the guards ran under, so a # batch of unsearchable rows, or only definitions, never waits on search. generations = pg_memory.policy_part(state.generations, state.memory_rows[0]["org"]) - admission = Admission(generations=generations, recheck=heads_unchanged(read)) + admission = Admission(generations=generations, recheck=recheck) else: admission = Admission( generations=dict(state.generations), - recheck=heads_unchanged(read), + recheck=recheck, gap_check=pg_admission.batch_gap_check(entries), ) state.retry.start() @@ -971,7 +983,10 @@ async def _bulk_audit( # Every write lands in the hash-chained audit log as hashes, not text: what was written, # by whom, with which verdict. try: - await _audit_bulk_write(input, state.memory_rows, plan) + if state.strict_append: + await _audit_bulk_write(input, state.memory_rows, plan, operation="memory.append_batch") + else: + await _audit_bulk_write(input, state.memory_rows, plan) except Exception as e: # noqa: BLE001 — the ledger row is best-effort log.warning("bulk_set: audit row failed (non-fatal): %s", e) @@ -1228,12 +1243,53 @@ async def _bulk_follow_row_supersessions( clock.mark("triples_follow_rows") +def _require_append_authority() -> None: + if not pg.enabled(): + raise ValidationError("memory.append_batch requires the Postgres memory authority") + + +def _fresh_append_head(_key: Any, head: dict[str, Any] | None) -> None: + # Checked under the authority's generation and head locks, including deleted heads. + if head is not None: + raise ValidationError("memory.append_batch keys must not already exist") + + +async def handle_memory_append_batch(input: MemoryAppendBatchInput) -> MemoryBulkSetOutput: + _require_append_authority() + keys = [entry.memory_key for entry in input.entries] + if len(set(keys)) != len(keys): + raise ValidationError("memory.append_batch requires distinct keys") + batch = MemoryBulkSetInput( + **input.model_dump(exclude={"entries"}), + entries=[_BulkSetEntry(**e.model_dump(), memory_type="episodic") for e in input.entries], + detect_updates=False, + ) + return await handle_memory_bulk_set(batch, reinforce=False, strict_append=True) + + +def _require_whole_append(input: MemoryBulkSetInput, state: _BulkWriteState) -> None: + if not state.strict_append: + return + if not state.existing_lookup_ok: + raise ValidationError("memory.append_batch could not verify fresh keys") + if state.existing_rows or state.deleted_heads: + raise ValidationError("memory.append_batch keys must not already exist") + if ( + len(state.memory_rows) != len(input.entries) + or any(result.version == 0 for result in state.results) + or state.retyped + or state.unchecked_keys + ): + raise ValidationError("memory.append_batch refused an entry; no items were appended") + + async def handle_memory_bulk_set( input: MemoryBulkSetInput, *, dry_run: bool = False, conflict_check_cap: int | None = None, reinforce: bool = True, + strict_append: bool = False, ) -> MemoryBulkSetOutput: """One CH insert for all rows; one more for embeddings. Versions are each unique (nanosecond) to preserve LWW semantics. @@ -1246,14 +1302,20 @@ async def handle_memory_bulk_set( commit; a batch whose scope or keys changed meanwhile starts over, reusing its vectors and contexts: until LOOM_PG_ADMISSION_DEADLINE_MS for a moved namespace, at most LOOM_PG_ADMISSION_ROUNDS passes for a head or policy that keeps changing.""" + if strict_append: + _require_append_authority() once = functools.partial( - _bulk_set_once, dry_run=dry_run, conflict_check_cap=conflict_check_cap, reinforce=reinforce + _bulk_set_once, + dry_run=dry_run, + conflict_check_cap=conflict_check_cap, + reinforce=reinforce, + strict_append=strict_append, ) - if not pg.enabled(): + if not strict_append and not pg.enabled(): return await once(input, get_embedder(), BatchRetry()) from server.settings import settings - embedder = ReusedVectors(get_embedder()) + embedder = None if strict_append else ReusedVectors(get_embedder()) retry = BatchRetry() head_moves = 0 spent = blacklist_guard.regex_work_spent() @@ -1281,6 +1343,7 @@ async def _bulk_set_once( dry_run: bool, conflict_check_cap: int | None, reinforce: bool, + strict_append: bool = False, ) -> MemoryBulkSetOutput: ch = get_client() # The write names its phases on the response. @@ -1292,16 +1355,17 @@ async def _bulk_set_once( retry=retry, conflict_check_cap=conflict_check_cap, reinforce=reinforce, + strict_append=strict_append, ) if pg.enabled(): # The verdicts below must see every commit through these generations (`catch_up`, below). state.generations = await pg_memory.capture(input.org, input.namespace) await _bulk_existing_lookup(ch, input, state) - identity_dupes = await _bulk_identity_dupes(ch, input) + identity_dupes = {} if strict_append else await _bulk_identity_dupes(ch, input) _bulk_clock.mark("existing_lookup") - entity_resolution = await _bulk_resolve_entities(ch, input) + entity_resolution = {} if strict_append else await _bulk_resolve_entities(ch, input) _bulk_clock.mark("entity_resolution") await _bulk_build_rows(input, state, identity_dupes, entity_resolution) @@ -1317,14 +1381,17 @@ async def _bulk_set_once( # on any replica. with sequential_reads() if pg.enabled() else nullcontext(): plan = await _bulk_verdicts(input, state, emb_idx_of, value_vectors, _bulk_clock) + _require_whole_append(input, state) if dry_run: _bulk_clock.finish() return _bulk_output(state, plan, _bulk_work) # After the verdicts: settling the batch moves entries to unchanged or pinned. _bulk_log_skips(state) - await _bulk_mark_same_key(ch, input, state) - _intra_pairs = await _bulk_intra_batch_pairs(ch, input, state) + _intra_pairs: list[tuple[str, str]] = [] + if not strict_append: + await _bulk_mark_same_key(ch, input, state) + _intra_pairs = await _bulk_intra_batch_pairs(ch, input, state) await _bulk_insert(ch, state) _bulk_clock.mark("insert") @@ -1385,6 +1452,11 @@ async def _bulk_follow_ups( ) -> None: """What a batch does once its rows are in: retirements, conflict marks and same-key supersession, projections, row supersessions.""" + if state.strict_append: + # Fresh episodic occurrences have no semantic or graph projections to settle. + _bulk_record_write_metrics(input, state) + await _bulk_audit(input, state, plan) + return await _record_intra_batch_retirements(input, state, intra_pairs) # A bulk write's retirement intents are left to the repair, which resolves them. @@ -1584,7 +1656,11 @@ async def _one(idx: int, vec: list[float], candidate_rows: list[dict[str, Any]] async def _audit_bulk_write( - input: Any, memory_rows: list[dict[str, Any]], plan: BulkVerdictPlan + input: Any, + memory_rows: list[dict[str, Any]], + plan: BulkVerdictPlan, + *, + operation: str = "memory.bulk_set", ) -> None: """One hash-chained audit row per bulk write: keys, ids and content hashes (never text — set_from_messages carries raw conversation), the writer and @@ -1595,7 +1671,7 @@ async def _audit_bulk_write( return try: await _audit.emit( - event_type="memory.bulk_set", + event_type=operation, org=input.org, namespace=input.namespace, actor=writer_actor() or "-", diff --git a/server/operations/dashboard.py b/server/operations/dashboard.py index a2e107b760..7eb1a0d242 100644 --- a/server/operations/dashboard.py +++ b/server/operations/dashboard.py @@ -166,6 +166,7 @@ def _named_once(*groups: dict[str, str]) -> dict[str, str]: "code.scip_status", "cost.record_turns", "knowledge.bulk_add", + "memory.append_batch", "memory.bulk_set", "memory.delete_version", "memory.export_evidence", diff --git a/server/operations/reach_manifest.py b/server/operations/reach_manifest.py index 0ebff31d12..ad5cd0e47a 100644 --- a/server/operations/reach_manifest.py +++ b/server/operations/reach_manifest.py @@ -365,6 +365,7 @@ ), "memory.agents_list": ("dashboard",), "memory.aggregate": ("sdk_method",), + "memory.append_batch": ("sdk_method",), "memory.apply": ("sdk_method",), "memory.ask": ("sdk_method",), "memory.assign_category": ("sdk_method",), diff --git a/server/operations/toolsets.py b/server/operations/toolsets.py index 25af057f52..e0a8b3208d 100644 --- a/server/operations/toolsets.py +++ b/server/operations/toolsets.py @@ -46,6 +46,8 @@ "memory.unlock", "memory.sweep_expired", "memory.search_trace", + # Native history clients require one indivisible episodic occurrence batch. + "memory.append_batch", "memory.bulk_set", "memory.bulk_get", # A sync job's reads and deletes: one answer of every key in the namespace, and a diff --git a/server/settings/catalog.py b/server/settings/catalog.py index c2a2f43d87..25cfd1f6da 100644 --- a/server/settings/catalog.py +++ b/server/settings/catalog.py @@ -6426,6 +6426,8 @@ class TrackerSettings(BaseModel): # One JSONExtract query over loom.memory FINAL; budget covers large group-by # cardinality. "memory.aggregate": 250, + # Fresh episodic rows: one Postgres commit, publication and the hash-only audit. + "memory.append_batch": 300, "memory.apply": 3000, # Retrieval (memory.search's own budget) plus at most one LLM synthesis # call; a retrieval_only / abstained call pays no LLM latency. @@ -6746,6 +6748,7 @@ class TrackerSettings(BaseModel): # "rewrite": deduplication dropped the saved turn, + the rewritten row's wait. "conversation.save_message": {"rewrite": 350}, "memory.bulk_get": {"audited": 130}, + "memory.append_batch": {"webhook": 350}, "memory.bulk_set": {"webhook": 350}, "memory.delete": {"webhook": 80}, "memory.delete_version": {"webhook": 130}, diff --git a/server/transport/gates.py b/server/transport/gates.py index 2a62959942..d1590671cd 100644 --- a/server/transport/gates.py +++ b/server/transport/gates.py @@ -186,6 +186,7 @@ def with_resolved_org(op: Operation, args: dict[str, Any], auth: AuthInfo | None # that falls back to any agent; "*" where MCP also reads across agents. SINGLE_USER_FILLED_OPS: Final[dict[str, str]] = { "memory.set": "http", + "memory.append_batch": "http", "memory.bulk_set": "http", "memory.get": "http", "memory.delete": "http", diff --git a/server/transport/mcp_op.py b/server/transport/mcp_op.py index 036a83d191..0316679ac3 100644 --- a/server/transport/mcp_op.py +++ b/server/transport/mcp_op.py @@ -253,6 +253,7 @@ def catalog( # "no op reachable" plus look-alikes and guess: a help call, a grep of the # source, a wrong argument name. _INSTEAD: dict[str, str] = { + "memory.append_batch": "a native session client appends its history; nothing to call", "memory.bulk_set": "call memory.set once per fact; a batch saves no round trip here", "memory.bulk_get": "call memory.get per key, or memory.list with a key prefix", "memory.keys": "call memory.list, which pages the namespace", diff --git a/tests/integration/test_cell_control_fence.py b/tests/integration/test_cell_control_fence.py index e18e3e4b3e..4731e35eee 100644 --- a/tests/integration/test_cell_control_fence.py +++ b/tests/integration/test_cell_control_fence.py @@ -612,7 +612,7 @@ async def call(self, path, body): } async def ready(): - await admin.execute("SELECT pg_terminate_backend($1)", a.get_server_pid()) + assert await admin.fetchval("SELECT pg_terminate_backend($1, 5000)", a.get_server_pid()) await apg.take_lock(b, "successor") return [] diff --git a/tests/integration/test_cross_principal_leakage.py b/tests/integration/test_cross_principal_leakage.py index 1a6354b9cc..aa21c73e59 100644 --- a/tests/integration/test_cross_principal_leakage.py +++ b/tests/integration/test_cross_principal_leakage.py @@ -66,6 +66,7 @@ "memory.copy_to", "memory.cross_promote", "memory.set_from_messages", + "memory.append_batch", "memory.bulk_set", "memory.set", "memory.update_by_id", diff --git a/tests/integration/test_pg_authority.py b/tests/integration/test_pg_authority.py index eee1707490..7bc066acc1 100644 --- a/tests/integration/test_pg_authority.py +++ b/tests/integration/test_pg_authority.py @@ -5859,3 +5859,208 @@ async def test_exact_head_locks_bound_a_large_absent_batch(pg_conn: Any, org: st except Exception as error: # The synthetic plan must not replace the native failure. error.add_note("Synthetic head-lock plan before query: " + json.dumps(plan)) raise + + +async def test_an_episodic_batch_preserves_occurrences_and_replays_once( + call: Any, pg_conn: Any, org: str +) -> None: + ns = f"append-{uuid.uuid4().hex[:8]}" + item = json.dumps({"role": "user", "content": "again"}) + body = { + "namespace": ns, + "agent": "a", + "user_id": "u", + "session_id": "s", + "entries": [{"memory_key": k, "value": item} for k in ("first", "second")], + } + headers = {"Idempotency-Key": f"append-{uuid.uuid4().hex}"} + first = await call("memory.append_batch", body, **headers) + assert first.status_code == 200, first.text + replay = await call("memory.append_batch", body, **headers) + assert replay.status_code == 200, replay.text + assert replay.json()["written"] == first.json()["written"] + assert [r["memory_key"] for r in first.json()["written"]] == ["first", "second"] + stored = await pg_conn.fetch( + "SELECT row, seq FROM loom.memory_current WHERE org = $1 AND namespace = $2 ORDER BY seq", + org, + ns, + ) + assert len(stored) == 2 + from server.memory.store import pg_memory + + for key, entry in zip(("first", "second"), stored, strict=True): + row = pg_memory.decode_row(entry["row"]) + assert tuple(row[c] for c in ("org", "namespace", "agent", "user_id", "session_id")) == ( + org, + ns, + "a", + "u", + "s", + ) + assert row["memory_key"] == key and row["memory_type"] == "episodic" + assert json.loads(row["value"]) == item + assert row["source_class"] == "asserted" and row["actor"] == "process:test-suite" + assert len(row["content_sha256"]) == 64 + got = await call( + "memory.get", {k: v for k, v in body.items() if k != "entries"} | {"memory_key": key} + ) + assert got.status_code == 200 and got.json()["value"] == item, got.text + assert ( + await pg_conn.fetchval( + "SELECT count(*) FROM loom.memory_revisions WHERE org = $1 AND namespace = $2", org, ns + ) + == 2 + ) + + +async def test_a_blacklisted_second_append_item_leaves_no_first_item( + call: Any, pg_conn: Any, org: str +) -> None: + ns = f"append-{uuid.uuid4().hex[:8]}" + blocked = f"refuse-{uuid.uuid4().hex}" + policy = await call("blacklist.add", {"namespace": ns, "pattern": blocked}) + assert policy.status_code == 200, policy.text + response = await call( + "memory.append_batch", + { + "namespace": ns, + "agent": "a", + "user_id": "u", + "session_id": "s", + "entries": [ + {"memory_key": "first", "value": "keep separate"}, + {"memory_key": "second", "value": blocked}, + ], + }, + ) + assert response.status_code == 422, response.text + assert response.json()["detail"]["code"] == "VALIDATION_FAILED" + for table in ("memory_current", "memory_revisions", "memory_deliveries"): + assert ( + await pg_conn.fetchval( + f"SELECT count(*) FROM loom.{table} WHERE org = $1 AND namespace = $2", org, ns + ) + == 0 + ) + + +async def test_concurrent_appends_refuse_the_whole_losing_batch( + call: Any, pg_conn: Any, org: str +) -> None: + ns = f"append-{uuid.uuid4().hex[:8]}" + + async def append(own: str) -> httpx.Response: + return await call( + "memory.append_batch", + { + "namespace": ns, + "agent": "a", + "user_id": "u", + "session_id": "s", + "entries": [ + {"memory_key": "contended", "value": own}, + {"memory_key": own, "value": own}, + ], + }, + ) + + left, right = await asyncio.gather(append("left"), append("right")) + assert sorted([left.status_code, right.status_code]) == [200, 422], (left.text, right.text) + winner = "left" if left.status_code == 200 else "right" + heads = await pg_conn.fetch( + "SELECT memory_key FROM loom.memory_current WHERE org = $1 AND namespace = $2", org, ns + ) + assert {r["memory_key"] for r in heads} == {"contended", winner} + assert ( + await pg_conn.fetchval( + "SELECT count(*) FROM loom.memory_revisions WHERE org = $1 AND namespace = $2", org, ns + ) + == 2 + ) + + +@pytest.mark.parametrize("refused", ["viewer", "namespace", "user"]) +async def test_append_batch_enforces_the_writer_namespace_and_user_gates( + control_plane_url: str, mint_test_token: Any, org: str, refused: str +) -> None: + ns = f"append-{uuid.uuid4().hex[:8]}" + token = mint_test_token( + org=org, + namespaces=["elsewhere" if refused == "namespace" else ns], + roles=["viewer" if refused == "viewer" else "writer"], + read_user_id="alice" if refused == "user" else None, + ) + async with httpx.AsyncClient(base_url=control_plane_url) as client: + response = await client.post( + "/v1/memory.append_batch", + headers={"Authorization": f"Bearer {token}"}, + json={ + "org": org, + "namespace": ns, + "agent": "a", + "user_id": "bob", + "entries": [{"memory_key": "first", "value": "private"}], + }, + ) + assert response.status_code == 403, response.text + + +async def test_append_respects_the_deployments_storage_authority(call: Any) -> None: + from server.core import pg + + ns = f"append-{uuid.uuid4().hex[:8]}" + response = await call( + "memory.append_batch", + { + "namespace": ns, + "agent": "a", + "entries": [{"memory_key": "first", "value": "not committed"}], + }, + ) + assert response.status_code == (200 if pg.enabled() else 422), response.text + got = await call("memory.get", {"namespace": ns, "agent": "a", "memory_key": "first"}) + assert got.status_code == 200 and got.json()["found"] is pg.enabled(), got.text + + +async def test_user_erasure_covers_every_appended_occurrence( + call: Any, pg_conn: Any, org: str +) -> None: + ns = f"append-{uuid.uuid4().hex[:8]}" + user = f"u-{uuid.uuid4().hex[:8]}" + for who in (user, "someone-else"): + response = await call( + "memory.append_batch", + { + "namespace": ns, + "agent": "a", + "user_id": who, + "session_id": "s", + "entries": [{"memory_key": key, "value": "again"} for key in ("first", "second")], + }, + ) + assert response.status_code == 200, response.text + erased = await call( + "erasure.request", + { + "scope_type": "user_id", + "scope_value": user, + "requester": "t", + }, + ) + assert erased.status_code == 200 and erased.json()["status"] == "completed", erased.text + for table in ("loom.memory_current", "loom.memory_revisions"): + users = await pg_conn.fetch( + f"SELECT user_id FROM {table} WHERE org = $1 AND namespace = $2", org, ns + ) + assert [row["user_id"] for row in users] == ["someone-else", "someone-else"], table + got = await call( + "memory.get", + { + "namespace": ns, + "agent": "a", + "user_id": user, + "session_id": "s", + "memory_key": "first", + }, + ) + assert got.status_code == 200 and not got.json()["found"], got.text diff --git a/tests/integration/test_read_scope_isolation.py b/tests/integration/test_read_scope_isolation.py index e2f858691c..18f698c3b4 100644 --- a/tests/integration/test_read_scope_isolation.py +++ b/tests/integration/test_read_scope_isolation.py @@ -586,14 +586,32 @@ def _attack_moves(seed: _Seed) -> list[tuple[str, dict[str, Any]]]: return moves +def _append_authority_refusal(base: str, response: httpx.Response) -> bool: + """An append must write on Postgres or give its exact unsupported-authority refusal.""" + with httpx.Client(base_url=base, timeout=60.0) as client: + health = client.get("/v1/health") + assert health.status_code == 200, health.text + authority = health.json()["storage_authority"] + if authority == "postgres": + assert response.status_code == 200, response.text + return False + assert authority == "clickhouse", health.text + assert response.status_code == 422, response.text + detail = response.json()["detail"] + assert detail["code"] == "VALIDATION_FAILED", response.text + assert detail["message"] == "memory.append_batch requires the Postgres memory authority" + return True + + def test_writes_aimed_at_vs_rows_never_answer_or_change_them( control_plane_url: str, seed: _Seed, tokens: dict[str, str], ch_client: Any ) -> None: """A token pinned to W (who holds nothing) sends every write a bounded token may call at V's keys, ids and brief, plus a delete, lock, touch and outcome (by key and by id) and an if_match probe at each of V's facts. No answer carries V's nonce or - hash, a refusal is a 409 or a 423 (W's own lock) at most, and each of V's rows and - its usage counters read back unchanged afterwards.""" + hash, a refusal is a 409 or a 423 (W's own lock), or an append's exact unsupported + authority refusal. Each of V's rows and its usage counters read back unchanged + afterwards.""" ns_tok, w_tok = tokens["namespace"], tokens["pinned_w"] shas: list[str] = [] for key, session in zip(seed.keys_v, ("sess-1", "sess-2", "-"), strict=True): @@ -659,11 +677,16 @@ def _stable() -> tuple[dict[str, tuple[Any, ...]], dict[tuple[str, str], int]] | unbuildable.append(op.name) continue r = _post(control_plane_url, w_tok, op.name, body) - if r.status_code == 422: + authority_refusal = ( + _append_authority_refusal(control_plane_url, r) + if op.name == "memory.append_batch" + else False + ) + if r.status_code == 422 and not authority_refusal: unbuildable.append(op.name) continue walked.add(op.name) - if r.status_code not in {200, 404, 409, 423}: # 409/423: a refused write + if not authority_refusal and r.status_code not in {200, 404, 409, 423}: bad[op.name] = r.status_code if seed.nonce_v in r.text: leaks[op.name] = r.text[:300] @@ -680,7 +703,12 @@ def _stable() -> tuple[dict[str, tuple[Any, ...]], dict[tuple[str, str], int]] | assert not leaks, f"V's canary or hash reached W's token: {leaks}" assert not bad, f"unexpected answers: {bad}" assert set(unbuildable) <= set(_UNBUILDABLE), unbuildable - assert {"session.brief_save", "session.brief_delete", "memory.set"} <= walked + assert { + "session.brief_save", + "session.brief_delete", + "memory.set", + "memory.append_batch", + } <= walked r = _post( control_plane_url, ns_tok, diff --git a/tests/unit/session_content_senders.json b/tests/unit/session_content_senders.json index bfb7cd41e8..2ab7d3f24c 100644 --- a/tests/unit/session_content_senders.json +++ b/tests/unit/session_content_senders.json @@ -345,6 +345,10 @@ "category": "reference", "reason": "Reads or removes stored records by their identity, prefix or exact version; automatic content writes remain classified." }, + "sdk/typescript/src/integrations/openai-agents.ts:ops.capabilities:d29e857647f728e2169d:1": { + "category": "scope", + "reason": "Sends only identity scope and an empty operation catalogue request; retained SessionCapture contributors still guard discovery and append." + }, "sdk/typescript/src/session.ts::f81d27efc95e87cdbb46:1": { "category": "delivery", "reason": "Forwards the operation and body from separately scanned session capture producers without selecting additional content; this module pins transcript, carry-over, query and thread builders." @@ -401,15 +405,15 @@ "sdk/python/loom/integrations/haystack.py": "8ab24c9dfb4220b35c8194605a77368a9b7b8a29d27a0b6b30569668947a31e0", "sdk/python/loom/integrations/langgraph.py": "5d68e6ab1645d868477a13b14aa6d734c72f72057464cac9a21aa707add17e7b", "sdk/python/loom/integrations/llamaindex.py": "06eaa071115cad752d27bdc9bf7c4d5b6f5d7d39d9896267fcd459a4e06bf661", - "sdk/python/loom/integrations/openai_agents.py": "e6fe0384a361c2c5e48db26837d8c454e52b5f7fb010fd7f6541f24a22459a7a", + "sdk/python/loom/integrations/openai_agents.py": "143fe1df060b96550a33f77fa44416e291c2154552e433f8309275f61c8fca2a", "sdk/python/loom/memory.py": "e1384e84ddf58b503dc430eca1b2e734eb5e5680ac6904a64785bb50a63316d1", - "sdk/python/loom/run.py": "3d408cedd849fe9c16a9cb1a57dbe88367993aa225f6a8b23bac8c644e04723b", + "sdk/python/loom/run.py": "1d47d5f4d197f78de347d2cf66b9f59f4292846d214f7116c444035971c6ab42", "sdk/python/loom/session.py": "5f0fd61fcfdbf9d60888b111887fca8f32e24eb51ca9ff04a256901657c3202b", "sdk/typescript/src/integrations/claude-memory.ts": "c846f0011d7cf1ba4b30e99fbfd8adb6b29abff56856d0dbe14fba620007f6bc", "sdk/typescript/src/integrations/google-adk.ts": "26035863d6a81630e1a10c2045128b4483151ad9e4d13e9202d9c90054cf62d6", "sdk/typescript/src/integrations/langchain.ts": "2e6b93cbd2eb3ed57d910404845f8de9b06b9d3e9e2c59194100ad95f8819ce8", "sdk/typescript/src/integrations/mastra-memory.ts": "a0c21387e645579117bd00c4961efd1c65938c0a7128eb31c88256d49522710a", - "sdk/typescript/src/integrations/openai-agents.ts": "6d3206f230d8c1be4fb1bf7069185c212c8995b0d0cd41d827adb924535b4322", + "sdk/typescript/src/integrations/openai-agents.ts": "f185d90da6512833c590c4729aaf2bfb140ff469e4864fad6d761e29a66c1b0a", "sdk/typescript/src/session.ts": "49c162dd55f578d508a9c537891a991d7e46866ee3dbaf5f9b9e4d67242b5e05" } } diff --git a/tests/unit/snapshots/sdk_public_api.json b/tests/unit/snapshots/sdk_public_api.json index 48adad0e03..eba60776ae 100644 --- a/tests/unit/snapshots/sdk_public_api.json +++ b/tests/unit/snapshots/sdk_public_api.json @@ -1049,6 +1049,10 @@ "body", "options" ], + "append_batch": [ + "body", + "options" + ], "apply": [ "body", "options" diff --git a/tests/unit/test_ch_coordination_session.py b/tests/unit/test_ch_coordination_session.py index 9f1c9a54df..d131eee398 100644 --- a/tests/unit/test_ch_coordination_session.py +++ b/tests/unit/test_ch_coordination_session.py @@ -209,12 +209,15 @@ async def proof() -> None: await asyncio.gather(proof(), proof()) await client.query_one("SELECT 3") - assert len(client.proofs) == 2 and all(driver.closed for driver in client.proofs) + assert len(client.proofs) == 2 + assert sum(not driver.closed for driver in client.proofs) <= 1 sessions = [ settings["session_id"] for driver in client.proofs for _, settings, _ in driver.calls ] assert len(set(sessions)) == 2 and all(sessions.count(value) == 2 for value in set(sessions)) assert len(client.client.calls) == 1 and "session_id" not in client.client.calls[0][1] + await client.disconnect() + assert all(driver.closed for driver in client.proofs) @pytest.mark.asyncio @@ -301,3 +304,266 @@ async def test_unsupported_operations_refuse_before_any_send_and_poison_the_scop with pytest.raises(ch.CoordinationSessionLost): await client.query_one("SELECT 3") assert len(client.proofs[0].calls) == 1 and other.client.calls == [] + + +@pytest.mark.asyncio +async def test_sequential_proofs_reuse_transport_but_bootstrap_distinct_sessions( + client: Any, +) -> None: + for index in range(3): + async with client.coordination_session(): + await client.query_one(f"SELECT {index + 2}") + + assert len(client.proofs) == 1 + driver = client.proofs[0] + assert not driver.closed + assert len(driver.calls) == 6 + sessions = [extra["session_id"] for _, extra, _ in driver.calls] + assert len(set(sessions)) == 3 + assert all(sessions.count(value) == 2 for value in set(sessions)) + assert [extra["session_check"] for _, extra, _ in driver.calls] == [0, 1] * 3 + assert all( + extra["use_query_cache"] == extra["use_query_condition_cache"] == 0 + for _, extra, _ in driver.calls + ) + assert client.client.calls == [] + await client.disconnect() + assert driver.closed + + +@pytest.mark.asyncio +async def test_a_caught_failure_discards_the_reusable_transport(client: Any) -> None: + async with client.coordination_session(): + first_channel = client._statement_client() + await client.query_one("SELECT 2") + first = client.proofs[0] + async with client.coordination_session(): + assert client._statement_client() is first_channel + first.error = ConnectionResetError("connection reset") + with pytest.raises(ConnectionResetError): + await client.query_one("SELECT 3") + assert first.closed + async with client.coordination_session(): + assert client._statement_client() is not first_channel + await client.query_one("SELECT 4") + assert len(client.proofs) == 2 + await client.disconnect() + assert all(driver.closed for driver in client.proofs) + + +@pytest.mark.asyncio +async def test_reusable_transport_is_not_shared_between_clients(client: Any) -> None: + async with client.coordination_session(): + original_channel = client._statement_client() + await client.query_one("SELECT 2") + original = client.proofs[0] + other = ch.CHClient() + other._client = _Driver() + other._connection_config = dict(client._connection_config) + async with other.coordination_session(): + assert other._statement_client() is not original_channel + await other.query_one("SELECT 3") + assert len(client.proofs) == 2 + assert not original.closed + await other.disconnect() + assert not original.closed + await client.disconnect() + assert original.closed + + +@pytest.mark.asyncio +@pytest.mark.parametrize("replacement", ["generation", "driver", "configuration"]) +async def test_idle_transport_cannot_cross_a_connection_replacement( + client: Any, replacement: str +) -> None: + async with client.coordination_session(): + original_channel = client._statement_client() + await client.query_one("SELECT 2") + original = client.proofs[0] + if replacement == "generation": + client._generation += 1 + elif replacement == "driver": + client._client = _Driver() + else: + client._connection_config = dict(client._connection_config, username="different") + async with client.coordination_session(): + assert client._statement_client() is not original_channel + await client.query_one("SELECT 3") + assert len(client.proofs) == 2 and original.closed + await client.disconnect() + assert all(driver.closed for driver in client.proofs) + + +@pytest.mark.asyncio +async def test_owner_exit_discards_a_channel_with_an_inflight_child(client: Any) -> None: + pending = None + try: + async with client.coordination_session(): + original_channel = client._statement_client() + original = client.proofs[0] + original.release = asyncio.Event() + pending = asyncio.create_task(client.query_one("SELECT 2")) + await original.entered.wait() + assert original.closed + async with client.coordination_session(): + assert client._statement_client() is not original_channel + await client.query_one("SELECT 3") + finally: + if pending is not None: + original.release.set() + outcomes = await asyncio.gather(pending, return_exceptions=True) + await client.disconnect() + assert isinstance(outcomes[0], ch.CoordinationSessionLost) + assert all(driver.closed for driver in client.proofs) + + +@pytest.mark.asyncio +async def test_idle_transport_reuse_never_revives_an_old_child_session(client: Any) -> None: + release = asyncio.Event() + + async def later() -> None: + await release.wait() + with pytest.raises(ch.CoordinationSessionLost): + await client.query_one("SELECT 2") + + pending = None + try: + async with client.coordination_session(): + original_channel = client._statement_client() + pending = asyncio.create_task(later()) + async with client.coordination_session(): + assert client._statement_client() is original_channel + release.set() + await pending + await client.query_one("SELECT 3") + assert len(client.proofs) == 1 and len(client.proofs[0].calls) == 3 + finally: + release.set() + if pending is not None: + await asyncio.gather(pending, return_exceptions=True) + await client.disconnect() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("failure", ["bootstrap", "preflight", "unsupported", "body"]) +async def test_a_refused_reused_transport_is_not_lent_to_the_next_proof( + client: Any, monkeypatch: pytest.MonkeyPatch, failure: str +) -> None: + async with client.coordination_session(): + original_channel = client._statement_client() + original = client.proofs[0] + try: + if failure == "bootstrap": + original.error = ConnectionResetError("connection reset") + with pytest.raises(ConnectionResetError): + async with client.coordination_session(): + pytest.fail("a failed bootstrap admitted a proof") + elif failure == "body": + with pytest.raises(ValueError, match="proof refused"): + async with client.coordination_session(): + raise ValueError("proof refused") + else: + async with client.coordination_session(): + if failure == "preflight": + with monkeypatch.context() as patch: + patch.setattr( + ch, "guard_pending_schema", AsyncMock(side_effect=ValueError()) + ) + with pytest.raises(ValueError): + await client.query_one("SELECT 2") + else: + with pytest.raises(ch.CoordinationSessionLost): + await client.insert("loom._migration_lock", [{"name": "test"}]) + assert original.closed + async with client.coordination_session(): + assert client._statement_client() is not original_channel + await client.query_one("SELECT 3") + assert len(client.proofs) == 2 + finally: + await client.disconnect() + + +@pytest.mark.asyncio +async def test_cancelled_reuse_bootstrap_discards_the_transport_and_returns_capacity( + client: Any, +) -> None: + async with client.coordination_session(): + pass + original = client.proofs[0] + original.release = asyncio.Event() + capacity = client._stream_slots._value + + async def proof() -> None: + async with client.coordination_session(): + pytest.fail("a blocked bootstrap admitted a proof") + + pending = asyncio.create_task(proof()) + try: + await original.entered.wait() + pending.cancel() + with pytest.raises(asyncio.CancelledError): + await pending + assert original.closed and client._stream_slots._value == capacity + async with client.coordination_session(): + await client.query_one("SELECT 2") + assert len(client.proofs) == 2 + finally: + pending.cancel() + await asyncio.gather(pending, return_exceptions=True) + await client.disconnect() + + +@pytest.mark.asyncio +async def test_disconnect_cannot_receive_an_idle_channel_from_an_active_proof( + client: Any, monkeypatch: pytest.MonkeyPatch +) -> None: + entered, release = asyncio.Event(), asyncio.Event() + closing, closed = asyncio.Event(), asyncio.Event() + + async def proof() -> None: + async with client.coordination_session(): + entered.set() + await release.wait() + + async def close_shared() -> None: + closing.set() + await closed.wait() + + monkeypatch.setattr(client.client, "close", close_shared) + pending = asyncio.create_task(proof()) + disconnect = None + try: + await entered.wait() + disconnect = asyncio.create_task(client.disconnect()) + await closing.wait() + release.set() + await pending + assert client.proofs[0].closed + with pytest.raises(ch.CoordinationSessionLost): + async with client.coordination_session(): + pytest.fail("a disconnecting client admitted a proof") + finally: + release.set() + closed.set() + await asyncio.gather(pending, *([disconnect] if disconnect is not None else [])) + assert not client.connected and client._coordination_idle is None + + +@pytest.mark.asyncio +async def test_reused_channel_keeps_the_service_principal_for_each_fresh_session( + client: Any, +) -> None: + try: + with ch.caller_shaped(): + for _ in range(2): + async with client.coordination_session(): + await client.query_one("SELECT 2") + await client.query_one("SELECT 3") + assert len(client.proofs) == 1 + calls = client.proofs[0].calls + assert all(kwargs == {} for _, _, kwargs in calls) + assert [settings["session_check"] for _, settings, _ in calls] == [0, 1, 0, 1] + assert len({settings["session_id"] for _, settings, _ in calls}) == 2 + assert "Authorization" in client.client.calls[0][2]["transport_settings"] + finally: + await client.disconnect() diff --git a/tests/unit/test_ch_coordination_transport.py b/tests/unit/test_ch_coordination_transport.py index de952e86e0..ea2f340d07 100644 --- a/tests/unit/test_ch_coordination_transport.py +++ b/tests/unit/test_ch_coordination_transport.py @@ -85,8 +85,8 @@ async def create(**kwargs: Any) -> _RoutedDriver: else: assert await client.raw_bytes("SELECT 1") == b"1" assert client.client is shared and shared.calls == [] and not shared.closed - assert len(channels) == 1 and channels[0].closed - assert channels[0].close_calls == 1 and channels[0].force_calls == 0 + assert len(channels) == 1 and not channels[0].closed + assert channels[0].close_calls == 0 and channels[0].force_calls == 0 assert [call["session_check"] for call in channels[0].calls] == [0, 1] assert len({call["session_id"] for call in channels[0].calls}) == 1 assert {key: options[0][key] for key in client._connection_config} == client._connection_config @@ -94,6 +94,9 @@ async def create(**kwargs: Any) -> _RoutedDriver: assert options[0]["query_retries"] == 0 assert options[0]["settings"]["use_query_cache"] == 0 assert options[0]["settings"]["use_query_condition_cache"] == 0 + await client.disconnect() + assert channels[0].closed and channels[0].close_calls == 1 + assert channels[0].force_calls == 0 def _client_with_channel( @@ -151,17 +154,19 @@ async def test_a_missing_checked_session_closes_only_its_channel_without_reboots @pytest.mark.asyncio -async def test_cancelled_close_force_closes_the_owned_channel_and_returns_capacity( +async def test_cancelled_disconnect_force_closes_the_idle_channel_and_keeps_capacity( monkeypatch: pytest.MonkeyPatch, ) -> None: channel = _RoutedDriver(pooled=False) monkeypatch.setattr(channel, "close", AsyncMock(side_effect=asyncio.CancelledError())) client, _ = _client_with_channel(monkeypatch, channel) capacity = client._stream_slots._value + async with client.coordination_session(): + pass + assert not channel.closed with pytest.raises(asyncio.CancelledError): - async with client.coordination_session(): - pass - assert channel.closed and channel.force_calls == 1 and not client.client.closed + await client.disconnect() + assert channel.closed and channel.force_calls == 1 and not client.connected assert client._stream_slots._value == capacity and ch._coordination_session.get() is None @@ -221,8 +226,10 @@ async def proof() -> None: client._stream_slots.release() await pending assert pending.done() and client._stream_slots._value == 1 - assert channel.closed and not client.client.closed + assert not channel.closed and not client.client.closed factory.assert_awaited_once() + await client.disconnect() + assert channel.closed @pytest.mark.asyncio @@ -244,7 +251,9 @@ async def ordinary() -> Any: assert len(client.client.calls) == 1 and "session_id" not in client.client.calls[0] assert len(channel.calls) == 1 assert await client.query_one("SELECT 1") == {"value": 1} - assert channel.closed and not client.client.closed + assert not channel.closed and not client.client.closed + await client.disconnect() + assert channel.closed @pytest.mark.asyncio @@ -330,14 +339,18 @@ async def proof(index: int) -> None: assert len(channels) == ch.STREAM_SLOTS and slots._value == 0 release[0].set() await entered[-1].wait() - assert channels[0].closed and len(channels) == ch.STREAM_SLOTS + 1 + assert not channels[0].closed and len(channels) == ch.STREAM_SLOTS + assert len({call["session_id"] for call in channels[0].calls}) == 2 assert sum(not channel.closed for channel in channels) == ch.STREAM_SLOTS finally: for event in release: event.set() await asyncio.gather(*tasks, return_exceptions=True) - assert all(channel.closed for channel in channels) and slots._value == ch.STREAM_SLOTS + assert sum(not channel.closed for channel in channels) == 1 + assert slots._value == ch.STREAM_SLOTS assert not client.client.closed + await client.disconnect() + assert all(channel.closed for channel in channels) @pytest.mark.asyncio diff --git a/tests/unit/test_erasure_replicated_fixture.py b/tests/unit/test_erasure_replicated_fixture.py new file mode 100644 index 0000000000..02c93ce77a --- /dev/null +++ b/tests/unit/test_erasure_replicated_fixture.py @@ -0,0 +1,96 @@ +"""The replicated erasure fixture supplies an isolated write-lease store.""" + +from __future__ import annotations + +from types import SimpleNamespace + +import pytest + +from server.core import leases +from server.settings import settings +from tests.integration import test_erasure_replicated_ch as replicated +from tests.unit.claims_fake import FakeLockTable + + +class Schema: + def __init__(self): + self.tables = {} + self.commands = [] + + def command(self, sql): + self.commands.append(sql) + if sql.startswith("CREATE TABLE "): + self.tables[sql.split()[2]] = sql + + def query(self, sql, parameters=None): + if "engine_full" in sql: + assert parameters["db"] == "source_schema" + return SimpleNamespace(result_rows=[("ReplacingMergeTree(version) ORDER BY tuple()",)]) + assert "SELECT DISTINCT engine" in sql + flags = {"ENGINE = Replicated" in ddl for ddl in self.tables.values()} + return SimpleNamespace(result_rows=[(flag,) for flag in sorted(flags)]) + + +class FixtureLocks(FakeLockTable): + def __init__(self, schema): + super().__init__() + self.schema = schema + + async def query_one(self, sql, params=None): + if "loom._migration_lock" in sql: + if f"{settings.ch_database}._migration_lock" not in self.schema.tables: + raise RuntimeError("Unknown table _migration_lock") + return await super().query_one(sql, params) + + +@pytest.fixture +def isolated_schema(monkeypatch): + monkeypatch.setattr(settings, "ch_database", "source_schema") + schema = Schema() + stream = replicated.db.__wrapped__(schema, monkeypatch) + name = next(stream) + try: + yield schema, name + finally: + stream.close() + assert schema.commands[-1] == f"DROP DATABASE IF EXISTS {name} SYNC" + + +async def test_replicated_erasure_fixture_can_take_and_release_a_write_lease(isolated_schema): + schema, name = isolated_schema + keeper = FixtureLocks(schema) + lease = await leases.take("memory_insert/fixture", ttl_s=60, ch=keeper) + assert lease is not None and lease.held + assert settings.ch_database == name + await lease.release() + assert keeper.keys == {} + + +async def test_a_missing_fixture_lease_table_still_refuses(isolated_schema): + schema, name = isolated_schema + schema.tables.pop(f"{name}._migration_lock", None) + with pytest.raises(leases.LeaseUnavailable): + await leases.take("memory_insert/fixture", ttl_s=60, ch=FixtureLocks(schema)) + + +def test_fixture_lease_paths_are_separate_from_source_and_each_other(monkeypatch): + paths = [] + for _ in range(2): + with monkeypatch.context() as patch: + patch.setattr(settings, "ch_database", "source_schema") + schema = Schema() + stream = replicated.db.__wrapped__(schema, patch) + name = next(stream) + try: + ddl = schema.tables[f"{name}._migration_lock"] + assert "AS source_schema._migration_lock" in ddl + assert f"KeeperMap('/loom/tests/{name}/migration_lock')" in ddl + assert "PRIMARY KEY name" in ddl + assert all( + "ENGINE = Replicated" in schema.tables[f"{name}.{table}"] + for table in replicated._TABLES + ) + paths.append(name) + finally: + stream.close() + assert paths[0] != paths[1] diff --git a/tests/unit/test_memory_append_batch.py b/tests/unit/test_memory_append_batch.py new file mode 100644 index 0000000000..b489b7368c --- /dev/null +++ b/tests/unit/test_memory_append_batch.py @@ -0,0 +1,389 @@ +"""Atomic episodic appends keep every occurrence or refuse the whole request.""" + +from __future__ import annotations + +import copy +import json +from contextvars import ContextVar +from datetime import UTC, datetime +from typing import Any +from unittest.mock import AsyncMock, Mock + +import pytest + +from server.core import idempotency, pg_lease +from server.core.errors import NotFoundError, ValidationError +from server.memory import store as memory_repo +from server.memory.api.models_bulk import MemoryAppendBatchInput +from server.memory.app import blacklist_guard, bulk_rows, bulk_set +from server.memory.store import identity_key, pg_memory, pg_publish +from server.settings import settings +from tests.unit.ch_fake import RecordingCH +from tests.unit.clock_fake import ManualClock +from tests.unit.memory_rows import memory_head + + +def _input(*pairs: tuple[str, Any]) -> MemoryAppendBatchInput: + return MemoryAppendBatchInput( + org="test", + namespace="personal", + agent="cursor", + user_id="u", + session_id="s", + entries=[{"memory_key": key, "value": value} for key, value in pairs], + ) + + +def _head(*, tombstoned: bool = False) -> Any: + return memory_head( + org="test", + namespace="personal", + agent="cursor", + user_id="u", + session_id="s", + memory_key="second", + memory_type="episodic", + value=json.dumps({"role": "user", "content": "old"}), + memory_id="11111111-1111-1111-1111-111111111111", + version=1, + tombstoned_at=datetime(2025, 1, 2, tzinfo=UTC) if tombstoned else None, + ) + + +class _Authority: + """Storage seams; preparation, verdicts and commit admission remain real. + + The commit stand-in applies the handler's locked-head callback to the whole + batch before recording any row. It does not qualify Postgres transactions. + """ + + def __init__(self, monkeypatch: pytest.MonkeyPatch) -> None: + self.ch = RecordingCH() + self.lookup: dict[Any, Any] = {} + self.locked_heads: dict[Any, dict[str, Any]] = {} + self.rows: dict[Any, dict[str, Any]] = {} + self.commits: list[pg_memory.Committed] = [] + self.commit_attempts = 0 + self.publication_error: BaseException | None = None + self.lookup_error: BaseException | None = None + self.checked_values: list[str] = [] + self.refused_value: str | None = None + self.clock = ManualClock(t=100.0) + monkeypatch.setattr(settings, "storage_authority", "postgres") + monkeypatch.setattr(bulk_rows.time, "monotonic", self.clock.monotonic) + monkeypatch.setattr(bulk_set, "get_client", lambda: self.ch) + monkeypatch.setattr(bulk_rows, "writer_actor", lambda: "process:append-test") + self.embedder = Mock(side_effect=AssertionError("an episodic append used an embedder")) + monkeypatch.setattr(bulk_set, "get_embedder", self.embedder) + monkeypatch.setattr( + bulk_set, + "_bulk_identity_dupes", + AsyncMock(side_effect=AssertionError("identity dedup")), + ) + monkeypatch.setattr( + bulk_set, + "_bulk_resolve_entities", + AsyncMock(side_effect=AssertionError("entity lookup")), + ) + monkeypatch.setattr(pg_memory, "capture", AsyncMock(return_value={})) + monkeypatch.setattr(memory_repo, "memory_rows_by_identity_keys", self.read_heads) + monkeypatch.setattr(blacklist_guard, "find_match", self.find_match) + monkeypatch.setattr(blacklist_guard, "note_hit", AsyncMock()) + monkeypatch.setattr(bulk_set, "_bulk_mark_same_key", AsyncMock()) + monkeypatch.setattr(bulk_set, "_bulk_intra_batch_pairs", AsyncMock(return_value=[])) + monkeypatch.setattr(bulk_set, "_bulk_follow_ups", AsyncMock()) + monkeypatch.setattr(bulk_set, "_fence_and_announce", AsyncMock()) + monkeypatch.setattr(bulk_set.after_write, "index_late_rows", Mock()) + monkeypatch.setattr(pg_publish, "commit", self.commit) + monkeypatch.setattr(pg_publish, "publish", self.publish) + + async def read_heads(self, keys: Any, **_: Any) -> dict[Any, Any]: + if self.lookup_error is not None: + raise self.lookup_error + return {key: self.lookup[key] for key in keys if key in self.lookup} + + async def find_match(self, *, value: str, **_: Any) -> Any: + self.checked_values.append(value) + if value == self.refused_value: + return blacklist_guard.BlacklistPattern( + "pattern", "test", "personal", "blocked", "substring", "namespace", "" + ) + return None + + async def commit( + self, + rows: list[dict[str, Any]], + *, + targets: list[int], + admission: pg_memory.Admission, + idempotent: bool, + ) -> pg_memory.Committed: + self.commit_attempts += 1 + assert idempotent + assert admission.recheck is not None + for row in rows: + key = identity_key(row) + admission.recheck(key, self.locked_heads.get(key)) + stored = copy.deepcopy(rows) + committed = pg_memory.Committed( + rows=stored, + audits=[], + seqs=[(row["org"], row["namespace"], i + 1) for i, row in enumerate(stored)], + targets=targets, + ) + self.commits.append(committed) + self.rows.update({identity_key(row): copy.deepcopy(row) for row in stored}) + return committed + + async def publish(self, committed: pg_memory.Committed, **_: Any) -> None: + assert committed is self.commits[-1] + if self.publication_error is not None: + raise self.publication_error + + +@pytest.fixture +def authority(monkeypatch: pytest.MonkeyPatch) -> _Authority: + return _Authority(monkeypatch) + + +@pytest.mark.asyncio +async def test_unsupported_authority_refuses_before_preparation_or_provider_work( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(settings, "storage_authority", "clickhouse") + prepare = AsyncMock(side_effect=AssertionError("append preparation started")) + client = Mock(side_effect=AssertionError("append acquired a client")) + embedder = Mock(side_effect=AssertionError("append acquired an embedder")) + monkeypatch.setattr(bulk_set, "handle_memory_bulk_set", prepare) + monkeypatch.setattr(bulk_set, "get_client", client) + monkeypatch.setattr(bulk_set, "get_embedder", embedder) + + with pytest.raises(ValidationError, match="Postgres"): + await bulk_set.handle_memory_append_batch(_input(("first", "hello"))) + + prepare.assert_not_awaited() + client.assert_not_called() + embedder.assert_not_called() + + +@pytest.mark.asyncio +async def test_equal_values_under_fresh_keys_remain_two_complete_occurrences( + authority: _Authority, +) -> None: + value = {"role": "user", "content": "hello", "metadata": {"sequence": [1, 2]}} + result = await bulk_set.handle_memory_append_batch(_input(("first", value), ("second", value))) + + assert len(authority.commits) == 1 + committed = authority.commits[0] + assert [r.memory_key for r in result.written] == ["first", "second"] + assert all(r.version > 0 for r in result.written) + assert len({r.memory_id for r in result.written}) == 2 + assert len(authority.rows) == 2 + for row, written in zip(committed.rows, result.written, strict=True): + assert authority.rows[identity_key(row)] == row + assert json.loads(row["value"]) == value + assert json.loads(row["value_json"]) == value + assert row["memory_type"] == "episodic" + assert (row["org"], row["namespace"], row["agent"], row["user_id"], row["session_id"]) == ( + "test", + "personal", + "cursor", + "u", + "s", + ) + assert row["version"] == written.version + assert row["memory_id"] == written.memory_id + assert row["source_class"] == "asserted" + assert row["actor"] == "process:append-test" + assert row["tombstoned_at"] is None + assert committed.targets == [pg_memory.MEMORY_ROW | pg_memory.AUDIT_ROW] * 2 + assert authority.ch.calls == [] + authority.embedder.assert_not_called() + + +@pytest.mark.asyncio +async def test_a_refused_second_entry_never_commits_the_valid_first_entry( + authority: _Authority, +) -> None: + authority.refused_value = json.dumps("blocked") + + with pytest.raises(ValidationError, match="refused an entry"): + await bulk_set.handle_memory_append_batch(_input(("first", "hello"), ("second", "blocked"))) + + assert authority.checked_values == [json.dumps("hello"), json.dumps("blocked")] + assert authority.commit_attempts == 0 + assert authority.rows == {} + assert authority.ch.calls == [] + + +@pytest.mark.asyncio +async def test_duplicate_keys_are_refused_before_preparing_either_occurrence( + authority: _Authority, +) -> None: + with pytest.raises(ValidationError, match="distinct keys"): + await bulk_set.handle_memory_append_batch(_input(("first", "hello"), ("first", "again"))) + + assert authority.checked_values == [] + assert authority.commit_attempts == 0 + assert authority.rows == {} + + +@pytest.mark.asyncio +@pytest.mark.parametrize("tombstoned", [False, True]) +async def test_existing_live_or_deleted_head_refuses_the_entire_append( + authority: _Authority, tombstoned: bool +) -> None: + head = _head(tombstoned=tombstoned) + key = ("test", "personal", "cursor", "u", "s", "second") + authority.lookup[key] = head + + with pytest.raises(ValidationError, match="must not already exist"): + await bulk_set.handle_memory_append_batch(_input(("first", "hello"), ("second", "again"))) + + assert authority.commit_attempts == 0 + assert authority.rows == {} + assert authority.lookup[key] is head + + +@pytest.mark.asyncio +async def test_unverifiable_freshness_never_becomes_a_first_write(authority: _Authority) -> None: + authority.lookup_error = RuntimeError("head read unavailable") + + with pytest.raises(ValidationError, match="verify fresh keys"): + await bulk_set.handle_memory_append_batch(_input(("first", "hello"))) + + assert authority.commit_attempts == 0 + assert authority.rows == {} + + +@pytest.mark.asyncio +@pytest.mark.parametrize("tombstoned", [False, True]) +async def test_a_head_appearing_at_locked_admission_is_not_overwritten( + authority: _Authority, tombstoned: bool +) -> None: + key = ("test", "personal", "cursor", "u", "s", "second") + raced = {"version": 2, "tombstoned_at": "deleted" if tombstoned else None} + authority.locked_heads[key] = raced + + with pytest.raises(ValidationError, match="must not already exist"): + await bulk_set.handle_memory_append_batch(_input(("first", "hello"), ("second", "again"))) + + assert authority.commit_attempts == 1 + assert authority.commits == [] + assert authority.rows == {} + assert authority.locked_heads[key] is raced + + +@pytest.mark.asyncio +async def test_a_committed_append_with_pending_publication_is_not_reported_as_rollback( + authority: _Authority, +) -> None: + authority.publication_error = pg_lease.PublicationPending() + + with pytest.raises(pg_lease.PublicationPending) as caught: + await bulk_set.handle_memory_append_batch(_input(("first", "hello"), ("second", "again"))) + + assert caught.value.extra["committed"] is True + assert authority.commit_attempts == 1 + assert len(authority.commits) == 1 + assert [row["memory_key"] for row in authority.rows.values()] == ["first", "second"] + + +@pytest.mark.asyncio +async def test_losing_pg_authority_before_insert_does_not_fall_back_to_clickhouse( + authority: _Authority, monkeypatch: pytest.MonkeyPatch +) -> None: + insert = bulk_set._bulk_insert + + async def lose_authority(ch: Any, state: bulk_rows._BulkWriteState) -> None: + monkeypatch.setattr(settings, "storage_authority", "clickhouse") + await insert(ch, state) + + monkeypatch.setattr(bulk_set, "_bulk_insert", lose_authority) + + with pytest.raises(ValidationError, match="Postgres"): + await bulk_set.handle_memory_append_batch(_input(("first", "hello"))) + + assert authority.commit_attempts == 0 + assert authority.rows == {} + assert authority.ch.calls == [] + + +def test_append_commit_can_bind_its_idempotency_claim(monkeypatch: pytest.MonkeyPatch) -> None: + claim = ("test", "request-key", 7) + monkeypatch.setattr(idempotency, "_bound", ContextVar("append_test_claim", default=claim)) + operation = ContextVar("append_test_operation", default="memory.append_batch") + monkeypatch.setattr(idempotency, "_bound_op", operation) + + assert idempotency.bound_key() == claim + operation.set("profile.set") + assert idempotency.bound_key() is None + + +@pytest.mark.asyncio +async def test_committed_replay_preserves_both_occurrence_receipts() -> None: + revisions = [ + { + "namespace": "personal", + "seq": i, + "memory_key": key, + "memory_id": f"id-{i}", + "version": 10 + i, + "content_sha256": "a" * 64, + } + for i, key in enumerate(("first", "second"), start=1) + ] + conn = Mock(fetch=AsyncMock(return_value=revisions)) + hit = await idempotency._committed_hit( + conn, + { + "operation": "memory.append_batch", + "revisions": [["test", "personal", 1], ["test", "personal", 2]], + }, + ) + + assert hit.status_code == 200 + assert hit.body == { + "written": [ + {"memory_key": "first", "version": 11, "memory_id": "id-1"}, + {"memory_key": "second", "version": 12, "memory_id": "id-2"}, + ], + "idempotent_replay": "committed", + } + assert hit.headers == {"X-Loom-Idempotent-Replay": "committed"} + + +@pytest.mark.asyncio +async def test_committed_replay_does_not_expose_an_erased_occurrence() -> None: + conn = Mock(fetch=AsyncMock(return_value=[])) + + with pytest.raises(NotFoundError, match="erased"): + await idempotency._committed_hit( + conn, {"operation": "memory.append_batch", "revisions": [["test", "personal", 1]]} + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("source", ["derived", "external", "imported"]) +async def test_append_preserves_a_declared_provenance_downgrade( + authority: _Authority, source: str +) -> None: + input = _input(("first", "hello")) + input.entries[0].source_class = source + input.entries[0].source_trace = "native-history" + await bulk_set.handle_memory_append_batch(input) + row = next(iter(authority.rows.values())) + assert row["source_class"] == source + assert row["source_trace"] == "native-history" + assert row["actor"] == "process:append-test" + + +@pytest.mark.asyncio +async def test_an_invalid_second_source_declaration_aborts_the_whole_batch( + authority: _Authority, +) -> None: + input = _input(("first", "hello"), ("second", "again")) + input.entries[1].source_class = "asserted" + with pytest.raises(ValidationError): + await bulk_set.handle_memory_append_batch(input) + assert authority.rows == {} and authority.commit_attempts == 0 diff --git a/tests/unit/test_pg_authority.py b/tests/unit/test_pg_authority.py index 218a7ca2ce..760d19ab05 100644 --- a/tests/unit/test_pg_authority.py +++ b/tests/unit/test_pg_authority.py @@ -1406,7 +1406,12 @@ def test_only_an_op_whose_answer_replays_marks_its_key_committed() -> None: the committed replay would answer in memory.set's shape.""" from server.core import idempotency - for op, marks in (("memory.set", True), ("memory.bulk_set", True), ("profile.set", False)): + for op, marks in ( + ("memory.set", True), + ("memory.bulk_set", True), + ("memory.append_batch", True), + ("profile.set", False), + ): idempotency._bound.set(("o", "k", 1)) idempotency._bound_op.set(op) assert (idempotency.bound_key() is not None) is marks, op @@ -6901,7 +6906,9 @@ async def _fence(items: Any, *, confirmed: bool = True, **_k: Any) -> None: monkeypatch.setattr(bulk_set, "_commit_batch", _commit) monkeypatch.setattr(bulk_set, "_land_batch", _land) monkeypatch.setattr(high_fence, "fence_committed", _fence) - state = SimpleNamespace(memory_rows=[{"org": "o", "namespace": "n"}], embedding_rows=[]) + state = SimpleNamespace( + memory_rows=[{"org": "o", "namespace": "n"}], embedding_rows=[], strict_append=False + ) state.results = [] with contextlib.suppress(pg.PostgresUnavailableError, asyncio.CancelledError): await bulk_set._bulk_insert(None, state) @@ -6936,7 +6943,7 @@ async def _announce(state: Any) -> None: monkeypatch.setattr(bulk_set, "_high_winners", lambda state: winners) monkeypatch.setattr(bulk_set, "fence_high", _fence) monkeypatch.setattr(bulk_set, "_bulk_announce", _announce) - state = SimpleNamespace(memory_rows=[{"org": "o", "namespace": "n"}]) + state = SimpleNamespace(memory_rows=[{"org": "o", "namespace": "n"}], strict_append=False) state.embedding_rows = [] with pytest.raises(RuntimeError, match="lift failed"): await bulk_set._land_batch(None, state, None, {}) @@ -7058,6 +7065,7 @@ async def _no_reach(_state: Any) -> list[Any]: key = {"org": "o", "namespace": "n", "agent": "a", "user_id": "-", "session_id": "-"} first, last = key | {"memory_key": "k", "version": 1}, key | {"memory_key": "k", "version": 2} state = SimpleNamespace( + strict_append=False, memory_rows=[first, last], embedding_rows=[ r | {"embedding": [0.1]} for r in ([first, last] if both_embedded else [last]) diff --git a/tests/unit/test_read_scope_admission.py b/tests/unit/test_read_scope_admission.py index 57700553ea..61bee06cf1 100644 --- a/tests/unit/test_read_scope_admission.py +++ b/tests/unit/test_read_scope_admission.py @@ -267,6 +267,7 @@ def test_a_write_under_an_unresolved_scope_is_refused() -> None: "knowledge.source.push_configure", "knowledge.synthesize", "knowledge.upsert", + "memory.append_batch", "memory.bulk_set", "memory.delete", "memory.import", diff --git a/tests/unit/test_session_content_senders.py b/tests/unit/test_session_content_senders.py index 5b16402faf..16d3632473 100644 --- a/tests/unit/test_session_content_senders.py +++ b/tests/unit/test_session_content_senders.py @@ -40,6 +40,7 @@ "tracker.duplicate_check", "memory.search", "memory.set", + "memory.append_batch", "memory.bulk_set", "knowledge.search", }