From 16a4dad363a0be1a713fda0147ea634a3dc127cf Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Mon, 9 Nov 2026 01:21:00 +0000 Subject: [PATCH 01/13] Under Postgres, a commit stores each row without the fields at their loom.memory default Co-Authored-By: Claude Opus 5.5 Signed-off-by: zlareb1 --- docs/architecture/storage-authority.md | 4 +++- server/memory/store/pg_memory.py | 10 ++++++++-- tests/integration/test_pg_authority.py | 5 +++-- tests/unit/test_pg_authority.py | 9 +++------ 4 files changed, 17 insertions(+), 11 deletions(-) diff --git a/docs/architecture/storage-authority.md b/docs/architecture/storage-authority.md index 603569d96f..403f32f655 100644 --- a/docs/architecture/storage-authority.md +++ b/docs/architecture/storage-authority.md @@ -97,7 +97,9 @@ A stored row (a head, or a revision's payload) may leave out a field at its (`rows.FIRST_VERSION_DEFAULTS`, `pg_memory.SPARSE_DEFAULTS`): every reader takes a missing field as that default, `decode_row` in Python and `coalesce` (or a test that holds for NULL) in SQL. Every other field is always stored, a default-valued one -included. This build stores every field. +included. A commit stores each row without the fields at their default +(`pg_memory.sparse_row`), which keeps it smaller: most of a row's fields hold their +default, and every stored field repeats its name. A memory id is not unique within an org: a namespace restored from another's snapshot keeps the memory ids under newer versions. Every head read by id therefore names its diff --git a/server/memory/store/pg_memory.py b/server/memory/store/pg_memory.py index 214941959a..d6b33734eb 100644 --- a/server/memory/store/pg_memory.py +++ b/server/memory/store/pg_memory.py @@ -111,6 +111,12 @@ def encode_row(row: Mapping[str, Any]) -> dict[str, Any]: SPARSE_DEFAULTS: Mapping[str, Any] = FIRST_VERSION_DEFAULTS +def sparse_row(encoded: Mapping[str, Any]) -> dict[str, Any]: + """An encoded row as it is stored: without the fields at their default + (`SPARSE_DEFAULTS`), which every reader fills back (`decode_row`).""" + return {c: v for c, v in encoded.items() if c not in SPARSE_DEFAULTS or v != SPARSE_DEFAULTS[c]} + + def decode_row(stored: Mapping[str, Any]) -> dict[str, Any]: """The insertable loom.memory row back from its JSON, each field it leaves out at its default (`SPARSE_DEFAULTS`), as ClickHouse fills it.""" @@ -727,7 +733,7 @@ async def commit_versions( [int(r["version"]) for r in head_rows], [seqs[heads[k]][2] for k in keys], [str(r.get("content_sha256") or "") for r in head_rows], - head_rows, + [sparse_row(r) for r in head_rows], ) await conn.execute( _INSERT_REVISIONS, @@ -741,7 +747,7 @@ async def commit_versions( [str(r.get("actor") or "") for r in stored], [str(r.get("source_class") or "") for r in stored], [request_id] * len(stored), - stored, + [sparse_row(r) for r in stored], [encode_audit(a) for a in audits], ) await conn.execute( diff --git a/tests/integration/test_pg_authority.py b/tests/integration/test_pg_authority.py index a4eac79b20..4781ee2b4d 100644 --- a/tests/integration/test_pg_authority.py +++ b/tests/integration/test_pg_authority.py @@ -324,7 +324,7 @@ def sparse(stored: dict[str, Any]) -> dict[str, Any]: stored = await pg_conn.fetchval( f"SELECT row FROM loom.{table} WHERE org = $1 AND namespace = $2", org, ns ) - assert len(sparse(stored)) < len(stored) + assert sparse(stored) == stored # a commit stores the row without its defaults full[table] = stored await pg_conn.execute( f"UPDATE loom.{table} SET row = $3 WHERE org = $1 AND namespace = $2", @@ -3156,7 +3156,8 @@ async def test_a_same_key_write_in_another_session_retires_a_first_write_at_once statuses = { r["session_id"]: r["status"] for r in await pg_conn.fetch( - "SELECT session_id, row->>'status' AS status FROM loom.memory_current " + "SELECT session_id, coalesce(row->>'status', 'active') AS status " + "FROM loom.memory_current " "WHERE org = $1 AND namespace = $2", org, ns, diff --git a/tests/unit/test_pg_authority.py b/tests/unit/test_pg_authority.py index 86437a5150..64e5e701e3 100644 --- a/tests/unit/test_pg_authority.py +++ b/tests/unit/test_pg_authority.py @@ -132,12 +132,9 @@ def test_a_row_that_leaves_out_its_default_fields_reads_as_the_full_row() -> Non now=datetime(2026, 1, 2, 3, 4, 5), ) ) - sparse = { - c: v - for c, v in full.items() - if c not in pg_memory.SPARSE_DEFAULTS or v != pg_memory.SPARSE_DEFAULTS[c] - } - assert len(sparse) < len(full) + sparse = pg_memory.sparse_row(full) + assert len(sparse) < len(full) and sparse["value"] == full["value"] + assert all(c not in sparse for c, v in pg_memory.SPARSE_DEFAULTS.items() if full[c] == v) assert pg_memory.decode_row(sparse) == pg_memory.decode_row(full) first, second = pg_memory.decode_row(sparse), pg_memory.decode_row(sparse) first["tags"].append("x") From 1c76937767d02981af4c35f074bc51d392a20f60 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Mon, 9 Nov 2026 22:26:06 +0000 Subject: [PATCH 02/13] The sparse-row test serves a full head first and the sparse one after, and unit tests pin falsy values and the fields SQL reads as stored Co-Authored-By: Claude Opus 5.5 Signed-off-by: zlareb1 --- docs/architecture/storage-authority.md | 4 ++-- server/memory/store/pg_memory.py | 7 +++--- tests/integration/test_pg_authority.py | 32 +++++++++++++++----------- tests/unit/test_pg_authority.py | 19 +++++++++++++++ 4 files changed, 43 insertions(+), 19 deletions(-) diff --git a/docs/architecture/storage-authority.md b/docs/architecture/storage-authority.md index 403f32f655..af1273a965 100644 --- a/docs/architecture/storage-authority.md +++ b/docs/architecture/storage-authority.md @@ -73,7 +73,7 @@ and every tenant index leads with `org`. | --------------------------- | ---------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | `loom.authority` | singleton | the recorded authority, its epoch and the deployment id | | `loom.scope_generations` | `(org, scope)` | the last revision number of a namespace, or a policy generation (`scope = '*'`, org `''` for the global one) | -| `loom.memory_current` | `(org, namespace, agent, user_id, session_id, memory_key)` | the head of each key: the complete `loom.memory` row as JSON, with `memory_id`, `version`, `seq`, `content_sha256` beside it | +| `loom.memory_current` | `(org, namespace, agent, user_id, session_id, memory_key)` | the head of each key: its `loom.memory` row as JSON, without its default fields, with `memory_id`, `version`, `seq`, `content_sha256` beside it | | `loom.memory_revisions` | `(org, namespace, seq)` | every committed row, append-only (a redaction scrubs one in place), with its hash-only audit row and its metadata in typed columns ([revision metadata](#revision-metadata)) | | `loom.memory_deliveries` | `(org, namespace, seq)` | the targets a revision still owes ClickHouse (bits: 1 memory row, 2 search document, 4 audit row, 8 a deleted decision's record, 16 the old vectors), its lease and fencing token; deleted when the last target lands | | `loom.idempotency` | `(org, key_hash)` | an op's request digest, its state (`running`, `committed`, `done`), the revisions it committed, its answer, a lease and its token | @@ -85,7 +85,7 @@ and every tenant index leads with `org`. | `loom.erased_documents` | `(org, namespace, doc_id)` | deletion evidence: the `doc_id` of every version of each document an erasure removed, ClickHouse's included, refused at a commit; no content, kept `LOOM_PG_ERASURE_EVIDENCE_HOURS` | A head is the row `loom.memory FINAL` would serve: a commit replaces it only when its -version is not older, as `ReplacingMergeTree(version)` decides. Holding the whole row +version is not older, as `ReplacingMergeTree(version)` decides. Holding the row keeps every one of the write paths that build `loom.memory` rows unchanged (`server/memory/store/rows.py`): they all reach the commit through the one insert funnel, `insert_memory_versions`. The row keeps ClickHouse's precision (whole-second diff --git a/server/memory/store/pg_memory.py b/server/memory/store/pg_memory.py index d6b33734eb..9b6f12eb29 100644 --- a/server/memory/store/pg_memory.py +++ b/server/memory/store/pg_memory.py @@ -1,8 +1,9 @@ """Every Postgres statement of the memory authority (the Postgres storage profile). -`loom.memory_current` holds the head of each key as the complete loom.memory row, -which is what `loom.memory FINAL` would serve: a row replaces the head only when -its version is not older, as ReplacingMergeTree(version) decides. Each commit +`loom.memory_current` holds the head of each key as its loom.memory row, without the +fields at their default (`sparse_row`), which is what `loom.memory FINAL` would serve: +a row replaces the head only when its version is not older, as +ReplacingMergeTree(version) decides. Each commit also appends the rows to `loom.memory_revisions`, numbered per namespace by the namespace's generation, and records what they owe ClickHouse in `loom.memory_deliveries`. The transaction is short and does no other I/O. diff --git a/tests/integration/test_pg_authority.py b/tests/integration/test_pg_authority.py index 4781ee2b4d..d38ad36404 100644 --- a/tests/integration/test_pg_authority.py +++ b/tests/integration/test_pg_authority.py @@ -298,7 +298,7 @@ async def test_a_revisions_metadata_is_kept_from_its_row_and_outlives_it( async def test_a_head_that_leaves_out_its_default_fields_is_served_as_the_full_one( call: Any, pg_conn: Any, org: str ) -> None: - """A stored row may leave out a field at its loom.memory default: point reads, + """A commit stores a row without the fields at their loom.memory default: point reads, listing and search serve such a head as they serve the full one, and the SQL readers of its fields (the journal's revisions, the cross-session heads, the usage fold, the erasure export) take a missing one as its default.""" @@ -313,25 +313,29 @@ async def test_a_head_that_leaves_out_its_default_fields_is_served_as_the_full_o ).status_code == 200 await _published(pg_conn, org) get = {"namespace": ns, "agent": "a", "memory_key": "k"} - before = (await call("memory.get", get)).json() + tables = ("memory_current", "memory_revisions") - def sparse(stored: dict[str, Any]) -> dict[str, Any]: - defaults = pg_memory.SPARSE_DEFAULTS - return {c: v for c, v in stored.items() if c not in defaults or v != defaults[c]} + async def store(table: str, row: dict[str, Any]) -> None: + await pg_conn.execute( + f"UPDATE loom.{table} SET row = $3 WHERE org = $1 AND namespace = $2", org, ns, row + ) + # The row as a build storing every field would: the defaults filled back in. full: dict[str, dict[str, Any]] = {} - for table in ("memory_current", "memory_revisions"): + for table in tables: stored = await pg_conn.fetchval( f"SELECT row FROM loom.{table} WHERE org = $1 AND namespace = $2", org, ns ) - assert sparse(stored) == stored # a commit stores the row without its defaults - full[table] = stored - await pg_conn.execute( - f"UPDATE loom.{table} SET row = $3 WHERE org = $1 AND namespace = $2", - org, - ns, - sparse(stored), - ) + assert pg_memory.sparse_row(stored) == stored # the commit left its defaults out + defaults = { + c: list(v) if isinstance(v, list) else v for c, v in pg_memory.SPARSE_DEFAULTS.items() + } + full[table] = {**defaults, **stored} + assert len(full[table]) > len(stored) + await store(table, full[table]) + before = (await call("memory.get", get)).json() + for table in tables: + await store(table, pg_memory.sparse_row(full[table])) assert (await call("memory.get", get)).json() == before listed = (await call("memory.list", {"namespace": ns, "agent": "a"})).json() assert [e["memory_key"] for e in listed["entries"]] == ["k"], listed diff --git a/tests/unit/test_pg_authority.py b/tests/unit/test_pg_authority.py index 64e5e701e3..da5556d5bd 100644 --- a/tests/unit/test_pg_authority.py +++ b/tests/unit/test_pg_authority.py @@ -139,6 +139,25 @@ def test_a_row_that_leaves_out_its_default_fields_reads_as_the_full_row() -> Non first, second = pg_memory.decode_row(sparse), pg_memory.decode_row(sparse) first["tags"].append("x") assert second["tags"] == [] + # A value other than its default is stored, a falsy one included. + changed = { + "evidence_count": 0, + "pinned": 1, + "tags": ["x"], + "status": "superseded", + "char_limit": 0, + } + kept = pg_memory.sparse_row({**full, **changed}) + assert {c: kept[c] for c in changed} == changed + assert pg_memory.decode_row(kept) == pg_memory.decode_row({**full, **changed}) + + +def test_the_fields_sql_reads_as_stored_are_never_left_out() -> None: + """The profile reach matches on `row->>'agent'` and `row->>'memory_type'` + (`memory_revisions_profile`, `_profile_built_from`), and other statements read the key, + the value and the creation time as stored: none has a default a row may leave out.""" + read_as_stored = {"agent", "memory_type", "memory_key", "session_id", "value", "created_at"} + assert not read_as_stored & pg_memory.SPARSE_DEFAULTS.keys() def test_a_row_round_trips_at_clickhouse_precision() -> None: From 5c1b703c398464a6c9a04537f0a5be439feaf450 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Mon, 9 Nov 2026 22:39:17 +0000 Subject: [PATCH 03/13] The sparse-row test fills the full row with the module's own decode, the pin covers the export's confidence, and the module docstring reads as the row FINAL serves Co-Authored-By: Claude Opus 5.5 Signed-off-by: zlareb1 --- server/memory/store/pg_memory.py | 9 ++++----- tests/integration/test_pg_authority.py | 5 +---- tests/unit/test_pg_authority.py | 17 +++++++++++++---- 3 files changed, 18 insertions(+), 13 deletions(-) diff --git a/server/memory/store/pg_memory.py b/server/memory/store/pg_memory.py index 9b6f12eb29..18f50790ca 100644 --- a/server/memory/store/pg_memory.py +++ b/server/memory/store/pg_memory.py @@ -1,10 +1,9 @@ """Every Postgres statement of the memory authority (the Postgres storage profile). -`loom.memory_current` holds the head of each key as its loom.memory row, without the -fields at their default (`sparse_row`), which is what `loom.memory FINAL` would serve: -a row replaces the head only when its version is not older, as -ReplacingMergeTree(version) decides. Each commit -also appends the rows to `loom.memory_revisions`, numbered per namespace by the +`loom.memory_current` holds the head of each key, the row `loom.memory FINAL` would +serve, stored without the fields at their default (`sparse_row`): a row replaces the +head only when its version is not older, as ReplacingMergeTree(version) decides. Each +commit also appends the rows to `loom.memory_revisions`, numbered per namespace by the namespace's generation, and records what they owe ClickHouse in `loom.memory_deliveries`. The transaction is short and does no other I/O. diff --git a/tests/integration/test_pg_authority.py b/tests/integration/test_pg_authority.py index d38ad36404..efb8936342 100644 --- a/tests/integration/test_pg_authority.py +++ b/tests/integration/test_pg_authority.py @@ -327,10 +327,7 @@ async def store(table: str, row: dict[str, Any]) -> None: f"SELECT row FROM loom.{table} WHERE org = $1 AND namespace = $2", org, ns ) assert pg_memory.sparse_row(stored) == stored # the commit left its defaults out - defaults = { - c: list(v) if isinstance(v, list) else v for c, v in pg_memory.SPARSE_DEFAULTS.items() - } - full[table] = {**defaults, **stored} + full[table] = pg_memory.encode_row(pg_memory.decode_row(stored)) assert len(full[table]) > len(stored) await store(table, full[table]) before = (await call("memory.get", get)).json() diff --git a/tests/unit/test_pg_authority.py b/tests/unit/test_pg_authority.py index da5556d5bd..b0def374f7 100644 --- a/tests/unit/test_pg_authority.py +++ b/tests/unit/test_pg_authority.py @@ -152,11 +152,20 @@ def test_a_row_that_leaves_out_its_default_fields_reads_as_the_full_row() -> Non assert pg_memory.decode_row(kept) == pg_memory.decode_row({**full, **changed}) -def test_the_fields_sql_reads_as_stored_are_never_left_out() -> None: +def test_the_fields_read_as_stored_are_never_left_out() -> None: """The profile reach matches on `row->>'agent'` and `row->>'memory_type'` - (`memory_revisions_profile`, `_profile_built_from`), and other statements read the key, - the value and the creation time as stored: none has a default a row may leave out.""" - read_as_stored = {"agent", "memory_type", "memory_key", "session_id", "value", "created_at"} + (`memory_revisions_profile`, `_profile_built_from`), other statements read the key, the + value and the creation time as stored, and the erasure export reads the type, value + and confidence from the stored JSON: none has a default a row may leave out.""" + read_as_stored = { + "agent", + "memory_type", + "memory_key", + "session_id", + "value", + "created_at", + "confidence", + } assert not read_as_stored & pg_memory.SPARSE_DEFAULTS.keys() From 14a10f1bfad1abe527cf3a80d75b90acfd8dd3bf Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Mon, 9 Nov 2026 22:45:06 +0000 Subject: [PATCH 04/13] The sparse-row unit test asserts its two checks apart, and the always-stored pin names the session it covers Co-Authored-By: Claude Opus 5.5 Signed-off-by: zlareb1 --- tests/unit/test_pg_authority.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/unit/test_pg_authority.py b/tests/unit/test_pg_authority.py index b0def374f7..0af5b2cea4 100644 --- a/tests/unit/test_pg_authority.py +++ b/tests/unit/test_pg_authority.py @@ -133,7 +133,8 @@ def test_a_row_that_leaves_out_its_default_fields_reads_as_the_full_row() -> Non ) ) sparse = pg_memory.sparse_row(full) - assert len(sparse) < len(full) and sparse["value"] == full["value"] + assert len(sparse) < len(full) + assert sparse["value"] == full["value"] assert all(c not in sparse for c, v in pg_memory.SPARSE_DEFAULTS.items() if full[c] == v) assert pg_memory.decode_row(sparse) == pg_memory.decode_row(full) first, second = pg_memory.decode_row(sparse), pg_memory.decode_row(sparse) @@ -154,8 +155,8 @@ def test_a_row_that_leaves_out_its_default_fields_reads_as_the_full_row() -> Non def test_the_fields_read_as_stored_are_never_left_out() -> None: """The profile reach matches on `row->>'agent'` and `row->>'memory_type'` - (`memory_revisions_profile`, `_profile_built_from`), other statements read the key, the - value and the creation time as stored, and the erasure export reads the type, value + (`memory_revisions_profile`, `_profile_built_from`), other statements read the key and + session, the value and the creation time as stored, and the erasure export reads the type, value and confidence from the stored JSON: none has a default a row may leave out.""" read_as_stored = { "agent", From 82c3696763806070346017bae7ee7605e7e95e47 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Mon, 9 Nov 2026 22:57:45 +0000 Subject: [PATCH 05/13] The sparse-row doc says the in-place writers may store a default, and the pin's docstring wraps evenly Co-Authored-By: Claude Opus 5.5 Signed-off-by: zlareb1 --- docs/architecture/storage-authority.md | 4 +++- tests/unit/test_pg_authority.py | 5 +++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/docs/architecture/storage-authority.md b/docs/architecture/storage-authority.md index af1273a965..628a2b8da9 100644 --- a/docs/architecture/storage-authority.md +++ b/docs/architecture/storage-authority.md @@ -99,7 +99,9 @@ missing field as that default, `decode_row` in Python and `coalesce` (or a test holds for NULL) in SQL. Every other field is always stored, a default-valued one included. A commit stores each row without the fields at their default (`pg_memory.sparse_row`), which keeps it smaller: most of a row's fields hold their -default, and every stored field repeats its name. +default, and every stored field repeats its name. The writers that change a stored row in +place (the usage fold, a redaction) merge fields into it, and may store a field at its +default. A memory id is not unique within an org: a namespace restored from another's snapshot keeps the memory ids under newer versions. Every head read by id therefore names its diff --git a/tests/unit/test_pg_authority.py b/tests/unit/test_pg_authority.py index 0af5b2cea4..0f96429266 100644 --- a/tests/unit/test_pg_authority.py +++ b/tests/unit/test_pg_authority.py @@ -156,8 +156,9 @@ def test_a_row_that_leaves_out_its_default_fields_reads_as_the_full_row() -> Non def test_the_fields_read_as_stored_are_never_left_out() -> None: """The profile reach matches on `row->>'agent'` and `row->>'memory_type'` (`memory_revisions_profile`, `_profile_built_from`), other statements read the key and - session, the value and the creation time as stored, and the erasure export reads the type, value - and confidence from the stored JSON: none has a default a row may leave out.""" + session, the value and the creation time as stored, and the erasure export reads the + type, value and confidence from the stored JSON: none has a default a row may leave + out.""" read_as_stored = { "agent", "memory_type", From 5cda0c24fdd5d990595bf8c89b19383b6a002e68 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Mon, 9 Nov 2026 23:06:32 +0000 Subject: [PATCH 06/13] The sparse-row test's docstring says what each read checks Co-Authored-By: Claude Opus 5.5 Signed-off-by: zlareb1 --- tests/integration/test_pg_authority.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tests/integration/test_pg_authority.py b/tests/integration/test_pg_authority.py index efb8936342..9a533ed2a7 100644 --- a/tests/integration/test_pg_authority.py +++ b/tests/integration/test_pg_authority.py @@ -298,10 +298,11 @@ async def test_a_revisions_metadata_is_kept_from_its_row_and_outlives_it( async def test_a_head_that_leaves_out_its_default_fields_is_served_as_the_full_one( call: Any, pg_conn: Any, org: str ) -> None: - """A commit stores a row without the fields at their loom.memory default: point reads, - listing and search serve such a head as they serve the full one, and the SQL readers - of its fields (the journal's revisions, the cross-session heads, the usage fold, the - erasure export) take a missing one as its default.""" + """A commit stores a row without the fields at their loom.memory default: a point read + serves such a head as it serves the full one, listing and search find it and bulk_get + serves its value, and the SQL readers of its fields (the journal's revisions, the + cross-session heads, the usage fold, the erasure export) take a missing one as its + default.""" from server.core import pg from server.erasure.store.exports import export_memory_heads from server.identity.domain.read_scope import WHOLE_NAMESPACE From 218e06c0ef9211c892ab7cd2ee7aab6c34dc99a8 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Mon, 9 Nov 2026 23:16:33 +0000 Subject: [PATCH 07/13] A commit builds each row's stored form once, and the sparse-row test's docstring names its Python readers as readers Co-Authored-By: Claude Opus 5.5 Signed-off-by: zlareb1 --- server/memory/store/pg_memory.py | 5 +++-- tests/integration/test_pg_authority.py | 5 ++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/server/memory/store/pg_memory.py b/server/memory/store/pg_memory.py index 18f50790ca..2258cfa865 100644 --- a/server/memory/store/pg_memory.py +++ b/server/memory/store/pg_memory.py @@ -726,6 +726,7 @@ async def commit_versions( next_seq[s] += 1 seqs.append((s[0], s[1], next_seq[s])) head_rows = [stored[heads[k]] for k in keys] + sparse = [sparse_row(r) for r in stored] await conn.execute( _UPSERT_HEADS, *_cols(head_rows, "org", "namespace", "agent", "user_id", "session_id", "memory_key"), @@ -733,7 +734,7 @@ async def commit_versions( [int(r["version"]) for r in head_rows], [seqs[heads[k]][2] for k in keys], [str(r.get("content_sha256") or "") for r in head_rows], - [sparse_row(r) for r in head_rows], + [sparse[heads[k]] for k in keys], ) await conn.execute( _INSERT_REVISIONS, @@ -747,7 +748,7 @@ async def commit_versions( [str(r.get("actor") or "") for r in stored], [str(r.get("source_class") or "") for r in stored], [request_id] * len(stored), - [sparse_row(r) for r in stored], + sparse, [encode_audit(a) for a in audits], ) await conn.execute( diff --git a/tests/integration/test_pg_authority.py b/tests/integration/test_pg_authority.py index 9a533ed2a7..e117dfd9d4 100644 --- a/tests/integration/test_pg_authority.py +++ b/tests/integration/test_pg_authority.py @@ -300,9 +300,8 @@ async def test_a_head_that_leaves_out_its_default_fields_is_served_as_the_full_o ) -> None: """A commit stores a row without the fields at their loom.memory default: a point read serves such a head as it serves the full one, listing and search find it and bulk_get - serves its value, and the SQL readers of its fields (the journal's revisions, the - cross-session heads, the usage fold, the erasure export) take a missing one as its - default.""" + serves its value, and the journal's revisions, the cross-session heads, the usage fold + and the erasure export take a missing field as its default.""" from server.core import pg from server.erasure.store.exports import export_memory_heads from server.identity.domain.read_scope import WHOLE_NAMESPACE From 7ee43a9cdb4bada7267199dd76bb995ed056c10d Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Mon, 9 Nov 2026 23:21:04 +0000 Subject: [PATCH 08/13] A commit builds the stored rows before its transaction, so no lock waits on them Co-Authored-By: Claude Opus 5.5 Signed-off-by: zlareb1 --- server/memory/store/pg_memory.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/server/memory/store/pg_memory.py b/server/memory/store/pg_memory.py index 2258cfa865..f8eb8f5058 100644 --- a/server/memory/store/pg_memory.py +++ b/server/memory/store/pg_memory.py @@ -667,6 +667,8 @@ async def commit_versions( whatever refusal the head at commit calls for.""" started = time.monotonic() stored = [encode_row(r) for r in rows] + # Built before the transaction: no lock waits on it. + sparse = [sparse_row(r) for r in stored] if any(r["namespace"] in POLICY_SCOPE_NAMES for r in stored): # A namespace's generation shares its key with the policy generations. raise ValidationError("'*' and '' are not namespace names") @@ -726,7 +728,6 @@ async def commit_versions( next_seq[s] += 1 seqs.append((s[0], s[1], next_seq[s])) head_rows = [stored[heads[k]] for k in keys] - sparse = [sparse_row(r) for r in stored] await conn.execute( _UPSERT_HEADS, *_cols(head_rows, "org", "namespace", "agent", "user_id", "session_id", "memory_key"), From 6b7120d2de1e57702d04b8f0825bbbeb4c8ad74a Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Mon, 9 Nov 2026 23:26:42 +0000 Subject: [PATCH 09/13] The commit builds its stored rows after the namespace check, without a comment on where Co-Authored-By: Claude Opus 5.5 Signed-off-by: zlareb1 --- server/memory/store/pg_memory.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/server/memory/store/pg_memory.py b/server/memory/store/pg_memory.py index f8eb8f5058..db96f8a8dc 100644 --- a/server/memory/store/pg_memory.py +++ b/server/memory/store/pg_memory.py @@ -667,11 +667,10 @@ async def commit_versions( whatever refusal the head at commit calls for.""" started = time.monotonic() stored = [encode_row(r) for r in rows] - # Built before the transaction: no lock waits on it. - sparse = [sparse_row(r) for r in stored] if any(r["namespace"] in POLICY_SCOPE_NAMES for r in stored): # A namespace's generation shares its key with the policy generations. raise ValidationError("'*' and '' are not namespace names") + sparse = [sparse_row(r) for r in stored] per_row = [targets] * len(stored) if isinstance(targets, int) else list(targets) per_row = [ t | (DECISION_RECORD if deleted_decision(r) else 0) From 6c1ddd7f7fd35470ad7d2ee14cddb915eba2a333 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Fri, 20 Nov 2026 20:28:23 +0000 Subject: [PATCH 10/13] Align the sparse-head table entry with the document formatter Signed-off-by: zlareb1 --- docs/architecture/storage-authority.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/architecture/storage-authority.md b/docs/architecture/storage-authority.md index 0c1371881a..334575a76d 100644 --- a/docs/architecture/storage-authority.md +++ b/docs/architecture/storage-authority.md @@ -78,7 +78,7 @@ that moves it. | --------------------------- | ---------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | `loom.authority` | singleton | the recorded authority, its epoch and the deployment id | | `loom.scope_generations` | `(org, scope)` | the last revision number of a namespace, or a policy generation (`scope = '*'`, org `''` for the global one) | -| `loom.memory_current` | `(org, namespace, agent, user_id, session_id, memory_key)` | the head of each key: its `loom.memory` row as JSON, without its default fields, with `memory_id`, `version`, `seq`, `content_sha256` beside it | +| `loom.memory_current` | `(org, namespace, agent, user_id, session_id, memory_key)` | the head of each key: its `loom.memory` row as JSON, without its default fields, with `memory_id`, `version`, `seq`, `content_sha256` beside it | | `loom.memory_revisions` | `(org, namespace, seq)` | every committed row, append-only (a redaction scrubs one in place, and with `LOOM_PG_PAYLOAD_CLEAR_ENABLED` the clearing pass clears a delivered one's payload past the horizon: [payload clearing](#payload-clearing)), with its hash-only audit row and its metadata in typed columns ([revision metadata](#revision-metadata)) | | `loom.memory_deliveries` | `(org, namespace, seq)` | the targets a revision still owes ClickHouse (bits: 1 memory row, 2 search document, 4 audit row, 8 a deleted decision's record, 16 the old vectors), its lease and fencing token; deleted when the last target lands | | `loom.idempotency` | `(org, key_hash)` | an op's request digest, its state (`running`, `committed`, `done`), the revisions it committed, its answer, a lease and its token | From f694f415eaa3a93d46ce67a9a7954f215d1297f0 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Fri, 20 Nov 2026 20:53:59 +0000 Subject: [PATCH 11/13] Decode sparse authority rows in the Cell correctness verifier Compare full logical heads and revisions through the existing decoder so omitted defaults retain the same verification and fault checks. Signed-off-by: zlareb1 --- bench/cell_trigger_gate.py | 7 +++++-- tests/unit/test_bench_cell_trigger_gate.py | 20 ++++++++++++++------ 2 files changed, 19 insertions(+), 8 deletions(-) diff --git a/bench/cell_trigger_gate.py b/bench/cell_trigger_gate.py index 53ef3dddf5..a84cb9e34f 100644 --- a/bench/cell_trigger_gate.py +++ b/bench/cell_trigger_gate.py @@ -1258,6 +1258,8 @@ async def _absent(ch: Any, block: Block) -> None: async def _verify( conn: Any, ch: Any, args: argparse.Namespace, identities: Sequence[RequestRow] | Block ) -> None: + from server.memory.store import pg_memory + block = identities if isinstance(identities, Block) else None wanted = identities.rows if isinstance(identities, Block) else [asdict(r) for r in identities] for offset in range(0, len(wanted), 100): @@ -1309,9 +1311,10 @@ async def _verify( and record["last_error"] in (None, ""), "delivery settled without failure", ) - head, revision, audit = ( - json.loads(record[name]) for name in ("head", "revision", "audit") + head, revision = ( + pg_memory.decode_row(json.loads(record[name])) for name in ("head", "revision") ) + audit = json.loads(record["audit"]) _compare(head, revision, tuple(head), "authority full row") if block: _compare(expected, head, tuple(expected), "authority staged row") diff --git a/tests/unit/test_bench_cell_trigger_gate.py b/tests/unit/test_bench_cell_trigger_gate.py index 2a76f58c08..f058e32a2d 100644 --- a/tests/unit/test_bench_cell_trigger_gate.py +++ b/tests/unit/test_bench_cell_trigger_gate.py @@ -1084,6 +1084,7 @@ def sync(_fd): @pytest.mark.asyncio +@pytest.mark.parametrize("sparse_storage", [False, True]) @pytest.mark.parametrize( "fault", [ @@ -1100,7 +1101,7 @@ def sync(_fd): ], ) async def test_composed_stored_verification_compares_authority_projection_vectors_and_audits( - monkeypatch, fault + monkeypatch, fault, sparse_storage ): import logging @@ -1118,17 +1119,20 @@ async def test_composed_stored_verification_compares_authority_projection_vector monkeypatch.setattr(server.embedding, "get_embedder", lambda: FakeNGramEmbedder(dim=16)) block = await gate._build_block(args(), 1) block.seqs = [("org", "ns", i + 1) for i in range(5)] + stored = [pg_memory.encode_row(row) for row in block.rows] + if sparse_storage: + stored = [pg_memory.sparse_row(row) for row in stored] records = [ { - "head": json.dumps(pg_memory.encode_row(row)), - "revision": json.dumps(pg_memory.encode_row(row)), + "head": json.dumps(stored[i]), + "revision": json.dumps(stored[i]), "audit": json.dumps(pg_memory.encode_audit(audit)), "seq": i + 1, "targets": 1 if fault == "targets" else 0, "attempts": 1 if fault == "attempts" else 0, "last_error": "", } - for i, (row, audit) in enumerate(zip(block.rows, block.audits, strict=True)) + for i, (_row, audit) in enumerate(zip(block.rows, block.audits, strict=True)) ] projections = [dict(row) for row in block.rows] documents = [dict(row) for row in block.documents] @@ -1177,10 +1181,13 @@ async def query(sql, params): @pytest.mark.asyncio +@pytest.mark.parametrize("sparse_storage", [False, True]) @pytest.mark.parametrize( "fault", ["", "memory_key", "value", "version", "content_sha256", "status"] ) -async def test_api_authority_failure_names_only_the_failed_field(monkeypatch, fault): +async def test_api_authority_failure_names_only_the_failed_field( + monkeypatch, fault, sparse_storage +): import logging import server.embedding @@ -1215,7 +1222,8 @@ async def test_api_authority_failure_names_only_the_failed_field(monkeypatch, fa "content_sha256": "0" * 64, "status": "conflicted", }[fault] - stored = json.dumps(pg_memory.encode_row(head)) + encoded = pg_memory.encode_row(head) + stored = json.dumps(pg_memory.sparse_row(encoded) if sparse_storage else encoded) conn = SimpleNamespace( fetch=AsyncMock( return_value=[ From 39154da188d704ae329c919699f32ce6835c65c8 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Fri, 20 Nov 2026 21:40:07 +0000 Subject: [PATCH 12/13] Replicated erasure fixtures provide isolated write leases Fenced memory inserts require a lease table in the selected schema. Give each fixture database its own Keeper root while preserving the replicated content checks. Signed-off-by: zlareb1 (cherry picked from commit 96be03c3d5fa4062c8ca44226a5e138642c86504) Signed-off-by: zlareb1 --- .../integration/test_erasure_replicated_ch.py | 5 + tests/unit/test_erasure_replicated_fixture.py | 96 +++++++++++++++++++ 2 files changed, 101 insertions(+) create mode 100644 tests/unit/test_erasure_replicated_fixture.py diff --git a/tests/integration/test_erasure_replicated_ch.py b/tests/integration/test_erasure_replicated_ch.py index a1fa3f60c9..611f07969d 100644 --- a/tests/integration/test_erasure_replicated_ch.py +++ b/tests/integration/test_erasure_replicated_ch.py @@ -74,6 +74,11 @@ def db(raw: Any, monkeypatch: pytest.MonkeyPatch) -> Iterator[str]: parameters={"db": name}, ).result_rows assert engines == [(True,)] + # Fenced memory inserts need leases isolated from the source schema's keys. + raw.command( + f"CREATE TABLE {name}._migration_lock AS {source}._migration_lock " + f"ENGINE = KeeperMap('/{name}/_migration_lock') PRIMARY KEY name" + ) monkeypatch.setattr(settings, "ch_database", name) yield name finally: diff --git a/tests/unit/test_erasure_replicated_fixture.py b/tests/unit/test_erasure_replicated_fixture.py new file mode 100644 index 0000000000..3147871591 --- /dev/null +++ b/tests/unit/test_erasure_replicated_fixture.py @@ -0,0 +1,96 @@ +"""The replicated erasure fixture supplies an isolated write-lease store.""" + +from __future__ import annotations + +from types import SimpleNamespace + +import pytest + +from server.core import leases +from server.settings import settings +from tests.integration import test_erasure_replicated_ch as replicated +from tests.unit.claims_fake import FakeLockTable + + +class Schema: + def __init__(self): + self.tables = {} + self.commands = [] + + def command(self, sql): + self.commands.append(sql) + if sql.startswith("CREATE TABLE "): + self.tables[sql.split()[2]] = sql + + def query(self, sql, parameters=None): + if "engine_full" in sql: + assert parameters["db"] == "source_schema" + return SimpleNamespace(result_rows=[("ReplacingMergeTree(version) ORDER BY tuple()",)]) + assert "SELECT DISTINCT engine" in sql + flags = {"ENGINE = Replicated" in ddl for ddl in self.tables.values()} + return SimpleNamespace(result_rows=[(flag,) for flag in sorted(flags)]) + + +class FixtureLocks(FakeLockTable): + def __init__(self, schema): + super().__init__() + self.schema = schema + + async def query_one(self, sql, params=None): + if "loom._migration_lock" in sql: + if f"{settings.ch_database}._migration_lock" not in self.schema.tables: + raise RuntimeError("Unknown table _migration_lock") + return await super().query_one(sql, params) + + +@pytest.fixture +def isolated_schema(monkeypatch): + monkeypatch.setattr(settings, "ch_database", "source_schema") + schema = Schema() + stream = replicated.db.__wrapped__(schema, monkeypatch) + name = next(stream) + try: + yield schema, name + finally: + stream.close() + assert schema.commands[-1] == f"DROP DATABASE IF EXISTS {name} SYNC" + + +async def test_replicated_erasure_fixture_can_take_and_release_a_write_lease(isolated_schema): + schema, name = isolated_schema + keeper = FixtureLocks(schema) + lease = await leases.take("memory_insert/fixture", ttl_s=60, ch=keeper) + assert lease is not None and lease.held + assert settings.ch_database == name + await lease.release() + assert keeper.keys == {} + + +async def test_a_missing_fixture_lease_table_still_refuses(isolated_schema): + schema, name = isolated_schema + schema.tables.pop(f"{name}._migration_lock", None) + with pytest.raises(leases.LeaseUnavailable): + await leases.take("memory_insert/fixture", ttl_s=60, ch=FixtureLocks(schema)) + + +def test_fixture_lease_paths_are_separate_from_source_and_each_other(monkeypatch): + paths = [] + for _ in range(2): + with monkeypatch.context() as patch: + patch.setattr(settings, "ch_database", "source_schema") + schema = Schema() + stream = replicated.db.__wrapped__(schema, patch) + name = next(stream) + try: + ddl = schema.tables[f"{name}._migration_lock"] + assert "AS source_schema._migration_lock" in ddl + assert f"KeeperMap('/{name}/_migration_lock')" in ddl + assert "PRIMARY KEY name" in ddl + assert all( + "ENGINE = Replicated" in schema.tables[f"{name}.{table}"] + for table in replicated._TABLES + ) + paths.append(name) + finally: + stream.close() + assert paths[0] != paths[1] From 34a8c1ddc4ecdb47428d92fd14e307a2d9a0afbf Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Fri, 20 Nov 2026 22:48:33 +0000 Subject: [PATCH 13/13] Exercise the merged replicated-erasure lease fixture Signed-off-by: zlareb1 (cherry picked from commit f6cab47df2c4616d64bdff586c7e409229ed6370) Signed-off-by: zlareb1 --- tests/unit/test_erasure_replicated_fixture.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/test_erasure_replicated_fixture.py b/tests/unit/test_erasure_replicated_fixture.py index 3147871591..02c93ce77a 100644 --- a/tests/unit/test_erasure_replicated_fixture.py +++ b/tests/unit/test_erasure_replicated_fixture.py @@ -84,7 +84,7 @@ def test_fixture_lease_paths_are_separate_from_source_and_each_other(monkeypatch try: ddl = schema.tables[f"{name}._migration_lock"] assert "AS source_schema._migration_lock" in ddl - assert f"KeeperMap('/{name}/_migration_lock')" in ddl + assert f"KeeperMap('/loom/tests/{name}/migration_lock')" in ddl assert "PRIMARY KEY name" in ddl assert all( "ENGINE = Replicated" in schema.tables[f"{name}.{table}"]