Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
16 commits
Select commit Hold shift + click to select a range
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 5 additions & 2 deletions bench/cell_trigger_gate.py
Original file line number Diff line number Diff line change
Expand Up @@ -1258,6 +1258,8 @@ async def _absent(ch: Any, block: Block) -> None:
async def _verify(
conn: Any, ch: Any, args: argparse.Namespace, identities: Sequence[RequestRow] | Block
) -> None:
from server.memory.store import pg_memory

block = identities if isinstance(identities, Block) else None
wanted = identities.rows if isinstance(identities, Block) else [asdict(r) for r in identities]
for offset in range(0, len(wanted), 100):
Expand Down Expand Up @@ -1309,9 +1311,10 @@ async def _verify(
and record["last_error"] in (None, ""),
"delivery settled without failure",
)
head, revision, audit = (
json.loads(record[name]) for name in ("head", "revision", "audit")
head, revision = (
pg_memory.decode_row(json.loads(record[name])) for name in ("head", "revision")
)
audit = json.loads(record["audit"])
_compare(head, revision, tuple(head), "authority full row")
if block:
_compare(expected, head, tuple(expected), "authority staged row")
Expand Down
10 changes: 7 additions & 3 deletions docs/architecture/storage-authority.md
Original file line number Diff line number Diff line change
Expand Up @@ -126,7 +126,7 @@ that moves it.
| --------------------------- | ---------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
| `loom.authority` | singleton | the recorded authority, its epoch and the deployment id |
| `loom.scope_generations` | `(org, scope)` | the last revision number of a namespace, or a policy generation (`scope = '*'`, org `''` for the global one) |
| `loom.memory_current` | `(org, namespace, agent, user_id, session_id, memory_key)` | the head of each key: the complete `loom.memory` row as JSON, with `memory_id`, `version`, `seq`, `content_sha256` beside it |
| `loom.memory_current` | `(org, namespace, agent, user_id, session_id, memory_key)` | the head of each key: its `loom.memory` row as JSON, without its default fields, with `memory_id`, `version`, `seq`, `content_sha256` beside it |
| `loom.memory_revisions` | `(org, namespace, seq)` | every committed row, append-only (a redaction scrubs one in place, and with `LOOM_PG_PAYLOAD_CLEAR_ENABLED` the clearing pass clears a delivered one's payload past the horizon: [payload clearing](#payload-clearing)), with its hash-only audit row and its metadata in typed columns ([revision metadata](#revision-metadata)) |
| `loom.memory_deliveries` | `(org, namespace, seq)` | the targets a revision still owes ClickHouse (bits: 1 memory row, 2 search document, 4 audit row, 8 a deleted decision's record, 16 the old vectors), its lease and fencing token; deleted when the last target lands |
| `loom.idempotency` | `(org, key_hash)` | an op's request digest, its state (`running`, `committed`, `done`), the revisions it committed, its answer, a lease and its token |
Expand All @@ -141,7 +141,7 @@ that moves it.
| `loom.knowledge_datasets` | `(org, namespace, source_uri, doc_version, dataset_id)` | each dataset's catalog row at a document version: the snapshot that serves it, its schema and approval ([datasets](#datasets)) |

A head is the row `loom.memory FINAL` would serve: a commit replaces it only when its
version is not older, as `ReplacingMergeTree(version)` decides. Holding the whole row
version is not older, as `ReplacingMergeTree(version)` decides. Holding the row
keeps every one of the write paths that build `loom.memory` rows unchanged
(`server/memory/store/rows.py`): they all reach the commit through the one insert
funnel, `insert_memory_versions`. The row keeps ClickHouse's precision (whole-second
Expand All @@ -153,7 +153,11 @@ A stored row (a head, or a revision's payload) may leave out a field at its
(`rows.FIRST_VERSION_DEFAULTS`, `pg_memory.SPARSE_DEFAULTS`): every reader takes a
missing field as that default, `decode_row` in Python and `coalesce` (or a test that
holds for NULL) in SQL. Every other field is always stored, a default-valued one
included. This build stores every field.
included. A commit stores each row without the fields at their default
(`pg_memory.sparse_row`), which keeps it smaller: most of a row's fields hold their
default, and every stored field repeats its name. The writers that change a stored row in
place (the usage fold, a redaction) merge fields into it, and may store a field at its
default.

A memory id is not unique within an org: a namespace restored from another's snapshot
keeps the memory ids under newer versions. Every head read by id therefore names its
Expand Down
19 changes: 13 additions & 6 deletions server/memory/store/pg_memory.py
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
"""Every Postgres statement of the memory authority (the Postgres storage profile).

`loom.memory_current` holds the head of each key as the complete loom.memory row,
which is what `loom.memory FINAL` would serve: a row replaces the head only when
its version is not older, as ReplacingMergeTree(version) decides. Each commit
also appends the rows to `loom.memory_revisions`, numbered per namespace by the
`loom.memory_current` holds the head of each key, the row `loom.memory FINAL` would
serve, stored without the fields at their default (`sparse_row`): a row replaces the
head only when its version is not older, as ReplacingMergeTree(version) decides. Each
commit also appends the rows to `loom.memory_revisions`, numbered per namespace by the
namespace's generation, and records what they owe ClickHouse in
`loom.memory_deliveries`. The transaction is short and does no other I/O.

Expand Down Expand Up @@ -118,6 +118,12 @@ def encode_row(row: Mapping[str, Any]) -> dict[str, Any]:
SPARSE_DEFAULTS: Mapping[str, Any] = FIRST_VERSION_DEFAULTS


def sparse_row(encoded: Mapping[str, Any]) -> dict[str, Any]:
"""An encoded row as it is stored: without the fields at their default
(`SPARSE_DEFAULTS`), which every reader fills back (`decode_row`)."""
return {c: v for c, v in encoded.items() if c not in SPARSE_DEFAULTS or v != SPARSE_DEFAULTS[c]}


def decode_row(stored: Mapping[str, Any]) -> dict[str, Any]:
"""The insertable loom.memory row back from its JSON, each field it leaves out at its
default (`SPARSE_DEFAULTS`), as ClickHouse fills it."""
Expand Down Expand Up @@ -680,6 +686,7 @@ async def commit_versions(
if any(r["namespace"] in POLICY_SCOPE_NAMES for r in stored):
# A namespace's generation shares its key with the policy generations.
raise ValidationError("'*' and '' are not namespace names")
sparse = [sparse_row(r) for r in stored]
per_row = [targets] * len(stored) if isinstance(targets, int) else list(targets)
per_row = [
t | (DECISION_RECORD if deleted_decision(r) else 0)
Expand Down Expand Up @@ -743,7 +750,7 @@ async def commit_versions(
[int(r["version"]) for r in head_rows],
[seqs[heads[k]][2] for k in keys],
[str(r.get("content_sha256") or "") for r in head_rows],
head_rows,
[sparse[heads[k]] for k in keys],
)
await conn.execute(
_INSERT_REVISIONS,
Expand All @@ -757,7 +764,7 @@ async def commit_versions(
[str(r.get("actor") or "") for r in stored],
[str(r.get("source_class") or "") for r in stored],
[request_id] * len(stored),
stored,
sparse,
[encode_audit(a) for a in audits],
)
await conn.execute(
Expand Down
38 changes: 20 additions & 18 deletions tests/integration/test_pg_authority.py
Original file line number Diff line number Diff line change
Expand Up @@ -298,10 +298,10 @@ async def test_a_revisions_metadata_is_kept_from_its_row_and_outlives_it(
async def test_a_head_that_leaves_out_its_default_fields_is_served_as_the_full_one(
call: Any, pg_conn: Any, org: str
) -> None:
"""A stored row may leave out a field at its loom.memory default: point reads,
listing and search serve such a head as they serve the full one, and the SQL readers
of its fields (the journal's revisions, the cross-session heads, the usage fold, the
erasure export) take a missing one as its default."""
"""A commit stores a row without the fields at their loom.memory default: a point read
serves such a head as it serves the full one, listing and search find it and bulk_get
serves its value, and the journal's revisions, the cross-session heads, the usage fold
and the erasure export take a missing field as its default."""
from server.core import pg
from server.erasure.store.exports import export_memory_heads
from server.identity.domain.read_scope import WHOLE_NAMESPACE
Expand All @@ -313,25 +313,26 @@ async def test_a_head_that_leaves_out_its_default_fields_is_served_as_the_full_o
).status_code == 200
await _published(pg_conn, org)
get = {"namespace": ns, "agent": "a", "memory_key": "k"}
before = (await call("memory.get", get)).json()
tables = ("memory_current", "memory_revisions")

def sparse(stored: dict[str, Any]) -> dict[str, Any]:
defaults = pg_memory.SPARSE_DEFAULTS
return {c: v for c, v in stored.items() if c not in defaults or v != defaults[c]}
async def store(table: str, row: dict[str, Any]) -> None:
await pg_conn.execute(
f"UPDATE loom.{table} SET row = $3 WHERE org = $1 AND namespace = $2", org, ns, row
)

# The row as a build storing every field would: the defaults filled back in.
full: dict[str, dict[str, Any]] = {}
for table in ("memory_current", "memory_revisions"):
for table in tables:
stored = await pg_conn.fetchval(
f"SELECT row FROM loom.{table} WHERE org = $1 AND namespace = $2", org, ns
)
assert len(sparse(stored)) < len(stored)
full[table] = stored
await pg_conn.execute(
f"UPDATE loom.{table} SET row = $3 WHERE org = $1 AND namespace = $2",
org,
ns,
sparse(stored),
)
assert pg_memory.sparse_row(stored) == stored # the commit left its defaults out
full[table] = pg_memory.encode_row(pg_memory.decode_row(stored))
assert len(full[table]) > len(stored)
await store(table, full[table])
before = (await call("memory.get", get)).json()
for table in tables:
await store(table, pg_memory.sparse_row(full[table]))
assert (await call("memory.get", get)).json() == before
listed = (await call("memory.list", {"namespace": ns, "agent": "a"})).json()
assert [e["memory_key"] for e in listed["entries"]] == ["k"], listed
Expand Down Expand Up @@ -3163,7 +3164,8 @@ async def test_a_same_key_write_in_another_session_retires_a_first_write_at_once
statuses = {
r["session_id"]: r["status"]
for r in await pg_conn.fetch(
"SELECT session_id, row->>'status' AS status FROM loom.memory_current "
"SELECT session_id, coalesce(row->>'status', 'active') AS status "
"FROM loom.memory_current "
"WHERE org = $1 AND namespace = $2",
org,
ns,
Expand Down
20 changes: 14 additions & 6 deletions tests/unit/test_bench_cell_trigger_gate.py
Original file line number Diff line number Diff line change
Expand Up @@ -1084,6 +1084,7 @@ def sync(_fd):


@pytest.mark.asyncio
@pytest.mark.parametrize("sparse_storage", [False, True])
@pytest.mark.parametrize(
"fault",
[
Expand All @@ -1100,7 +1101,7 @@ def sync(_fd):
],
)
async def test_composed_stored_verification_compares_authority_projection_vectors_and_audits(
monkeypatch, fault
monkeypatch, fault, sparse_storage
):
import logging

Expand All @@ -1118,17 +1119,20 @@ async def test_composed_stored_verification_compares_authority_projection_vector
monkeypatch.setattr(server.embedding, "get_embedder", lambda: FakeNGramEmbedder(dim=16))
block = await gate._build_block(args(), 1)
block.seqs = [("org", "ns", i + 1) for i in range(5)]
stored = [pg_memory.encode_row(row) for row in block.rows]
if sparse_storage:
stored = [pg_memory.sparse_row(row) for row in stored]
records = [
{
"head": json.dumps(pg_memory.encode_row(row)),
"revision": json.dumps(pg_memory.encode_row(row)),
"head": json.dumps(stored[i]),
"revision": json.dumps(stored[i]),
"audit": json.dumps(pg_memory.encode_audit(audit)),
"seq": i + 1,
"targets": 1 if fault == "targets" else 0,
"attempts": 1 if fault == "attempts" else 0,
"last_error": "",
}
for i, (row, audit) in enumerate(zip(block.rows, block.audits, strict=True))
for i, (_row, audit) in enumerate(zip(block.rows, block.audits, strict=True))
]
projections = [dict(row) for row in block.rows]
documents = [dict(row) for row in block.documents]
Expand Down Expand Up @@ -1177,10 +1181,13 @@ async def query(sql, params):


@pytest.mark.asyncio
@pytest.mark.parametrize("sparse_storage", [False, True])
@pytest.mark.parametrize(
"fault", ["", "memory_key", "value", "version", "content_sha256", "status"]
)
async def test_api_authority_failure_names_only_the_failed_field(monkeypatch, fault):
async def test_api_authority_failure_names_only_the_failed_field(
monkeypatch, fault, sparse_storage
):
import logging

import server.embedding
Expand Down Expand Up @@ -1215,7 +1222,8 @@ async def test_api_authority_failure_names_only_the_failed_field(monkeypatch, fa
"content_sha256": "0" * 64,
"status": "conflicted",
}[fault]
stored = json.dumps(pg_memory.encode_row(head))
encoded = pg_memory.encode_row(head)
stored = json.dumps(pg_memory.sparse_row(encoded) if sparse_storage else encoded)
conn = SimpleNamespace(
fetch=AsyncMock(
return_value=[
Expand Down
96 changes: 96 additions & 0 deletions tests/unit/test_erasure_replicated_fixture.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,96 @@
"""The replicated erasure fixture supplies an isolated write-lease store."""

from __future__ import annotations

from types import SimpleNamespace

import pytest

from server.core import leases
from server.settings import settings
from tests.integration import test_erasure_replicated_ch as replicated
from tests.unit.claims_fake import FakeLockTable


class Schema:
def __init__(self):
self.tables = {}
self.commands = []

def command(self, sql):
self.commands.append(sql)
if sql.startswith("CREATE TABLE "):
self.tables[sql.split()[2]] = sql

def query(self, sql, parameters=None):
if "engine_full" in sql:
assert parameters["db"] == "source_schema"
return SimpleNamespace(result_rows=[("ReplacingMergeTree(version) ORDER BY tuple()",)])
assert "SELECT DISTINCT engine" in sql
flags = {"ENGINE = Replicated" in ddl for ddl in self.tables.values()}
return SimpleNamespace(result_rows=[(flag,) for flag in sorted(flags)])


class FixtureLocks(FakeLockTable):
def __init__(self, schema):
super().__init__()
self.schema = schema

async def query_one(self, sql, params=None):
if "loom._migration_lock" in sql:
if f"{settings.ch_database}._migration_lock" not in self.schema.tables:
raise RuntimeError("Unknown table _migration_lock")
return await super().query_one(sql, params)


@pytest.fixture
def isolated_schema(monkeypatch):
monkeypatch.setattr(settings, "ch_database", "source_schema")
schema = Schema()
stream = replicated.db.__wrapped__(schema, monkeypatch)
name = next(stream)
try:
yield schema, name
finally:
stream.close()
assert schema.commands[-1] == f"DROP DATABASE IF EXISTS {name} SYNC"


async def test_replicated_erasure_fixture_can_take_and_release_a_write_lease(isolated_schema):
schema, name = isolated_schema
keeper = FixtureLocks(schema)
lease = await leases.take("memory_insert/fixture", ttl_s=60, ch=keeper)
assert lease is not None and lease.held
assert settings.ch_database == name
await lease.release()
assert keeper.keys == {}


async def test_a_missing_fixture_lease_table_still_refuses(isolated_schema):
schema, name = isolated_schema
schema.tables.pop(f"{name}._migration_lock", None)
with pytest.raises(leases.LeaseUnavailable):
await leases.take("memory_insert/fixture", ttl_s=60, ch=FixtureLocks(schema))


def test_fixture_lease_paths_are_separate_from_source_and_each_other(monkeypatch):
paths = []
for _ in range(2):
with monkeypatch.context() as patch:
patch.setattr(settings, "ch_database", "source_schema")
schema = Schema()
stream = replicated.db.__wrapped__(schema, patch)
name = next(stream)
try:
ddl = schema.tables[f"{name}._migration_lock"]
assert "AS source_schema._migration_lock" in ddl
assert f"KeeperMap('/loom/tests/{name}/migration_lock')" in ddl
assert "PRIMARY KEY name" in ddl
assert all(
"ENGINE = Replicated" in schema.tables[f"{name}.{table}"]
for table in replicated._TABLES
)
paths.append(name)
finally:
stream.close()
assert paths[0] != paths[1]
Loading