From f6819e94b7e0f7069c9171f14ea92701cd52678e Mon Sep 17 00:00:00 2001 From: Lareb Zafar Date: Sat, 21 Nov 2026 05:45:52 +0530 Subject: [PATCH 1/3] Namespace retention preserves audit records Audit evidence follows its dedicated deployment retention window. Keep namespace warm tiering while removing inherited namespace deletion TTLs. Signed-off-by: Lareb Zafar --- docs/architecture/clickhouse.md | 4 +- docs/architecture/storage-layout.md | 2 +- docs/features/memory/retention-and-erasure.md | 5 ++ docs/operations/maintenance.md | 3 +- docs/reference/configuration.md | 2 +- loom.example.yaml | 7 +- schemas/loom-settings.schema.json | 2 +- server/settings/catalog.py | 6 +- server/storage/retention_overrides.py | 7 +- tests/unit/test_retention_overrides.py | 86 +++++++++++++------ 10 files changed, 86 insertions(+), 38 deletions(-) diff --git a/docs/architecture/clickhouse.md b/docs/architecture/clickhouse.md index 652a7ec27f..148fb11ca8 100644 --- a/docs/architecture/clickhouse.md +++ b/docs/architecture/clickhouse.md @@ -366,7 +366,9 @@ Operators can add retention to `audit_log`, `agent_events`, `memory`, `memory_em `conversations`, `chunks` and `documents` with `LOOM_RETENTION__DAYS`, and tiering with `LOOM_TIERING_
_WARM_DAYS` plus `LOOM_TIERING_WARM_VOLUME_NAME`. Per-namespace `retention_days` or `warm_days` overrides become a `multiIf` in the same TTL expression -(`server/storage/retention_overrides.py`). The worker leader applies them at boot and +(`server/storage/retention_overrides.py`). Audit deletion uses only its dedicated +`LOOM_RETENTION_AUDIT_LOG_DAYS`; namespace `warm_days` can still tier audit rows. +The worker leader applies them at boot and `retention_reconcile` keeps them in step, skipping unchanged tables because each `MODIFY TTL` is a mutation. When the last window on a table is cleared, the same pass issues `ALTER TABLE … REMOVE TTL`; none of these tables has a schema TTL. Every window diff --git a/docs/architecture/storage-layout.md b/docs/architecture/storage-layout.md index 6a4285189e..278d9712cc 100644 --- a/docs/architecture/storage-layout.md +++ b/docs/architecture/storage-layout.md @@ -1137,7 +1137,7 @@ it holds no provider identity or credential. | `events` | MT `(org, namespace, kind, ts)`, by `toYYYYMM(ts)`, TTL | stays | Log | census, `events.delete`, user scope too | — | the event stream, 30-day TTL; named in `server/observability/events_stream.py`, `server/insights/store/admin.py` | | `agent_events` | MT `(org, namespace, agent, ts)`, by `toYYYYMM(ts)` | stays | Log | step `agent_events.scrub` | — | ordinary op ledger, monthly partitions; named in `server/observability/events.py` | | `agent_events_private` | MT `(org, namespace, agent, ts)`, by `toYYYYMM(ts)` | stays | Log | step `agent_events.scrub` | — | marked requests and mirror-worker events, without analyst grants; named in `server/observability/events.py` | -| `audit_log` | MT `(org, namespace, event_type, ts)`, by `toYYYYMM(ts)` | stays | Log | no content | — | hash-only audit rows, kept unless an operator's retention (`retention_reconcile`) expires them; named in `server/memory/store/pg_publish.py`, `server/observability/audit.py` | +| `audit_log` | MT `(org, namespace, event_type, ts)`, by `toYYYYMM(ts)` | stays | Log | no content | — | hash-only audit rows, kept unless the dedicated audit retention expires them; namespace retention does not expire audit rows; named in `server/memory/store/pg_publish.py`, `server/observability/audit.py` | | `audit_checkpoints` | MT `window_start` | stays | Log | no content | — | verifier checkpoints over windows of `audit_log` across every org; named in `server/insights/store/audit.py`, `server/observability/otel_probes.py` | | `search_trace_log` | MT `(org, namespace, ts)`, by `toYYYYMM(ts)`, TTL | stays | Log | census, `search_trace_log.delete` | — | search traces, 180-day TTL; named in `server/insights/store/deploy_delta.py`, `server/tasks/store/runs.py` | | `search_misses` | MT `(org, namespace, ts)`, by `toYYYYMM(ts)`, TTL | stays | Log | census, `search_misses.delete`, user scope too | — | searches that found nothing, normalised query and asker only, 16-day TTL; named in `server/knowledge/store/gaps.py`, `server/erasure/store/steps_ledgers.py` | diff --git a/docs/features/memory/retention-and-erasure.md b/docs/features/memory/retention-and-erasure.md index 19803bfc6e..830898de8f 100644 --- a/docs/features/memory/retention-and-erasure.md +++ b/docs/features/memory/retention-and-erasure.md @@ -85,6 +85,11 @@ Postgres still serves, so the boot refuses `LOOM_RETENTION_MEMORY_DAYS`, `retention_days` leaves `chunks` too, so a document that stays keeps the chunks search finds it by; `LOOM_RETENTION_CHUNKS_DAYS` still applies. +Namespace `retention_days` never expires `audit_log` rows. Only +`LOOM_RETENTION_AUDIT_LOG_DAYS` sets their deletion window; its default of 0 keeps them +indefinitely. Namespace `warm_days` can still move audit rows to warm storage. Removing a +namespace retention window does not change the deployment's explicit audit retention. + The `retention_reconcile` worker applies the last two rows as one idempotent `ALTER TABLE … MODIFY TTL` per table, per `(org, namespace)` when overrides exist, and removes a table's TTL once no namespace and no deployment window is left on it. On diff --git a/docs/operations/maintenance.md b/docs/operations/maintenance.md index 5df99557fe..1decb06de6 100644 --- a/docs/operations/maintenance.md +++ b/docs/operations/maintenance.md @@ -190,7 +190,8 @@ the retention gauges (`loom.retention.*`) only count candidates. `memory`, `memory_embeddings`, `conversations`, `chunks` and `documents`. - **Per namespace.** `retention_days` in the namespace's registration (`namespace.register`); the managed tables' TTL becomes one `multiIf` over every - namespace that sets it. + namespace that sets it. Audit deletion uses only `LOOM_RETENTION_AUDIT_LOG_DAYS`; + namespace `warm_days` still applies to audit tiering. - The workers' leader applies the `MODIFY TTL` at boot. After that only the `retention_reconcile` worker writes table TTLs, so a policy change reaches the tables within `LOOM_RETENTION_RECONCILE_INTERVAL_SECONDS` and a burst of registrations does diff --git a/docs/reference/configuration.md b/docs/reference/configuration.md index 1e6e899d1c..a80be64230 100644 --- a/docs/reference/configuration.md +++ b/docs/reference/configuration.md @@ -1872,7 +1872,7 @@ Opt-in retention, applied as idempotent ALTER TABLE MODIFY TTL by server/storage | Variable | Type | Default | Live | Meaning | | --- | --- | --- | --- | --- | -| `LOOM_RETENTION_AUDIT_LOG_DAYS` | `int` | `0` | | Days `loom.audit_log` rows are kept; 0 = forever. A person's Undo of a retirement holds by its record there: one aged out stops holding, and the conflict resolver may judge and retire that pair again. | +| `LOOM_RETENTION_AUDIT_LOG_DAYS` | `int` | `0` | | Days `loom.audit_log` rows are kept; 0 = forever, independent of namespace retention. A person's Undo of a retirement holds by its record there: one aged out stops holding, and the conflict resolver may judge and retire that pair again. | | `LOOM_RETENTION_AGENT_EVENTS_DAYS` | `int` | `0` | | Days `loom.agent_events` rows are kept; 0 = forever. | | `LOOM_RETENTION_MEMORY_DAYS` | `int` | `0` | | Days `loom.memory` rows are kept; 0 = forever. The Postgres authority refuses it. | | `LOOM_RETENTION_MEMORY_EMBEDDINGS_DAYS` | `int` | `0` | | Days `loom.memory_embeddings` rows are kept; 0 = forever. The Postgres authority refuses it. | diff --git a/loom.example.yaml b/loom.example.yaml index 2bc42fdfdd..523368cf27 100644 --- a/loom.example.yaml +++ b/loom.example.yaml @@ -5300,9 +5300,10 @@ settings: # server/storage/retention_overrides.py: a retention_*_days setting deletes a table's # rows older than that many days. - # Days `loom.audit_log` rows are kept; 0 = forever. A person's Undo of a retirement - # holds by its record there: one aged out stops holding, and the conflict resolver may - # judge and retire that pair again. [restart] + # Days `loom.audit_log` rows are kept; 0 = forever, independent of namespace + # retention. A person's Undo of a retirement holds by its record there: one aged out + # stops holding, and the conflict resolver may judge and retire that pair again. + # [restart] # retention_audit_log_days: 0 # Days `loom.agent_events` rows are kept; 0 = forever. [restart] diff --git a/schemas/loom-settings.schema.json b/schemas/loom-settings.schema.json index 75c2c45c1f..dea7d18cc5 100644 --- a/schemas/loom-settings.schema.json +++ b/schemas/loom-settings.schema.json @@ -6339,7 +6339,7 @@ "retention_audit_log_days": { "default": 0, "type": "integer", - "description": "Days `loom.audit_log` rows are kept; 0 = forever. A person's Undo of a retirement holds by its record there: one aged out stops holding, and the conflict resolver may judge and retire that pair again." + "description": "Days `loom.audit_log` rows are kept; 0 = forever, independent of namespace retention. A person's Undo of a retirement holds by its record there: one aged out stops holding, and the conflict resolver may judge and retire that pair again." }, "retention_chunks_days": { "default": 0, diff --git a/server/settings/catalog.py b/server/settings/catalog.py index c2a2f43d87..db908c0ca1 100644 --- a/server/settings/catalog.py +++ b/server/settings/catalog.py @@ -5551,9 +5551,9 @@ class StorageSettings(BaseModel): # server/storage/retention_overrides.py: a retention_*_days setting deletes a table's # rows older than that many days. # ---- retention ---- - # Days `loom.audit_log` rows are kept; 0 = forever. A person's Undo of a - # retirement holds by its record there: one aged out stops holding, and the - # conflict resolver may judge and retire that pair again. + # Days `loom.audit_log` rows are kept; 0 = forever, independent of namespace retention. + # A person's Undo of a retirement holds by its record there: one aged out stops + # holding, and the conflict resolver may judge and retire that pair again. retention_audit_log_days: int = 0 # Days `loom.agent_events` rows are kept; 0 = forever. retention_agent_events_days: int = 0 diff --git a/server/storage/retention_overrides.py b/server/storage/retention_overrides.py index ce8bdbd124..5f1cded869 100644 --- a/server/storage/retention_overrides.py +++ b/server/storage/retention_overrides.py @@ -12,8 +12,8 @@ means no TTL: a table that still carries one from an earlier pass gets `ALTER TABLE … REMOVE TTL` (none of the managed tables has a schema TTL). -Per-namespace retention applies uniformly across every managed table; -per-table-per-namespace granularity is deliberately not exposed. +Namespace retention never expires audit rows; their retention comes only from +the audit setting. Namespace tiering may still move them to warm storage. """ from __future__ import annotations @@ -253,6 +253,9 @@ def compute_alters( r_days = max(0, target.retention_days(settings_obj) or 0) t_days = max(0, target.warm_days(settings_obj) or 0) overrides = ovr_all + if target.table == "loom.audit_log": + # Memory retention must not erase the evidence of its own deletion. + overrides = [{**o, "retention_days": 0} for o in ovr_all] if target.table in (*_PG_HELD, "loom.chunks") and pg.enabled(): # Postgres holds memory and document heads: a TTL here would delete only # ClickHouse's copy (or only its vectors, leaving live memories search no diff --git a/tests/unit/test_retention_overrides.py b/tests/unit/test_retention_overrides.py index 1e255c7182..21ba63aae7 100644 --- a/tests/unit/test_retention_overrides.py +++ b/tests/unit/test_retention_overrides.py @@ -60,6 +60,19 @@ async def execute(self, sql: str, params: Any = None) -> None: self.execute_calls.append(sql) +@pytest.mark.parametrize("audit_days", [0, 365]) +def test_namespace_retention_cannot_shorten_or_create_audit_retention(audit_days: int) -> None: + out = retention_overrides.compute_alters( + _FakeSettings(retention_audit_log_days=audit_days), + namespace_overrides=[{"org": "o", "namespace": "n", "retention_days": 1, "warm_days": 0}], + ) + audit = [sql for target, sql, _ in out if target.table == "loom.audit_log"] + if audit_days: + assert audit == ["ALTER TABLE loom.audit_log MODIFY TTL ts + INTERVAL 365 DAY DELETE"] + else: + assert audit == [] + + # ---------------------------------------------------------------------- # compute_alters — pure function # ---------------------------------------------------------------------- @@ -110,22 +123,22 @@ def test_compute_alters_per_namespace_override_uses_multiif() -> None: }, ], ) - # Override applies to every managed table — one ALTER per table. - assert len(out) == 8 - audit_alter = next(s for t, s, _ in out if t.table == "loom.audit_log") - assert "multiIf" in audit_alter - assert "org = 'clickhouse'" in audit_alter - assert "namespace = 'compliance'" in audit_alter - assert "INTERVAL 3650 DAY" in audit_alter + # Namespace retention leaves the audit table alone. + assert len(out) == 7 + event_alter = next(s for t, s, _ in out if t.table == "loom.agent_events") + assert "multiIf" in event_alter + assert "org = 'clickhouse'" in event_alter + assert "namespace = 'compliance'" in event_alter + assert "INTERVAL 3650 DAY" in event_alter # 2099 fallback for unmatched rows - assert "2099-12-31" in audit_alter + assert "2099-12-31" in event_alter def test_compute_alters_override_with_global_falls_through_to_global() -> None: """Override + global both set: matching rows use override; others use the global default (not 2099).""" out = retention_overrides.compute_alters( - _FakeSettings(retention_audit_log_days=365), + _FakeSettings(retention_agent_events_days=365), namespace_overrides=[ { "org": "clickhouse", @@ -135,10 +148,10 @@ def test_compute_alters_override_with_global_falls_through_to_global() -> None: }, ], ) - audit_alter = next(s for t, s, _ in out if t.table == "loom.audit_log") - assert "INTERVAL 3650 DAY" in audit_alter # override - assert "INTERVAL 365 DAY" in audit_alter # global fallback - assert "2099" not in audit_alter # global covers fallback + event_alter = next(s for t, s, _ in out if t.table == "loom.agent_events") + assert "INTERVAL 3650 DAY" in event_alter # override + assert "INTERVAL 365 DAY" in event_alter # global fallback + assert "2099" not in event_alter # global covers fallback def test_compute_alters_memory_uses_coalesce_for_caller_expiry() -> None: @@ -204,13 +217,12 @@ def test_compute_alters_sql_quotes_org_namespace_safely() -> None: }, ], ) - audit_alter = next(s for t, s, _ in out if t.table == "loom.audit_log") - assert "tenant\\'s-org" in audit_alter + event_alter = next(s for t, s, _ in out if t.table == "loom.agent_events") + assert "tenant\\'s-org" in event_alter -def test_compute_alters_per_namespace_applies_to_all_tables() -> None: - """One namespace override produces one ALTER per managed table — - retention is uniform across tables for a namespace, by design.""" +def test_namespace_retention_preserves_audit_rows() -> None: + """A memory-retention policy leaves its audit evidence intact.""" out = retention_overrides.compute_alters( _FakeSettings(), namespace_overrides=[ @@ -221,7 +233,6 @@ def test_compute_alters_per_namespace_applies_to_all_tables() -> None: assert tables == [ "loom.agent_events", "loom.agent_events_private", - "loom.audit_log", "loom.chunks", "loom.conversations", "loom.documents", @@ -252,7 +263,7 @@ async def test_apply_uses_provided_overrides_without_ch_query() -> None: @pytest.mark.asyncio async def test_apply_per_namespace_emits_alters_for_all_tables() -> None: - """One namespace override → one ALTER per managed table (8).""" + """Namespace retention applies to seven tables and leaves audit rows alone.""" ch = _DiffCH({}) n = await retention_overrides.apply( _FakeSettings(), @@ -262,7 +273,7 @@ async def test_apply_per_namespace_emits_alters_for_all_tables() -> None: ], leader=True, ) - assert n == 8 + assert n == 7 @pytest.mark.asyncio @@ -440,13 +451,15 @@ async def test_a_cleared_window_removes_the_ttl_on_real_tables(tmp_path: Path) - namespace_overrides=[{"org": "o", "namespace": "ns1", "retention_days": 30}], leader=True, ) - assert set_ttl == len(tables) - assert all(" TTL " in _engine_full(tmp_path, t) for t in tables) + retained = [t for t in tables if t != "audit_log"] + assert set_ttl == len(retained) + assert all(" TTL " in _engine_full(tmp_path, t) for t in retained) + assert " TTL " not in _engine_full(tmp_path, "audit_log") removed = await retention_overrides.apply( _FakeSettings(), ch=ch, namespace_overrides=[], leader=True ) - assert removed == len(tables) + assert removed == len(retained) assert not [t for t in tables if " TTL " in _engine_full(tmp_path, t)] # an overlapping pass sends the same statement; the text apply matches is ClickHouse's @@ -623,7 +636,7 @@ def test_no_ttl_deletes_memory_or_its_vectors_under_the_postgres_authority( ], ) by_table = {target.table: sql for target, sql, _ in out} - assert "DELETE" in by_table["loom.audit_log"] + assert "loom.audit_log" not in by_table assert "DELETE" not in by_table["loom.memory"] and "TO VOLUME" in by_table["loom.memory"] assert ( "loom.memory_embeddings" not in by_table @@ -685,3 +698,26 @@ async def test_a_ttl_left_from_the_clickhouse_authority_goes_under_postgres(monk assert n >= 1 documents = [c for c in ch.execute_calls if "loom.documents " in c] assert documents == ["ALTER TABLE loom.documents REMOVE TTL"], ch.execute_calls + + +@pytest.mark.asyncio +async def test_namespace_retention_removes_an_inherited_audit_ttl() -> None: + ch = _DiffCH({"audit_log": "ts + toIntervalDay(1)"}) + await retention_overrides.apply( + _FakeSettings(), + ch=ch, + leader=True, + namespace_overrides=[{"org": "o", "namespace": "n", "retention_days": 1}], + ) + assert "ALTER TABLE loom.audit_log REMOVE TTL" in ch.execute_calls + + +def test_namespace_tiering_preserves_dedicated_audit_retention() -> None: + out = retention_overrides.compute_alters( + _FakeSettings(retention_audit_log_days=365), + namespace_overrides=[{"org": "o", "namespace": "n", "retention_days": 1, "warm_days": 7}], + ) + sql = next(sql for target, sql, _ in out if target.table == "loom.audit_log") + assert "TO VOLUME" in sql and "INTERVAL 7 DAY" in sql + assert "INTERVAL 365 DAY DELETE" in sql + assert "INTERVAL 1 DAY" not in sql From fb1e8fd24f0bba95040262523bcc7a7dfdf654ec Mon Sep 17 00:00:00 2001 From: Lareb Zafar Date: Sat, 21 Nov 2026 06:09:50 +0530 Subject: [PATCH 2/3] Check audit retention separately in live storage tests Signed-off-by: Lareb Zafar --- tests/integration/test_repositories_live_ch.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/tests/integration/test_repositories_live_ch.py b/tests/integration/test_repositories_live_ch.py index 3f62a45442..f2f7237cd1 100644 --- a/tests/integration/test_repositories_live_ch.py +++ b/tests/integration/test_repositories_live_ch.py @@ -350,14 +350,18 @@ async def _drain(attempts: int = 120) -> None: namespace_overrides=overrides, leader=True, ) - if n >= 8: + if n >= 7: break await _drain() - assert n == 8 # one ALTER per managed table + assert n == 7 # Audit retention is configured independently. - row = await ch.query_one( + audit = await ch.query_one( "SELECT engine_full FROM system.tables WHERE database='loom' AND name='audit_log'", ) + assert "TTL" not in (audit or {}).get("engine_full", "") + row = await ch.query_one( + "SELECT engine_full FROM system.tables WHERE database='loom' AND name='agent_events'", + ) engine_full = (row or {}).get("engine_full", "") assert "multiIf" in engine_full assert f"'{test_org}'" in engine_full From b35df8b7db467e0766b21468f2b3329e47528f11 Mon Sep 17 00:00:00 2001 From: Lareb Zafar Date: Sat, 21 Nov 2026 06:27:35 +0530 Subject: [PATCH 3/3] Isolate every managed retention table in the storage regression Signed-off-by: Lareb Zafar --- .../integration/test_repositories_live_ch.py | 27 +++++++------------ 1 file changed, 10 insertions(+), 17 deletions(-) diff --git a/tests/integration/test_repositories_live_ch.py b/tests/integration/test_repositories_live_ch.py index f2f7237cd1..4a663f29b4 100644 --- a/tests/integration/test_repositories_live_ch.py +++ b/tests/integration/test_repositories_live_ch.py @@ -335,6 +335,13 @@ async def _drain(attempts: int = 120) -> None: await asyncio.sleep(0.5) try: + for table in managed: + row = await ch.query_one( + "SELECT engine_full FROM system.tables WHERE database='loom' AND name={name:String}", + {"name": table.removeprefix("loom.")}, + ) + if " TTL " in str((row or {}).get("engine_full", "")): + await ch.execute(f"ALTER TABLE {table} REMOVE TTL") # apply() leaves a table alone while it has # unfinished mutations (an earlier test's DELETE may still be # running on this shared service) and the reconcile worker retries @@ -372,25 +379,11 @@ async def _drain(attempts: int = 120) -> None: assert "2099" in engine_full finally: - try: - for t in ( - "loom.audit_log", - "loom.agent_events", - "loom.memory_embeddings", - "loom.conversations", - "loom.chunks", - "loom.documents", - ): - try: - await ch.execute(f"ALTER TABLE {t} REMOVE TTL") - except Exception: # noqa: BLE001 — teardown: a table may lack a TTL; must not raise - pass + for t in managed: try: - await ch.execute("ALTER TABLE loom.memory REMOVE TTL") - except Exception: # noqa: BLE001 — teardown: memory may lack a TTL; must not raise + await ch.execute(f"ALTER TABLE {t} REMOVE TTL") + except Exception: # noqa: BLE001 — teardown: a table may lack a TTL; must not raise pass - except Exception: # noqa: BLE001 — teardown must not raise - pass def dataclass_compat(cls):