From e2c4d26d0d3c1c9c6907015f1c9b43941bac6da3 Mon Sep 17 00:00:00 2001 From: Lareb Zafar Date: Fri, 20 Nov 2026 17:55:42 +0530 Subject: [PATCH 1/2] Usage folds revisit boundary seconds and repair stale counters Retain the boundary second and reconcile destination counters on each pass. Persist scan progress and an explicit reset so late usage and stale memory versions converge without another event. Signed-off-by: Lareb Zafar --- docs/architecture/clickhouse.md | 15 ++- server/memory/workers/usage_fold.py | 115 +++++++++------- tests/analyzer/cases/workers.py | 2 + tests/analyzer/shapes.txt | 3 + tests/unit/test_usage_fold_sql.py | 161 ++++++++++++++++++++++- tests/unit/test_usage_fold_worker.py | 188 +++++++++++++++++++++++---- 6 files changed, 415 insertions(+), 69 deletions(-) diff --git a/docs/architecture/clickhouse.md b/docs/architecture/clickhouse.md index 4371a6e37f..17b98d32a1 100644 --- a/docs/architecture/clickhouse.md +++ b/docs/architecture/clickhouse.md @@ -668,7 +668,20 @@ with `purge=true`, runner state, and the supersession rows `knowledge.restore` r (`server/memory/workers/usage_fold.py`). On ClickHouse Cloud each statement is a patch part and a Keeper entry, so cost is per statement. A pass is skipped while the table has too many unfinished mutations. Only "lightweight update unsupported" errors fall back to - `ALTER TABLE … UPDATE`; a transient error is retried next pass. + `ALTER TABLE … UPDATE`; a transient error is retried next pass. Every batch retains + its boundary second because `bumped_at` has second resolution. A cursor ordered by + `(bumped_at, org, namespace, memory_id)` pages through ties; `worker_status` stores it + when a pass stops at its time budget. A completed scan stores an epoch cursor to + reset the scan: an empty cursor tells the supervisor to keep its previous value. + + A full-version insert can carry counters read before a fold and replace its result. + Every pass therefore compares the absolute totals and last-recall time with the live + rows, even when the usage high-water mark is unchanged, and rewinds to the earliest + discrepancy. This adds an aggregate join per nonempty pass. Repair is eventual once + the competing writes are visible; a read followed by an insert is not atomic. Absolute + targets and `greatest` make counter repair idempotent across replicas. The usage events + remain authoritative; the counters on `memory` are a derived projection. + - **Erasure purges** remove rows. On `erasure/store/mutations.py::LIGHTWEIGHT_PURGE_TABLES` (vector, chunk, summary and evidence tables) a purge uses lightweight `DELETE FROM`, which writes only the row mask: rows vanish from every read at once and bytes leave at the next merge. diff --git a/server/memory/workers/usage_fold.py b/server/memory/workers/usage_fold.py index 747b355d10..990a382eb5 100644 --- a/server/memory/workers/usage_fold.py +++ b/server/memory/workers/usage_fold.py @@ -55,17 +55,19 @@ statements instead of 2,000; on ClickHouse Cloud each lightweight UPDATE is a patch part and the cost is per statement, not per row. -The watermark is process-local. On a cold start it is derived from the -destination tables (the earliest `bumped_at` whose totals exceed what -`loom.memory` already carries) instead of the epoch, so a redeploy does -not re-walk the whole history at the batch cap before reaching live -rows, and a pass keeps taking batches until it is caught up or its time -budget is spent. +The watermark is process-local and retains the boundary second. Every pass +compares totals with the destination counters: a concurrent full-version +insert can carry counts read before a fold and must be repaired even when +no usage timestamp changes. This is eventual reconciliation, not an atomic +update of future versions. A cold start begins at the earliest discrepancy +instead of the epoch. A cursor on (bumped_at, org, namespace, memory_id) +pages through ties and is saved in worker_status when the time budget is spent. """ from __future__ import annotations import hashlib +import json import logging import time from dataclasses import dataclass, field @@ -94,6 +96,10 @@ _UINT32_MAX = 2**32 - 1 +_Cursor = tuple[datetime, str, str, str] +# Empty cursors preserve the supervisor's last value; an epoch cursor resets it. +_RESET_CURSOR = json.dumps((UNIX_EPOCH.isoformat(), "", "", "")) + _watermark: datetime = UNIX_EPOCH _watermark_derived: bool = False @@ -454,13 +460,12 @@ async def _fold_reinforcement_rows(ch: _CHClient, targets: list[_Target], chunk: return statements -async def _derive_cold_start_watermark(ch: _CHClient, hwm: datetime) -> datetime: - """Where to start after a restart: just before the earliest totals row - whose recall/evidence exceed what `loom.memory` already carries. Targets +async def _derive_watermark(ch: _CHClient, hwm: datetime) -> datetime: + """Reconcile from the earliest totals row ahead of a live memory. Targets are absolute, so the join is exact for the two counters that live on `loom.memory`; `used`-only activity has no destination counter there - and re-folds when its row is next bumped. No unfolded rows → the high - water mark (nothing to do).""" + and re-folds when its row is next bumped. With no discrepancy, retain the + high water mark's second for usage that becomes visible after this read.""" row = await ch.query_one( """ SELECT min(t.bumped) AS first_unfolded @@ -468,22 +473,24 @@ async def _derive_cold_start_watermark(ch: _CHClient, hwm: datetime) -> datetime SELECT org, namespace, memory_id, sum(recall_count) AS recall_total, sum(evidence_count) AS evidence_total, + max(last_recalled_at) AS last_recalled, max(bumped_at) AS bumped FROM loom.memory_usage_totals GROUP BY org, namespace, memory_id ) AS t INNER JOIN ( - SELECT org, namespace, memory_id, recall_count, evidence_count + SELECT org, namespace, memory_id, recall_count, evidence_count, last_recalled_at FROM loom.memory FINAL WHERE tombstoned_at IS NULL ) AS m ON m.org = t.org AND m.namespace = t.namespace AND m.memory_id = t.memory_id WHERE t.recall_total > m.recall_count OR (1 + t.evidence_total) > m.evidence_count + OR t.last_recalled > coalesce(m.last_recalled_at, toDateTime(0)) """ ) first = _utc((row or {}).get("first_unfolded")) if first <= UNIX_EPOCH: - return hwm + return max(UNIX_EPOCH, hwm - timedelta(seconds=1)) return first - timedelta(seconds=1) @@ -509,9 +516,13 @@ async def _derive_cold_start_watermark(ch: _CHClient, hwm: datetime) -> datetime """ -async def _fold_batch(ch: _CHClient, cfg: UsageFoldConfig, result: UsageFoldResult) -> bool: - """One batch past the watermark. Returns True when a full batch was - taken (there may be more), False when the backlog is drained.""" +async def _fold_batch( + ch: _CHClient, + cfg: UsageFoldConfig, + result: UsageFoldResult, + after: _Cursor, +) -> _Cursor | None: + """Fold one batch; return its tie-break cursor if more rows may remain.""" global _watermark # Backpressure: while the destination already carries a # queue of unfinished mutations, folding more only lengthens it. Skip the @@ -526,7 +537,7 @@ async def _fold_batch(ch: _CHClient, cfg: UsageFoldConfig, result: UsageFoldResu settings.memory_usage_fold_max_pending_mutations, _watermark.isoformat(), ) - return False + return None # Absolute totals per memory. The HAVING filter must come AFTER # the GROUP BY (a row-level bumped_at predicate would drop the # older unmerged SummingMergeTree part-rows of a freshly-bumped @@ -543,14 +554,15 @@ async def _fold_batch(ch: _CHClient, cfg: UsageFoldConfig, result: UsageFoldResu FROM loom.memory_usage_totals GROUP BY org, namespace, memory_id HAVING bumped > {wm:DateTime} - ORDER BY bumped ASC + AND (bumped, org, namespace, memory_id) > {after:Tuple(DateTime, String, String, String)} + ORDER BY bumped ASC, org ASC, namespace ASC, memory_id ASC LIMIT {lim:UInt32} """, - params={"wm": _watermark, "lim": int(cfg.max_memories_per_pass)}, + params={"wm": _watermark, "after": after, "lim": int(cfg.max_memories_per_pass)}, ) result.candidates += len(rows) if not rows: - return False + return None result.batches += 1 # Resolve each (org, namespace, memory_id) to the keys holding it, with their @@ -574,17 +586,27 @@ async def _fold_batch(ch: _CHClient, cfg: UsageFoldConfig, result: UsageFoldResu # Advance the watermark only after every fold in the batch landed — a # partial batch re-folds from the old watermark, which is safe because # targets are absolute. - max_bumped = max(_utc(r["bumped"]) for r in rows) - full = len(rows) >= int(cfg.max_memories_per_pass) - if full: - # Full batch: bumped_at is second-resolution, so boundary ties may - # exist past the LIMIT. Step back one second to re-fold the boundary - # next batch (idempotent) — unless that would stall the watermark. - new_wm = max_bumped - timedelta(seconds=1) - _watermark = new_wm if new_wm > _watermark else max_bumped - else: - _watermark = max_bumped - return full + cursor = max( + (_utc(r["bumped"]), *_fold_key(r["org"], r["namespace"], r["memory_id"])) for r in rows + ) + # Every batch retains the boundary second for the next pass. The cursor + # makes progress within this pass without dropping ties past the LIMIT. + _watermark = max(UNIX_EPOCH, cursor[0] - timedelta(seconds=1)) + return cursor if len(rows) >= int(cfg.max_memories_per_pass) else None + + +def _resume_after(watermark: datetime) -> _Cursor: + """Resume a partial scan unless reconciliation found an earlier second.""" + try: + saved = json.loads(pass_context.resume_cursor() or "[]") + if isinstance(saved, list) and len(saved) == 4 and all(isinstance(v, str) for v in saved): + at, org, namespace, mid = saved + bumped = _utc(datetime.fromisoformat(at)) + if bumped > UNIX_EPOCH and bumped - timedelta(seconds=1) <= watermark: + return bumped, org, namespace, mid + except ValueError: + pass # A truncated cursor restarts from destination reconciliation. + return UNIX_EPOCH, "", "", "" # ---------------------------------------------------------------- acted @@ -824,8 +846,7 @@ async def run_pass( cfg: UsageFoldConfig | None = None, ch: _CHClient | None = None, ) -> UsageFoldResult: - """One fold pass: probe for new usage, derive the watermark on a cold - start, then fold batches until caught up or the time budget is spent.""" + """Reconcile destination counters, then fold batches within the time budget.""" global _watermark, _watermark_derived cfg = cfg or UsageFoldConfig() result = UsageFoldResult(watermark=_watermark) @@ -857,25 +878,31 @@ def _done(outcome: dict[str, Any]) -> UsageFoldResult: log.warning("usage fold: acted pass failed (non-fatal): %s", e) pass_context.note_failure(e, "acted") - # Cheap no-op probe: skip the GROUP BY entirely when nothing new - # has landed since the last pass. + # Only an empty totals table can skip destination reconciliation: a + # stale version may have replaced folded counts without another event. probe = await ch.query_one("SELECT max(bumped_at) AS hwm FROM loom.memory_usage_totals") hwm = _utc((probe or {}).get("hwm")) if hwm <= UNIX_EPOCH: + pass_context.set_cursor(_RESET_CURSOR) return _done({"candidates": 0, "folded": 0, "skipped_no_memory": 0}) - if not _watermark_derived and _watermark <= UNIX_EPOCH: - _watermark = await _derive_cold_start_watermark(ch, hwm) - _watermark_derived = True - log.info("usage fold: cold-start watermark derived as %s", _watermark.isoformat()) - - if hwm <= _watermark: - return _done({"candidates": 0, "folded": 0, "skipped_no_memory": 0}) + resume = await _derive_watermark(ch, hwm) + _watermark = min(_watermark, resume) if _watermark_derived else resume + _watermark_derived = True + after = _resume_after(_watermark) + if after[0] > UNIX_EPOCH: + _watermark = min(_watermark, after[0] - timedelta(seconds=1)) + pass_context.set_cursor(json.dumps((after[0].isoformat(), *after[1:]))) while True: - more = await _fold_batch(ch, cfg, result) - if not more: + pass_context.check_stop() + cursor = await _fold_batch(ch, cfg, result, after) + if cursor is None: + if not result.skipped_pending_mutations: + pass_context.set_cursor(_RESET_CURSOR) break + after = cursor + pass_context.set_cursor(json.dumps((after[0].isoformat(), *after[1:]))) if time.monotonic() - started >= float(cfg.pass_time_budget_seconds): log.info("usage fold: time budget spent, resuming next pass") break diff --git a/tests/analyzer/cases/workers.py b/tests/analyzer/cases/workers.py index 20baad5ba3..075661a0b2 100644 --- a/tests/analyzer/cases/workers.py +++ b/tests/analyzer/cases/workers.py @@ -149,6 +149,8 @@ async def _worker_passes(ch: RecordingCH, c: dict[str, Any]) -> None: last_activity=now, ) await uf._fold_memory_rows(ch, [target], 10) + await uf._derive_watermark(ch, now) + await uf._fold_batch(ch, uf.UsageFoldConfig(), uf.UsageFoldResult(), (now, ORG, NS, "m1")) async def _admin_readers(ch: RecordingCH, c: dict[str, Any]) -> None: diff --git a/tests/analyzer/shapes.txt b/tests/analyzer/shapes.txt index 15e53e2e17..72db293d43 100644 --- a/tests/analyzer/shapes.txt +++ b/tests/analyzer/shapes.txt @@ -678,6 +678,7 @@ 516db52eddcc SELECT count() AS n FROM loom.session_brief_revisions WHERE org = {org:String} AND namespace = {ns:String} AND ? 51af520b7286 SELECT DISTINCT tier, qualified_name, symbol_id FROM loom.code_symbol_names WHERE org = {org:String} AND namespace = {ns:String} AND name IN {names:Array(String)} AND (qualified_name = {full:String} O 51f662fe04fe SELECT count() AS n FROM loom.learnings_queue WHERE org = {org:String} AND candidate_id IN {cids:Array(UUID)} AND status = ? +52083af6b301 SELECT count() AS n FROM system.mutations WHERE database = {db:String} AND table = {t:String} AND NOT is_done 52448e9cfd6b ALTER TABLE loom.memory_record_conflicts UPDATE tombstoned_at = {now:DateTime} WHERE (org = {org:String} AND tombstoned_at IS NULL AND namespace = {ns:String}) AND ({erasure_mutation_tag:String} = {er 524493debfe9 SELECT count() AS n FROM loom.datasets WHERE org = {org:String} AND namespace IN {ns:Array(String)} 525d9e3b3774 SELECT count() AS n FROM loom.working_memory FINAL WHERE org = {org:String} AND user_id = {uid:String} AND tombstoned_at IS NULL @@ -1047,6 +1048,7 @@ 7baded054596 SELECT count() AS n FROM loom.knowledge_ingestions WHERE org = {org:String} AND namespace = {ns:String} AND ? 7bb1d4059098 SELECT (SELECT count() FROM ( SELECT e.org AS org, e.namespace AS namespace, e.memory_key AS memory_key FROM ( SELECT org, namespace, agent, user_id, memory_key FROM loom.entity_vectors FINAL WHERE to 7bb9d891b190 SELECT thread_id, msg_id, role, content_excerpt AS embedded_text, ts, cosineDistance(embedding, {q:Array(Float32)}) AS distance FROM loom.conversation_embeddings FINAL WHERE org = {org:String} AND nam +7bc09b2b31bc SELECT org, namespace, toString(memory_id) AS memory_id, sum(recall_count) AS recall_total, sum(evidence_count) AS evidence_total, sum(used_count) AS used_total, sum(harm_count) AS harm_total, max(las 7bcba9ce8741 SELECT state_value FROM loom.code_index_state FINAL WHERE org = {org:String} AND namespace = {ns:String} AND state_key = ? LIMIT ? 7c06e9fdda88 ALTER TABLE loom.knowledge_uploads DELETE WHERE (org = {org:String} AND namespace = {ns:String}) AND ({erasure_mutation_tag:String} = {erasure_mutation_tag:String}) SETTINGS mutations_sync = ? 7c313a3b4c8e SELECT term, toUInt64(uniqExactMerge(df_state)) AS df FROM loom.code_term_freq WHERE org = {org:String} AND namespace = {ns:String} AND term IN {tokens:Array(String)} GROUP BY term @@ -1739,6 +1741,7 @@ c7ae15b6a560 SELECT path, count() AS chunks, min(start_line) AS first_line, coun c7c5d35e8502 SELECT memory_id, org, namespace, agent, user_id, session_id, memory_key, memory_type, value, ? AS value_truncated, confidence, version, pinned, char_limit, created_at, updated_at, expires_at, valid_a c7c8b7f2f21c ALTER TABLE loom.memory_record_conflicts DELETE WHERE (org = {org:String} AND (prior_record_id IN {ids:Array(UUID)} OR incoming_record_id IN {ids:Array(UUID)})) AND ({erasure_mutation_tag:String} = {e c7ec8090e470 SELECT memory_key, memory_type, content_excerpt, ? - cosineDistance(embedding, {qv:Array(Float32)}) AS similarity, toString(coalesce(temporal_anchor, toDateTime(?))) AS temporal_anchor, temporal_confi +c7f724d1ca61 SELECT min(t.bumped) AS first_unfolded FROM ( SELECT org, namespace, memory_id, sum(recall_count) AS recall_total, sum(evidence_count) AS evidence_total, max(last_recalled_at) AS last_recalled, max(bu c8167139ec34 DELETE FROM loom.memory_embeddings WHERE (org = {org:String} AND has(tags, ?) AND hasAny(tags, {src:Array(String)}) AND hasAny(arrayMap(t -> (namespace, t), tags), {pairs:Array(Tuple(String, String))} c8342f918649 SELECT concat(?, toString(c.session_id), ?, toString(c.chunk_index)) AS memory_key, ? AS memory_type, substring(c.content, ?, ?) AS content_excerpt, c.content AS content_full, c.agent AS agent, c.user c8424367f9c9 SELECT namespace, source_uri, count() AS chunk_count FROM loom.chunks WHERE org = {org:String} AND (namespace, source_uri) IN {keys:Array(Tuple(String, String))} AND doc_superseded_at IS NULL AND (org diff --git a/tests/unit/test_usage_fold_sql.py b/tests/unit/test_usage_fold_sql.py index 9729da7e43..3af165e928 100644 --- a/tests/unit/test_usage_fold_sql.py +++ b/tests/unit/test_usage_fold_sql.py @@ -137,7 +137,7 @@ async def test_a_cold_start_resumes_at_the_namespace_still_behind(tmp_path: Path f"('{MID}', 'o', 'b', 0, 3, 0, 0, '2026-01-01 00:00:00', '2026-01-02 00:00:00')" ) hwm = datetime(2026, 2, 1, tzinfo=UTC) - got = await usage_fold._derive_cold_start_watermark(kept, hwm) + got = await usage_fold._derive_watermark(kept, hwm) assert got == datetime(2026, 1, 1, 23, 59, 59, tzinfo=UTC) @@ -164,3 +164,162 @@ async def test_the_identity_read_returns_only_the_memories_of_the_batch(tmp_path ("ka", 0), ("kd", 2), ] + + +@pytest.fixture +def fold_db(tmp_path: Path, monkeypatch: pytest.MonkeyPatch): + async def _no_acted(_ch: Any, _result: Any) -> None: + pass + + usage_fold.reset_watermark() + monkeypatch.setattr(usage_fold, "acted_pass", _no_acted) + kept = KeptCH( + tmp_path, + ddl("memory"), + ddl("memory_usage_totals"), + ddl("memory_usage_events"), + ddl("memory_reinforcement"), + ) + yield kept + usage_fold.reset_watermark() + + +async def _usage( + kept: KeptCH, *, mid: str = MID, recall: int = 0, evidence: int = 0, at: datetime = SEEN +) -> None: + await kept.insert( + "loom.memory_usage_totals", + [ + { + "org": "o", + "namespace": "a", + "memory_id": mid, + "recall_count": recall, + "evidence_count": evidence, + "last_recalled_at": at if recall else datetime(1970, 1, 1), + "bumped_at": at, + } + ], + ) + + +async def test_late_same_second_totals_are_folded_on_the_next_pass(fold_db: KeptCH) -> None: + kept = fold_db + await _memory(kept, ("o", "a", "k", MID)) + await _usage(kept, recall=1, evidence=1) + await usage_fold.run_pass(ch=kept) + assert (await _counters(kept))[("o", "a", "k")][:2] == (1, 2) + + await _usage(kept, recall=2, evidence=3) # Same bumped_at, a later visible part. + await usage_fold.run_pass(ch=kept) + assert (await _counters(kept))[("o", "a", "k")] == (3, 5, SEEN.replace(tzinfo=None)) + await usage_fold.run_pass(ch=kept) # Absolute targets never double-count. + assert (await _counters(kept))[("o", "a", "k")][:2] == (3, 5) + + +@pytest.mark.parametrize("preserve_counts", [False, True], ids=["all-counters", "last-recall-only"]) +async def test_a_stale_winning_version_is_repaired_without_new_usage( + fold_db: KeptCH, preserve_counts: bool +) -> None: + from datetime import timedelta + + kept = fold_db + await _memory(kept, ("o", "a", "k", MID), ("o", "a", "later", MID2)) + [stale] = await kept.query_all("SELECT * FROM loom.memory FINAL WHERE memory_key = 'k'") + await _usage(kept, recall=3, evidence=2) + await _usage(kept, mid=MID2, recall=1, at=SEEN + timedelta(hours=1)) + await usage_fold.run_pass(ch=kept) + assert (await _counters(kept))[("o", "a", "k")][:2] == (3, 3) + + stale["version"] += 1 + if preserve_counts: + stale.update(recall_count=3, evidence_count=3) + await kept.insert("loom.memory", [stale]) + expected_counts = (3, 3) if preserve_counts else (0, 1) + assert (await _counters(kept))[("o", "a", "k")] == (*expected_counts, None) + await usage_fold.run_pass(ch=kept) + [actual] = await kept.query_all("SELECT * FROM loom.memory FINAL WHERE memory_key = 'k'") + assert actual == { + **stale, + "recall_count": 3, + "evidence_count": 3, + "last_recalled_at": SEEN.replace(tzinfo=None), + } + + +@pytest.mark.parametrize("handover", [False, True], ids=["one-pass", "handover-between-batches"]) +async def test_timestamp_ties_larger_than_the_cap_are_not_skipped( + fold_db: KeptCH, handover: bool +) -> None: + from server.workers import pass_context + + kept = fold_db + mids = [f"6b47c9b7-0000-0000-0000-{i:012d}" for i in range(1, 6)] + await _memory(kept, *(("o", "a", mid, mid) for mid in mids)) + for mid in mids: + await _usage(kept, mid=mid, recall=1) + cfg = usage_fold.UsageFoldConfig( + max_memories_per_pass=2, pass_time_budget_seconds=0 if handover else 120 + ) + folded = batches = 0 + cursor = "" + for _ in range(3 if handover else 1): + usage_fold.reset_watermark() + ctx = pass_context.begin("usage_fold", resume_cursor=cursor) + try: + result = await usage_fold.run_pass(ch=kept, cfg=cfg) + folded += result.folded + batches += result.batches + cursor = ctx.cursor + finally: + pass_context.clear() + assert folded == 5 + assert batches == 3 + assert cursor # An explicit reset survives the supervisor's nonempty-cursor filter. + assert set((await _counters(kept)).values()) == {(1, 1, SEEN.replace(tzinfo=None))} + + +async def _supervised_fold(kept: KeptCH, cfg: Any = None) -> Any: + import time + + from server.workers import pass_context, status_store, supervisor + + saved = await status_store.resume_cursor("usage_fold", ch=kept) + state = supervisor.LoopState(cursor=saved) + ctx = pass_context.begin("usage_fold", resume_cursor=saved) + started = time.monotonic() + try: + result = await usage_fold.run_pass(ch=kept, cfg=cfg) + supervisor._record_status_row( + "usage_fold", started=started, success=True, partial=ctx, state=state + ) + await status_store.flush() + return result + finally: + pass_context.clear() + + +async def test_completed_scan_persists_its_reset_before_same_second_repairs( + fold_db: KeptCH, +) -> None: + from server.workers import status_store + + kept = fold_db + kept.create(ddl("worker_status")) + mids = [f"6b47c9b7-0000-0000-0000-{i:012d}" for i in range(1, 4)] + await _memory(kept, *(("o", "a", mid, mid) for mid in mids)) + for mid in mids: + await _usage(kept, mid=mid, recall=1) + status_store.reset_for_tests(client=kept) + try: + first = await _supervised_fold( + kept, usage_fold.UsageFoldConfig(max_memories_per_pass=2, pass_time_budget_seconds=0) + ) + assert first.folded == 2 + assert (await _supervised_fold(kept)).folded == 1 + await _usage(kept, mid=mids[0], recall=1) + usage_fold.reset_watermark() # A restarted supervisor reads the stored reset. + await _supervised_fold(kept) + assert (await _counters(kept))[("o", "a", mids[0])][:2] == (2, 1) + finally: + status_store.reset_for_tests() diff --git a/tests/unit/test_usage_fold_worker.py b/tests/unit/test_usage_fold_worker.py index 65925cc563..c096bd05ee 100644 --- a/tests/unit/test_usage_fold_worker.py +++ b/tests/unit/test_usage_fold_worker.py @@ -1,7 +1,7 @@ """Usage fold-back worker — runs against a stub CH client. The stub answers queries in order: probe (query_one) → cold-start -watermark derivation (query_one, first pass only) → per batch: the +watermark reconciliation (query_one, every pass) → per batch: the pending-mutations probe (query_one, unseeded reads as 0), totals (query_all), then one identity read (query_all) per size chunk of its keys. A batch below the cap ends the pass; a full batch is followed by another @@ -196,19 +196,19 @@ async def test_noop_when_no_usage_at_all() -> None: assert ch.execute_calls == [] -async def test_noop_when_watermark_already_at_high_water() -> None: - """Second pass with no new events stops at the probe (no re-derivation).""" +async def test_unchanged_high_water_still_checks_for_regressed_counters() -> None: + """An unchanged timestamp cannot rule out a stale full-version insert.""" ch = StubCH() _seed_active_pass(ch, totals=[_totals_row()], idents=[_ident_row()]) await run_pass(ch=ch) - assert usage_fold.current_watermark() == T0.replace(tzinfo=UTC) + assert usage_fold.current_watermark() == (T0 - timedelta(seconds=1)).replace(tzinfo=UTC) ch2 = StubCH() - ch2.query_one_returns.append({"hwm": T0}) + ch2.query_one_returns += [{"hwm": T0}, {"first_unfolded": None}] result = await run_pass(ch=ch2) assert result.folded == 0 - assert len(ch2.query_one_calls) == 1 - assert ch2.query_all_calls == [] + assert len(ch2.query_one_calls) == 3 + assert len(ch2.query_all_calls) == 1 assert ch2.execute_calls == [] @@ -536,7 +536,7 @@ async def test_totals_for_erased_memory_are_skipped() -> None: assert result.folded == 0 assert result.skipped_no_memory == 1 assert ch.execute_calls == [] - assert usage_fold.current_watermark() == T0.replace(tzinfo=UTC) + assert usage_fold.current_watermark() == (T0 - timedelta(seconds=1)).replace(tzinfo=UTC) async def test_cold_start_watermark_is_derived_not_epoch() -> None: @@ -561,14 +561,14 @@ async def test_cold_start_watermark_is_derived_not_epoch() -> None: assert params["wm"] == (first_unfolded - timedelta(seconds=1)).replace(tzinfo=UTC) -async def test_cold_start_with_nothing_unfolded_jumps_to_high_water() -> None: +async def test_cold_start_with_nothing_unfolded_retains_high_water_second() -> None: ch = StubCH() ch.query_one_returns.append({"hwm": T0}) ch.query_one_returns.append({"first_unfolded": None}) result = await run_pass(ch=ch) assert result.candidates == 0 - assert ch.query_all_calls == [] - assert usage_fold.current_watermark() == T0.replace(tzinfo=UTC) + assert len(ch.query_all_calls) == 1 + assert usage_fold.current_watermark() == (T0 - timedelta(seconds=1)).replace(tzinfo=UTC) async def test_watermark_advances_to_max_bumped_of_batch() -> None: @@ -585,16 +585,15 @@ async def test_watermark_advances_to_max_bumped_of_batch() -> None: ) result = await run_pass(ch=ch) assert result.folded == 2 - assert usage_fold.current_watermark() == later.replace(tzinfo=UTC) + assert usage_fold.current_watermark() == (later - timedelta(seconds=1)).replace(tzinfo=UTC) # Filter AFTER aggregation: a row-level predicate undercounts unmerged parts. totals_sql = ch.query_all_calls[0][0] assert "HAVING bumped > {wm:DateTime}" in totals_sql assert "GROUP BY org, namespace, memory_id" in totals_sql -async def test_full_batch_steps_watermark_back_and_keeps_going() -> None: - """At the batch cap the pass takes another batch (stepping the watermark - back 1 s to re-fold tied boundary rows) until a short batch drains it.""" +async def test_full_batch_keeps_the_boundary_and_pages_after_its_last_identity() -> None: + """A full batch resumes past its last identity while retaining the boundary.""" later = T0 + timedelta(minutes=5) ch = StubCH() _seed_active_pass( @@ -606,16 +605,15 @@ async def test_full_batch_steps_watermark_back_and_keeps_going() -> None: idents=[_ident_row(), _ident_row(memory_id=MID2, memory_key="k2")], hwm=later, ) - # Second batch: the boundary row again (tie re-fold), short, so the pass ends. - ch.query_all_returns.append([_totals_row(memory_id=MID2, bumped=later)]) - ch.query_all_returns.append([_ident_row(memory_id=MID2, memory_key="k2")]) + # The next batch has no rows past the cursor. + ch.query_all_returns.append([]) result = await run_pass(cfg=UsageFoldConfig(max_memories_per_pass=2), ch=ch) - assert result.batches == 2 - assert result.candidates == 3 - assert len(ch.query_all_calls) == 4 - # Batch 2 started from later − 1 s, then the short batch pinned the watermark. + assert result.batches == 1 + assert result.candidates == 2 + assert len(ch.query_all_calls) == 3 + assert ch.query_all_calls[2][1]["after"] == (later.replace(tzinfo=UTC), *_key(MID2)) assert ch.query_all_calls[2][1]["wm"] == (later - timedelta(seconds=1)).replace(tzinfo=UTC) - assert usage_fold.current_watermark() == later.replace(tzinfo=UTC) + assert usage_fold.current_watermark() == (later - timedelta(seconds=1)).replace(tzinfo=UTC) async def test_time_budget_ends_a_pass_between_batches() -> None: @@ -862,3 +860,147 @@ async def test_the_identity_read_is_cut_by_size_too() -> None: for p in reads: for value in p.values(): assert len(format_bind_value(value, top_level=False).encode()) <= ch_params.FIELD_BYTES + + +@pytest.mark.parametrize("recall,evidence", [(2, 0), (0, 2)]) +async def test_late_usage_in_the_boundary_second_is_folded(recall: int, evidence: int) -> None: + """A second totals part becomes visible after a short batch, without a new bump.""" + ch = StubCH() + _seed_active_pass(ch, totals=[_totals_row(recall=1)], idents=[_ident_row()]) + await run_pass(ch=ch) + + late = StubCH() + _seed_active_pass( + late, + totals=[_totals_row(recall=recall, evidence=evidence)], + idents=[_ident_row()], + ) + result = await run_pass(ch=late) + assert result.folded == 1 + [(_, params)] = _updates(late, "loom.memory") + assert params["rcls"] == [recall] + assert params["evs"] == [1 + evidence] + assert late.query_all_calls[0][1]["wm"] < T0.replace(tzinfo=UTC) + assert usage_fold.current_watermark() == (T0 - timedelta(seconds=1)).replace(tzinfo=UTC) + + +async def test_a_later_stale_version_is_repaired_without_another_usage_event() -> None: + """Another replica inserts counters it read before the fold, behind the cursor.""" + later = T0 + timedelta(hours=1) + ch = StubCH() + _seed_active_pass( + ch, + totals=[_totals_row(), _totals_row(memory_id=MID2, bumped=later)], + idents=[_ident_row(), _ident_row(memory_id=MID2, memory_key="k2")], + hwm=later, + ) + await run_pass(ch=ch) + + # The earlier memory's winning version lost its folded counts. The totals + # and their high water mark are unchanged; destination comparison finds it. + repair = StubCH() + _seed_active_pass(repair, totals=[_totals_row()], idents=[_ident_row()], hwm=later) + result = await run_pass(ch=repair) + assert result.folded == 1 + assert repair.query_all_calls[0][1]["wm"] == (T0 - timedelta(seconds=1)).replace(tzinfo=UTC) + assert _updates(repair, "loom.memory")[0][1]["rcls"] == [3] + + +async def test_more_than_a_batch_of_tied_totals_all_fold() -> None: + """Ties paginate by identity, so the boundary overlap cannot starve a memory.""" + from tests.unit.ch_fake import RecordingCH + + mids = [f"6b47c9b7-0000-0000-0000-{i:012d}" for i in range(1, 6)] + + def _page(_sql: str, params: dict[str, Any]) -> list[dict[str, Any]]: + candidates = [ + _totals_row(memory_id=mid) + for mid in mids + if T0.replace(tzinfo=UTC) > params["wm"] + and (T0.replace(tzinfo=UTC), "clickhouse", "ns", mid) + > params.get("after", (datetime.min.replace(tzinfo=UTC), "", "", "")) + ] + return candidates[: params["lim"]] + + ch = ( + RecordingCH() + .on("max(bumped_at) AS hwm", [{"hwm": T0}]) + .on("min(t.bumped) AS first_unfolded", [{"first_unfolded": T0}]) + .on("system.mutations", [{"n": 0}]) + .on("HAVING bumped >", _page) + .on( + "toString(m.memory_id) AS mid", + lambda _sql, p: [_ident_row(memory_id=mid, memory_key=mid) for mid in p["ids"]], + ) + ) + result = await run_pass(cfg=UsageFoldConfig(max_memories_per_pass=2), ch=ch) + assert result.folded == len(mids) + assert result.batches == 3 + assert {str(row["memory_key"]) for row in ch.inserted(REINF)} == set(mids) + + +async def test_a_time_limited_tie_scan_resumes_after_handover() -> None: + from server.core.datetimes import UNIX_EPOCH + from server.workers import pass_context + + cfg = UsageFoldConfig(max_memories_per_pass=1, pass_time_budget_seconds=0) + ch = StubCH() + _seed_active_pass(ch, totals=[_totals_row()], idents=[_ident_row()]) + ctx = pass_context.begin("usage_fold") + try: + await run_pass(cfg=cfg, ch=ch) + cursor = ctx.cursor + assert cursor + finally: + pass_context.clear() + + usage_fold.reset_watermark() # Another replica resumes the stored cursor. + next_ch = StubCH() + _seed_active_pass( + next_ch, + totals=[_totals_row(memory_id=MID2)], + idents=[_ident_row(memory_id=MID2)], + ) + ctx = pass_context.begin("usage_fold", resume_cursor=cursor) + try: + await run_pass(ch=next_ch) + assert next_ch.query_all_calls[0][1]["after"] == (T0.replace(tzinfo=UTC), *_key(MID)) + assert ctx.cursor # The supervisor persists a nonempty reset for a drained scan. + ctx.resume_cursor = ctx.cursor + assert usage_fold._resume_after(T0.replace(tzinfo=UTC)) == (UNIX_EPOCH, "", "", "") + finally: + pass_context.clear() + + +@pytest.mark.parametrize( + "saved", ["broken", "null", "{}", "[1, 2, 3, 4]", '["bad", "o", "n", "m"]'] +) +def test_invalid_resume_cursor_restarts_from_reconciliation(saved: str) -> None: + from server.core.datetimes import UNIX_EPOCH + from server.workers import pass_context + + pass_context.begin("usage_fold", resume_cursor=saved) + try: + assert usage_fold._resume_after(T0.replace(tzinfo=UTC)) == (UNIX_EPOCH, "", "", "") + finally: + pass_context.clear() + + +async def test_reconciliation_rewinds_a_saved_cursor_for_an_older_stale_head() -> None: + import json + + from server.core.datetimes import UNIX_EPOCH + from server.workers import pass_context + + later = T0 + timedelta(hours=1) + cursor = json.dumps((later.isoformat(), *_key(MID2))) + ch = StubCH() + _seed_active_pass(ch, totals=[_totals_row()], idents=[_ident_row()], hwm=later) + ctx = pass_context.begin("usage_fold", resume_cursor=cursor) + try: + result = await run_pass(ch=ch) + assert result.folded == 1 + assert ch.query_all_calls[0][1]["after"] == (UNIX_EPOCH, "", "", "") + assert ctx.cursor + finally: + pass_context.clear() From ee11ba6e45931804c31a4787c79605d5845de4a8 Mon Sep 17 00:00:00 2001 From: Lareb Zafar Date: Fri, 20 Nov 2026 18:32:26 +0530 Subject: [PATCH 2/2] Usage fold scans resume within cursor and counter bounds Persist fixed-width identity digests so supported names do not truncate scan progress. Compare the UInt32 targets written by the fold so saturated totals cannot keep rewinding time-limited scans. Read the isolated SQL fixture before selecting its key to observe patches despite automatic PREWHERE in the pinned engine. Signed-off-by: Lareb Zafar --- docs/architecture/clickhouse.md | 9 ++-- server/memory/workers/usage_fold.py | 33 ++++++++---- tests/analyzer/shapes.txt | 4 +- tests/unit/test_usage_fold_sql.py | 79 ++++++++++++++++++++++++++-- tests/unit/test_usage_fold_worker.py | 18 ++++--- 5 files changed, 118 insertions(+), 25 deletions(-) diff --git a/docs/architecture/clickhouse.md b/docs/architecture/clickhouse.md index 17b98d32a1..652a7ec27f 100644 --- a/docs/architecture/clickhouse.md +++ b/docs/architecture/clickhouse.md @@ -670,14 +670,17 @@ with `purge=true`, runner state, and the supersession rows `knowledge.restore` r unfinished mutations. Only "lightweight update unsupported" errors fall back to `ALTER TABLE … UPDATE`; a transient error is retried next pass. Every batch retains its boundary second because `bumped_at` has second resolution. A cursor ordered by - `(bumped_at, org, namespace, memory_id)` pages through ties; `worker_status` stores it - when a pass stops at its time budget. A completed scan stores an epoch cursor to + `bumped_at`, SHA-256 digests of `org` and `namespace`, and `memory_id` pages through + ties. Fixed-width digests fit supported identity sizes within the cursor limit. + `worker_status` stores the cursor when a pass stops at its time budget. A completed scan stores an epoch cursor to reset the scan: an empty cursor tells the supervisor to keep its previous value. A full-version insert can carry counters read before a fold and replace its result. Every pass therefore compares the absolute totals and last-recall time with the live rows, even when the usage high-water mark is unchanged, and rewinds to the earliest - discrepancy. This adds an aggregate join per nonempty pass. Repair is eventual once + discrepancy. Comparisons use the same UInt32 saturation as the counter writes, so + an unattainable total cannot keep rewinding the scan. This adds an aggregate join + per nonempty pass. Repair is eventual once the competing writes are visible; a read followed by an insert is not atomic. Absolute targets and `greatest` make counter repair idempotent across replicas. The usage events remain authoritative; the counters on `memory` are a derived projection. diff --git a/server/memory/workers/usage_fold.py b/server/memory/workers/usage_fold.py index 990a382eb5..e6c78c2034 100644 --- a/server/memory/workers/usage_fold.py +++ b/server/memory/workers/usage_fold.py @@ -60,8 +60,9 @@ insert can carry counts read before a fold and must be repaired even when no usage timestamp changes. This is eventual reconciliation, not an atomic update of future versions. A cold start begins at the earliest discrepancy -instead of the epoch. A cursor on (bumped_at, org, namespace, memory_id) -pages through ties and is saved in worker_status when the time budget is spent. +instead of the epoch. A cursor on bumped_at, fixed-width identity digests, +and memory_id pages through ties and is saved in worker_status when the time +budget is spent. Digests keep supported identity sizes within its cursor limit. """ from __future__ import annotations @@ -290,6 +291,16 @@ def _fold_key(org: Any, namespace: Any, memory_id: Any) -> tuple[str, str, str]: return str(org), str(namespace), str(memory_id) +def _cursor_identity(org: Any, namespace: Any, memory_id: Any) -> tuple[str, str, str]: + """Fixed-width sort components fit the supervisor's bounded cursor.""" + org, namespace, memory_id = _fold_key(org, namespace, memory_id) + return ( + hashlib.sha256(org.encode()).hexdigest().upper(), + hashlib.sha256(namespace.encode()).hexdigest().upper(), + memory_id, + ) + + def _scope_params(keys: list[tuple[str, str, str]]) -> dict[str, Any]: """`_FOLD_SCOPE`'s parameters. `keys` stays in the caller's order: the UPDATE's transform() pairs it with the targets by position.""" @@ -467,7 +478,7 @@ async def _derive_watermark(ch: _CHClient, hwm: datetime) -> datetime: and re-folds when its row is next bumped. With no discrepancy, retain the high water mark's second for usage that becomes visible after this read.""" row = await ch.query_one( - """ + f""" SELECT min(t.bumped) AS first_unfolded FROM ( SELECT org, namespace, memory_id, @@ -483,8 +494,8 @@ async def _derive_watermark(ch: _CHClient, hwm: datetime) -> datetime: FROM loom.memory FINAL WHERE tombstoned_at IS NULL ) AS m ON m.org = t.org AND m.namespace = t.namespace AND m.memory_id = t.memory_id - WHERE t.recall_total > m.recall_count - OR (1 + t.evidence_total) > m.evidence_count + WHERE least(t.recall_total, {_UINT32_MAX}) > m.recall_count + OR (1 + least(t.evidence_total, {_UINT32_MAX - 1})) > m.evidence_count OR t.last_recalled > coalesce(m.last_recalled_at, toDateTime(0)) """ ) @@ -554,8 +565,9 @@ async def _fold_batch( FROM loom.memory_usage_totals GROUP BY org, namespace, memory_id HAVING bumped > {wm:DateTime} - AND (bumped, org, namespace, memory_id) > {after:Tuple(DateTime, String, String, String)} - ORDER BY bumped ASC, org ASC, namespace ASC, memory_id ASC + AND (bumped, hex(SHA256(org)), hex(SHA256(namespace)), memory_id) + > {after:Tuple(DateTime, String, String, String)} + ORDER BY bumped ASC, hex(SHA256(org)) ASC, hex(SHA256(namespace)) ASC, memory_id ASC LIMIT {lim:UInt32} """, params={"wm": _watermark, "after": after, "lim": int(cfg.max_memories_per_pass)}, @@ -587,7 +599,8 @@ async def _fold_batch( # partial batch re-folds from the old watermark, which is safe because # targets are absolute. cursor = max( - (_utc(r["bumped"]), *_fold_key(r["org"], r["namespace"], r["memory_id"])) for r in rows + (_utc(r["bumped"]), *_cursor_identity(r["org"], r["namespace"], r["memory_id"])) + for r in rows ) # Every batch retains the boundary second for the next pass. The cursor # makes progress within this pass without dropping ties past the LIMIT. @@ -600,10 +613,10 @@ def _resume_after(watermark: datetime) -> _Cursor: try: saved = json.loads(pass_context.resume_cursor() or "[]") if isinstance(saved, list) and len(saved) == 4 and all(isinstance(v, str) for v in saved): - at, org, namespace, mid = saved + at, org_hash, namespace_hash, mid = saved bumped = _utc(datetime.fromisoformat(at)) if bumped > UNIX_EPOCH and bumped - timedelta(seconds=1) <= watermark: - return bumped, org, namespace, mid + return bumped, org_hash, namespace_hash, mid except ValueError: pass # A truncated cursor restarts from destination reconciliation. return UNIX_EPOCH, "", "", "" diff --git a/tests/analyzer/shapes.txt b/tests/analyzer/shapes.txt index 72db293d43..be55ad6d2e 100644 --- a/tests/analyzer/shapes.txt +++ b/tests/analyzer/shapes.txt @@ -1048,7 +1048,6 @@ 7baded054596 SELECT count() AS n FROM loom.knowledge_ingestions WHERE org = {org:String} AND namespace = {ns:String} AND ? 7bb1d4059098 SELECT (SELECT count() FROM ( SELECT e.org AS org, e.namespace AS namespace, e.memory_key AS memory_key FROM ( SELECT org, namespace, agent, user_id, memory_key FROM loom.entity_vectors FINAL WHERE to 7bb9d891b190 SELECT thread_id, msg_id, role, content_excerpt AS embedded_text, ts, cosineDistance(embedding, {q:Array(Float32)}) AS distance FROM loom.conversation_embeddings FINAL WHERE org = {org:String} AND nam -7bc09b2b31bc SELECT org, namespace, toString(memory_id) AS memory_id, sum(recall_count) AS recall_total, sum(evidence_count) AS evidence_total, sum(used_count) AS used_total, sum(harm_count) AS harm_total, max(las 7bcba9ce8741 SELECT state_value FROM loom.code_index_state FINAL WHERE org = {org:String} AND namespace = {ns:String} AND state_key = ? LIMIT ? 7c06e9fdda88 ALTER TABLE loom.knowledge_uploads DELETE WHERE (org = {org:String} AND namespace = {ns:String}) AND ({erasure_mutation_tag:String} = {erasure_mutation_tag:String}) SETTINGS mutations_sync = ? 7c313a3b4c8e SELECT term, toUInt64(uniqExactMerge(df_state)) AS df FROM loom.code_term_freq WHERE org = {org:String} AND namespace = {ns:String} AND term IN {tokens:Array(String)} GROUP BY term @@ -1441,6 +1440,7 @@ a613795f2fe3 SELECT count() AS n FROM loom.operator_edges FINAL WHERE org = {org a62720ec9473 SELECT org, namespace, agent, user_id, session_id, queue_id, payload, message_count, content_chars, status, attempts, lease_until, last_error, written, enqueued_at, updated_at, version, enqueued_by FR a627d0337788 ALTER TABLE loom.memory_ask_precedents DELETE WHERE (org = {org:String} AND namespace IN {ns:Array(String)}) AND ({erasure_mutation_tag:String} = {erasure_mutation_tag:String}) SETTINGS mutations_sync a63dc159ea03 SELECT count() AS n, argMax(role, ts) AS role, argMax(content, ts) AS content, argMax(run_id, ts) AS run_id, max(ts) AS last_ts, max(saved_at) AS last_saved_at, groupUniqArray(user_id) AS owners, coun +a64937812d3d SELECT org, namespace, toString(memory_id) AS memory_id, sum(recall_count) AS recall_total, sum(evidence_count) AS evidence_total, sum(used_count) AS used_total, sum(harm_count) AS harm_total, max(las a661d7b26199 ALTER TABLE loom.memory_extraction_queue DELETE WHERE (org = {org:String} AND user_id = {uid:String}) AND ({erasure_mutation_tag:String} = {erasure_mutation_tag:String}) SETTINGS mutations_sync = ? a662cd95ca59 SELECT memory_key, (status != ? OR tombstoned_at IS NOT NULL OR redacted_at IS NOT NULL) AS taken_out FROM loom.memory FINAL WHERE org = {org:String} AND namespace = {ns:String} AND agent = {agent:Str a67dc6a1b98a SELECT path, start_line, end_line, content_hash FROM loom.code_chunks FINAL WHERE org = {org:String} AND namespace = {ns:String} AND path IN {paths:Array(String)} AND tombstoned_at IS NULL AND start_l @@ -1741,7 +1741,6 @@ c7ae15b6a560 SELECT path, count() AS chunks, min(start_line) AS first_line, coun c7c5d35e8502 SELECT memory_id, org, namespace, agent, user_id, session_id, memory_key, memory_type, value, ? AS value_truncated, confidence, version, pinned, char_limit, created_at, updated_at, expires_at, valid_a c7c8b7f2f21c ALTER TABLE loom.memory_record_conflicts DELETE WHERE (org = {org:String} AND (prior_record_id IN {ids:Array(UUID)} OR incoming_record_id IN {ids:Array(UUID)})) AND ({erasure_mutation_tag:String} = {e c7ec8090e470 SELECT memory_key, memory_type, content_excerpt, ? - cosineDistance(embedding, {qv:Array(Float32)}) AS similarity, toString(coalesce(temporal_anchor, toDateTime(?))) AS temporal_anchor, temporal_confi -c7f724d1ca61 SELECT min(t.bumped) AS first_unfolded FROM ( SELECT org, namespace, memory_id, sum(recall_count) AS recall_total, sum(evidence_count) AS evidence_total, max(last_recalled_at) AS last_recalled, max(bu c8167139ec34 DELETE FROM loom.memory_embeddings WHERE (org = {org:String} AND has(tags, ?) AND hasAny(tags, {src:Array(String)}) AND hasAny(arrayMap(t -> (namespace, t), tags), {pairs:Array(Tuple(String, String))} c8342f918649 SELECT concat(?, toString(c.session_id), ?, toString(c.chunk_index)) AS memory_key, ? AS memory_type, substring(c.content, ?, ?) AS content_excerpt, c.content AS content_full, c.agent AS agent, c.user c8424367f9c9 SELECT namespace, source_uri, count() AS chunk_count FROM loom.chunks WHERE org = {org:String} AND (namespace, source_uri) IN {keys:Array(Tuple(String, String))} AND doc_superseded_at IS NULL AND (org @@ -1750,6 +1749,7 @@ c853d7bbd170 SELECT org, agent, count() AS calls, countIf(if(consumer = ?, args[ c85b7e038508 SELECT namespace, kind FROM loom.namespaces FINAL WHERE org = {org:String} c873c5cd3338 SELECT retention_days, dedup_threshold, hot_quota, conflict_policy FROM loom.namespace_types FINAL WHERE org = {org:String} AND namespace = {ns:String} AND memory_type = {mt:String} LIMIT ? c87677674058 SELECT DISTINCT namespace FROM loom.search_misses WHERE org = {org:String} AND (has({spellings:Array(String)}, asker) OR has({spellings:Array(String)}, lowerUTF8(user_id))) +c8870c302e09 SELECT min(t.bumped) AS first_unfolded FROM ( SELECT org, namespace, memory_id, sum(recall_count) AS recall_total, sum(evidence_count) AS evidence_total, max(last_recalled_at) AS last_recalled, max(bu c892414a0cf3 SELECT role, granted_by, status, pending_role, pending_since FROM loom.namespace_grants FINAL WHERE org = {org:String} AND namespace = {ns:String} AND subject = {sub:String} AND revoked_at IS NULL AND c8ce735d214f SELECT count() AS n FROM loom.memory_embeddings FINAL WHERE org = {org:String} AND tombstoned_at IS NULL AND namespace IN {ns:Array(String)} c8e4a254b959 SELECT count() AS n FROM loom.conversation_threads FINAL WHERE org = {org:String} AND tombstoned_at IS NULL AND user_id = {uid:String} diff --git a/tests/unit/test_usage_fold_sql.py b/tests/unit/test_usage_fold_sql.py index 3af165e928..e422b66939 100644 --- a/tests/unit/test_usage_fold_sql.py +++ b/tests/unit/test_usage_fold_sql.py @@ -185,14 +185,21 @@ async def _no_acted(_ch: Any, _result: Any) -> None: async def _usage( - kept: KeptCH, *, mid: str = MID, recall: int = 0, evidence: int = 0, at: datetime = SEEN + kept: KeptCH, + *, + mid: str = MID, + recall: int = 0, + evidence: int = 0, + at: datetime = SEEN, + org: str = "o", + namespace: str = "a", ) -> None: await kept.insert( "loom.memory_usage_totals", [ { - "org": "o", - "namespace": "a", + "org": org, + "namespace": namespace, "memory_id": mid, "recall_count": recall, "evidence_count": evidence, @@ -238,7 +245,12 @@ async def test_a_stale_winning_version_is_repaired_without_new_usage( expected_counts = (3, 3) if preserve_counts else (0, 1) assert (await _counters(kept))[("o", "a", "k")] == (*expected_counts, None) await usage_fold.run_pass(ch=kept) - [actual] = await kept.query_all("SELECT * FROM loom.memory FINAL WHERE memory_key = 'k'") + # A key predicate can move into PREWHERE and hide unapplied patch columns. + [actual] = [ + row + for row in await kept.query_all("SELECT * FROM loom.memory FINAL") + if row["memory_key"] == "k" + ] assert actual == { **stale, "recall_count": 3, @@ -323,3 +335,62 @@ async def test_completed_scan_persists_its_reset_before_same_second_repairs( assert (await _counters(kept))[("o", "a", mids[0])][:2] == (2, 1) finally: status_store.reset_for_tests() + + +@pytest.mark.parametrize( + "org,namespace", [("o" * 500, "a"), ("o", "n" * 500)], ids=["long-org", "long-namespace"] +) +async def test_long_identity_ties_resume_through_the_bounded_status_cursor( + fold_db: KeptCH, org: str, namespace: str +) -> None: + from server.workers import status_store + + kept = fold_db + kept.create(ddl("worker_status")) + mids = [f"6b47c9b7-0000-0000-0000-{i:012d}" for i in range(1, 6)] + await _memory(kept, *((org, namespace, mid, mid) for mid in mids)) + for mid in mids: + await _usage(kept, mid=mid, recall=1, org=org, namespace=namespace) + cfg = usage_fold.UsageFoldConfig(max_memories_per_pass=2, pass_time_budget_seconds=0) + status_store.reset_for_tests(client=kept) + try: + for _ in range(3): + usage_fold.reset_watermark() + await _supervised_fold(kept, cfg) + assert set((await _counters(kept)).values()) == {(1, 1, SEEN.replace(tzinfo=None))} + finally: + status_store.reset_for_tests() + + +@pytest.mark.parametrize("counter", ["recall", "evidence"]) +async def test_saturated_counters_do_not_keep_rewinding_a_time_limited_scan( + fold_db: KeptCH, counter: str +) -> None: + from datetime import timedelta + + from server.workers import status_store + + kept = fold_db + kept.create(ddl("worker_status")) + mids = [f"6b47c9b7-0000-0000-0000-{i:012d}" for i in range(1, 4)] + await _memory(kept, *(("o", "a", mid, mid) for mid in mids)) + for i, mid in enumerate(mids): + await _usage( + kept, + mid=mid, + recall=2**32 if i == 0 and counter == "recall" else 1, + evidence=2**32 if i == 0 and counter == "evidence" else 0, + at=SEEN + timedelta(hours=i), + ) + cfg = usage_fold.UsageFoldConfig(max_memories_per_pass=1, pass_time_budget_seconds=0) + status_store.reset_for_tests(client=kept) + try: + for _ in range(3): + usage_fold.reset_watermark() + await _supervised_fold(kept, cfg) + counters = await _counters(kept) + expected = (2**32 - 1, 1) if counter == "recall" else (1, 2**32 - 1) + assert counters[("o", "a", mids[0])][:2] == expected + assert counters[("o", "a", mids[-1])][:2] == (1, 1) + finally: + status_store.reset_for_tests() diff --git a/tests/unit/test_usage_fold_worker.py b/tests/unit/test_usage_fold_worker.py index c096bd05ee..9c8465dd7f 100644 --- a/tests/unit/test_usage_fold_worker.py +++ b/tests/unit/test_usage_fold_worker.py @@ -554,8 +554,8 @@ async def test_cold_start_watermark_is_derived_not_epoch() -> None: await run_pass(ch=ch) derive_sql, _ = ch.query_one_calls[1] assert "min(t.bumped) AS first_unfolded" in derive_sql - assert "t.recall_total > m.recall_count" in derive_sql - assert "(1 + t.evidence_total) > m.evidence_count" in derive_sql + assert "least(t.recall_total, 4294967295) > m.recall_count" in derive_sql + assert "(1 + least(t.evidence_total, 4294967294)) > m.evidence_count" in derive_sql # The totals query was bounded by the derived watermark (first_unfolded − 1 s). _, params = ch.query_all_calls[0] assert params["wm"] == (first_unfolded - timedelta(seconds=1)).replace(tzinfo=UTC) @@ -611,7 +611,10 @@ async def test_full_batch_keeps_the_boundary_and_pages_after_its_last_identity() assert result.batches == 1 assert result.candidates == 2 assert len(ch.query_all_calls) == 3 - assert ch.query_all_calls[2][1]["after"] == (later.replace(tzinfo=UTC), *_key(MID2)) + assert ch.query_all_calls[2][1]["after"] == ( + later.replace(tzinfo=UTC), + *usage_fold._cursor_identity(*_key(MID2)), + ) assert ch.query_all_calls[2][1]["wm"] == (later - timedelta(seconds=1)).replace(tzinfo=UTC) assert usage_fold.current_watermark() == (later - timedelta(seconds=1)).replace(tzinfo=UTC) @@ -917,7 +920,7 @@ def _page(_sql: str, params: dict[str, Any]) -> list[dict[str, Any]]: _totals_row(memory_id=mid) for mid in mids if T0.replace(tzinfo=UTC) > params["wm"] - and (T0.replace(tzinfo=UTC), "clickhouse", "ns", mid) + and (T0.replace(tzinfo=UTC), *usage_fold._cursor_identity("clickhouse", "ns", mid)) > params.get("after", (datetime.min.replace(tzinfo=UTC), "", "", "")) ] return candidates[: params["lim"]] @@ -964,7 +967,10 @@ async def test_a_time_limited_tie_scan_resumes_after_handover() -> None: ctx = pass_context.begin("usage_fold", resume_cursor=cursor) try: await run_pass(ch=next_ch) - assert next_ch.query_all_calls[0][1]["after"] == (T0.replace(tzinfo=UTC), *_key(MID)) + assert next_ch.query_all_calls[0][1]["after"] == ( + T0.replace(tzinfo=UTC), + *usage_fold._cursor_identity(*_key(MID)), + ) assert ctx.cursor # The supervisor persists a nonempty reset for a drained scan. ctx.resume_cursor = ctx.cursor assert usage_fold._resume_after(T0.replace(tzinfo=UTC)) == (UNIX_EPOCH, "", "", "") @@ -993,7 +999,7 @@ async def test_reconciliation_rewinds_a_saved_cursor_for_an_older_stale_head() - from server.workers import pass_context later = T0 + timedelta(hours=1) - cursor = json.dumps((later.isoformat(), *_key(MID2))) + cursor = json.dumps((later.isoformat(), *usage_fold._cursor_identity(*_key(MID2)))) ch = StubCH() _seed_active_pass(ch, totals=[_totals_row()], idents=[_ident_row()], hwm=later) ctx = pass_context.begin("usage_fold", resume_cursor=cursor)