diff --git a/docs/architecture/clickhouse.md b/docs/architecture/clickhouse.md index 4371a6e37f..652a7ec27f 100644 --- a/docs/architecture/clickhouse.md +++ b/docs/architecture/clickhouse.md @@ -668,7 +668,23 @@ with `purge=true`, runner state, and the supersession rows `knowledge.restore` r (`server/memory/workers/usage_fold.py`). On ClickHouse Cloud each statement is a patch part and a Keeper entry, so cost is per statement. A pass is skipped while the table has too many unfinished mutations. Only "lightweight update unsupported" errors fall back to - `ALTER TABLE … UPDATE`; a transient error is retried next pass. + `ALTER TABLE … UPDATE`; a transient error is retried next pass. Every batch retains + its boundary second because `bumped_at` has second resolution. A cursor ordered by + `bumped_at`, SHA-256 digests of `org` and `namespace`, and `memory_id` pages through + ties. Fixed-width digests fit supported identity sizes within the cursor limit. + `worker_status` stores the cursor when a pass stops at its time budget. A completed scan stores an epoch cursor to + reset the scan: an empty cursor tells the supervisor to keep its previous value. + + A full-version insert can carry counters read before a fold and replace its result. + Every pass therefore compares the absolute totals and last-recall time with the live + rows, even when the usage high-water mark is unchanged, and rewinds to the earliest + discrepancy. Comparisons use the same UInt32 saturation as the counter writes, so + an unattainable total cannot keep rewinding the scan. This adds an aggregate join + per nonempty pass. Repair is eventual once + the competing writes are visible; a read followed by an insert is not atomic. Absolute + targets and `greatest` make counter repair idempotent across replicas. The usage events + remain authoritative; the counters on `memory` are a derived projection. + - **Erasure purges** remove rows. On `erasure/store/mutations.py::LIGHTWEIGHT_PURGE_TABLES` (vector, chunk, summary and evidence tables) a purge uses lightweight `DELETE FROM`, which writes only the row mask: rows vanish from every read at once and bytes leave at the next merge. diff --git a/server/memory/workers/usage_fold.py b/server/memory/workers/usage_fold.py index 747b355d10..e6c78c2034 100644 --- a/server/memory/workers/usage_fold.py +++ b/server/memory/workers/usage_fold.py @@ -55,17 +55,20 @@ statements instead of 2,000; on ClickHouse Cloud each lightweight UPDATE is a patch part and the cost is per statement, not per row. -The watermark is process-local. On a cold start it is derived from the -destination tables (the earliest `bumped_at` whose totals exceed what -`loom.memory` already carries) instead of the epoch, so a redeploy does -not re-walk the whole history at the batch cap before reaching live -rows, and a pass keeps taking batches until it is caught up or its time -budget is spent. +The watermark is process-local and retains the boundary second. Every pass +compares totals with the destination counters: a concurrent full-version +insert can carry counts read before a fold and must be repaired even when +no usage timestamp changes. This is eventual reconciliation, not an atomic +update of future versions. A cold start begins at the earliest discrepancy +instead of the epoch. A cursor on bumped_at, fixed-width identity digests, +and memory_id pages through ties and is saved in worker_status when the time +budget is spent. Digests keep supported identity sizes within its cursor limit. """ from __future__ import annotations import hashlib +import json import logging import time from dataclasses import dataclass, field @@ -94,6 +97,10 @@ _UINT32_MAX = 2**32 - 1 +_Cursor = tuple[datetime, str, str, str] +# Empty cursors preserve the supervisor's last value; an epoch cursor resets it. +_RESET_CURSOR = json.dumps((UNIX_EPOCH.isoformat(), "", "", "")) + _watermark: datetime = UNIX_EPOCH _watermark_derived: bool = False @@ -284,6 +291,16 @@ def _fold_key(org: Any, namespace: Any, memory_id: Any) -> tuple[str, str, str]: return str(org), str(namespace), str(memory_id) +def _cursor_identity(org: Any, namespace: Any, memory_id: Any) -> tuple[str, str, str]: + """Fixed-width sort components fit the supervisor's bounded cursor.""" + org, namespace, memory_id = _fold_key(org, namespace, memory_id) + return ( + hashlib.sha256(org.encode()).hexdigest().upper(), + hashlib.sha256(namespace.encode()).hexdigest().upper(), + memory_id, + ) + + def _scope_params(keys: list[tuple[str, str, str]]) -> dict[str, Any]: """`_FOLD_SCOPE`'s parameters. `keys` stays in the caller's order: the UPDATE's transform() pairs it with the targets by position.""" @@ -454,36 +471,37 @@ async def _fold_reinforcement_rows(ch: _CHClient, targets: list[_Target], chunk: return statements -async def _derive_cold_start_watermark(ch: _CHClient, hwm: datetime) -> datetime: - """Where to start after a restart: just before the earliest totals row - whose recall/evidence exceed what `loom.memory` already carries. Targets +async def _derive_watermark(ch: _CHClient, hwm: datetime) -> datetime: + """Reconcile from the earliest totals row ahead of a live memory. Targets are absolute, so the join is exact for the two counters that live on `loom.memory`; `used`-only activity has no destination counter there - and re-folds when its row is next bumped. No unfolded rows → the high - water mark (nothing to do).""" + and re-folds when its row is next bumped. With no discrepancy, retain the + high water mark's second for usage that becomes visible after this read.""" row = await ch.query_one( - """ + f""" SELECT min(t.bumped) AS first_unfolded FROM ( SELECT org, namespace, memory_id, sum(recall_count) AS recall_total, sum(evidence_count) AS evidence_total, + max(last_recalled_at) AS last_recalled, max(bumped_at) AS bumped FROM loom.memory_usage_totals GROUP BY org, namespace, memory_id ) AS t INNER JOIN ( - SELECT org, namespace, memory_id, recall_count, evidence_count + SELECT org, namespace, memory_id, recall_count, evidence_count, last_recalled_at FROM loom.memory FINAL WHERE tombstoned_at IS NULL ) AS m ON m.org = t.org AND m.namespace = t.namespace AND m.memory_id = t.memory_id - WHERE t.recall_total > m.recall_count - OR (1 + t.evidence_total) > m.evidence_count + WHERE least(t.recall_total, {_UINT32_MAX}) > m.recall_count + OR (1 + least(t.evidence_total, {_UINT32_MAX - 1})) > m.evidence_count + OR t.last_recalled > coalesce(m.last_recalled_at, toDateTime(0)) """ ) first = _utc((row or {}).get("first_unfolded")) if first <= UNIX_EPOCH: - return hwm + return max(UNIX_EPOCH, hwm - timedelta(seconds=1)) return first - timedelta(seconds=1) @@ -509,9 +527,13 @@ async def _derive_cold_start_watermark(ch: _CHClient, hwm: datetime) -> datetime """ -async def _fold_batch(ch: _CHClient, cfg: UsageFoldConfig, result: UsageFoldResult) -> bool: - """One batch past the watermark. Returns True when a full batch was - taken (there may be more), False when the backlog is drained.""" +async def _fold_batch( + ch: _CHClient, + cfg: UsageFoldConfig, + result: UsageFoldResult, + after: _Cursor, +) -> _Cursor | None: + """Fold one batch; return its tie-break cursor if more rows may remain.""" global _watermark # Backpressure: while the destination already carries a # queue of unfinished mutations, folding more only lengthens it. Skip the @@ -526,7 +548,7 @@ async def _fold_batch(ch: _CHClient, cfg: UsageFoldConfig, result: UsageFoldResu settings.memory_usage_fold_max_pending_mutations, _watermark.isoformat(), ) - return False + return None # Absolute totals per memory. The HAVING filter must come AFTER # the GROUP BY (a row-level bumped_at predicate would drop the # older unmerged SummingMergeTree part-rows of a freshly-bumped @@ -543,14 +565,16 @@ async def _fold_batch(ch: _CHClient, cfg: UsageFoldConfig, result: UsageFoldResu FROM loom.memory_usage_totals GROUP BY org, namespace, memory_id HAVING bumped > {wm:DateTime} - ORDER BY bumped ASC + AND (bumped, hex(SHA256(org)), hex(SHA256(namespace)), memory_id) + > {after:Tuple(DateTime, String, String, String)} + ORDER BY bumped ASC, hex(SHA256(org)) ASC, hex(SHA256(namespace)) ASC, memory_id ASC LIMIT {lim:UInt32} """, - params={"wm": _watermark, "lim": int(cfg.max_memories_per_pass)}, + params={"wm": _watermark, "after": after, "lim": int(cfg.max_memories_per_pass)}, ) result.candidates += len(rows) if not rows: - return False + return None result.batches += 1 # Resolve each (org, namespace, memory_id) to the keys holding it, with their @@ -574,17 +598,28 @@ async def _fold_batch(ch: _CHClient, cfg: UsageFoldConfig, result: UsageFoldResu # Advance the watermark only after every fold in the batch landed — a # partial batch re-folds from the old watermark, which is safe because # targets are absolute. - max_bumped = max(_utc(r["bumped"]) for r in rows) - full = len(rows) >= int(cfg.max_memories_per_pass) - if full: - # Full batch: bumped_at is second-resolution, so boundary ties may - # exist past the LIMIT. Step back one second to re-fold the boundary - # next batch (idempotent) — unless that would stall the watermark. - new_wm = max_bumped - timedelta(seconds=1) - _watermark = new_wm if new_wm > _watermark else max_bumped - else: - _watermark = max_bumped - return full + cursor = max( + (_utc(r["bumped"]), *_cursor_identity(r["org"], r["namespace"], r["memory_id"])) + for r in rows + ) + # Every batch retains the boundary second for the next pass. The cursor + # makes progress within this pass without dropping ties past the LIMIT. + _watermark = max(UNIX_EPOCH, cursor[0] - timedelta(seconds=1)) + return cursor if len(rows) >= int(cfg.max_memories_per_pass) else None + + +def _resume_after(watermark: datetime) -> _Cursor: + """Resume a partial scan unless reconciliation found an earlier second.""" + try: + saved = json.loads(pass_context.resume_cursor() or "[]") + if isinstance(saved, list) and len(saved) == 4 and all(isinstance(v, str) for v in saved): + at, org_hash, namespace_hash, mid = saved + bumped = _utc(datetime.fromisoformat(at)) + if bumped > UNIX_EPOCH and bumped - timedelta(seconds=1) <= watermark: + return bumped, org_hash, namespace_hash, mid + except ValueError: + pass # A truncated cursor restarts from destination reconciliation. + return UNIX_EPOCH, "", "", "" # ---------------------------------------------------------------- acted @@ -824,8 +859,7 @@ async def run_pass( cfg: UsageFoldConfig | None = None, ch: _CHClient | None = None, ) -> UsageFoldResult: - """One fold pass: probe for new usage, derive the watermark on a cold - start, then fold batches until caught up or the time budget is spent.""" + """Reconcile destination counters, then fold batches within the time budget.""" global _watermark, _watermark_derived cfg = cfg or UsageFoldConfig() result = UsageFoldResult(watermark=_watermark) @@ -857,25 +891,31 @@ def _done(outcome: dict[str, Any]) -> UsageFoldResult: log.warning("usage fold: acted pass failed (non-fatal): %s", e) pass_context.note_failure(e, "acted") - # Cheap no-op probe: skip the GROUP BY entirely when nothing new - # has landed since the last pass. + # Only an empty totals table can skip destination reconciliation: a + # stale version may have replaced folded counts without another event. probe = await ch.query_one("SELECT max(bumped_at) AS hwm FROM loom.memory_usage_totals") hwm = _utc((probe or {}).get("hwm")) if hwm <= UNIX_EPOCH: + pass_context.set_cursor(_RESET_CURSOR) return _done({"candidates": 0, "folded": 0, "skipped_no_memory": 0}) - if not _watermark_derived and _watermark <= UNIX_EPOCH: - _watermark = await _derive_cold_start_watermark(ch, hwm) - _watermark_derived = True - log.info("usage fold: cold-start watermark derived as %s", _watermark.isoformat()) - - if hwm <= _watermark: - return _done({"candidates": 0, "folded": 0, "skipped_no_memory": 0}) + resume = await _derive_watermark(ch, hwm) + _watermark = min(_watermark, resume) if _watermark_derived else resume + _watermark_derived = True + after = _resume_after(_watermark) + if after[0] > UNIX_EPOCH: + _watermark = min(_watermark, after[0] - timedelta(seconds=1)) + pass_context.set_cursor(json.dumps((after[0].isoformat(), *after[1:]))) while True: - more = await _fold_batch(ch, cfg, result) - if not more: + pass_context.check_stop() + cursor = await _fold_batch(ch, cfg, result, after) + if cursor is None: + if not result.skipped_pending_mutations: + pass_context.set_cursor(_RESET_CURSOR) break + after = cursor + pass_context.set_cursor(json.dumps((after[0].isoformat(), *after[1:]))) if time.monotonic() - started >= float(cfg.pass_time_budget_seconds): log.info("usage fold: time budget spent, resuming next pass") break diff --git a/tests/analyzer/cases/workers.py b/tests/analyzer/cases/workers.py index 20baad5ba3..075661a0b2 100644 --- a/tests/analyzer/cases/workers.py +++ b/tests/analyzer/cases/workers.py @@ -149,6 +149,8 @@ async def _worker_passes(ch: RecordingCH, c: dict[str, Any]) -> None: last_activity=now, ) await uf._fold_memory_rows(ch, [target], 10) + await uf._derive_watermark(ch, now) + await uf._fold_batch(ch, uf.UsageFoldConfig(), uf.UsageFoldResult(), (now, ORG, NS, "m1")) async def _admin_readers(ch: RecordingCH, c: dict[str, Any]) -> None: diff --git a/tests/analyzer/shapes.txt b/tests/analyzer/shapes.txt index 15e53e2e17..be55ad6d2e 100644 --- a/tests/analyzer/shapes.txt +++ b/tests/analyzer/shapes.txt @@ -678,6 +678,7 @@ 516db52eddcc SELECT count() AS n FROM loom.session_brief_revisions WHERE org = {org:String} AND namespace = {ns:String} AND ? 51af520b7286 SELECT DISTINCT tier, qualified_name, symbol_id FROM loom.code_symbol_names WHERE org = {org:String} AND namespace = {ns:String} AND name IN {names:Array(String)} AND (qualified_name = {full:String} O 51f662fe04fe SELECT count() AS n FROM loom.learnings_queue WHERE org = {org:String} AND candidate_id IN {cids:Array(UUID)} AND status = ? +52083af6b301 SELECT count() AS n FROM system.mutations WHERE database = {db:String} AND table = {t:String} AND NOT is_done 52448e9cfd6b ALTER TABLE loom.memory_record_conflicts UPDATE tombstoned_at = {now:DateTime} WHERE (org = {org:String} AND tombstoned_at IS NULL AND namespace = {ns:String}) AND ({erasure_mutation_tag:String} = {er 524493debfe9 SELECT count() AS n FROM loom.datasets WHERE org = {org:String} AND namespace IN {ns:Array(String)} 525d9e3b3774 SELECT count() AS n FROM loom.working_memory FINAL WHERE org = {org:String} AND user_id = {uid:String} AND tombstoned_at IS NULL @@ -1439,6 +1440,7 @@ a613795f2fe3 SELECT count() AS n FROM loom.operator_edges FINAL WHERE org = {org a62720ec9473 SELECT org, namespace, agent, user_id, session_id, queue_id, payload, message_count, content_chars, status, attempts, lease_until, last_error, written, enqueued_at, updated_at, version, enqueued_by FR a627d0337788 ALTER TABLE loom.memory_ask_precedents DELETE WHERE (org = {org:String} AND namespace IN {ns:Array(String)}) AND ({erasure_mutation_tag:String} = {erasure_mutation_tag:String}) SETTINGS mutations_sync a63dc159ea03 SELECT count() AS n, argMax(role, ts) AS role, argMax(content, ts) AS content, argMax(run_id, ts) AS run_id, max(ts) AS last_ts, max(saved_at) AS last_saved_at, groupUniqArray(user_id) AS owners, coun +a64937812d3d SELECT org, namespace, toString(memory_id) AS memory_id, sum(recall_count) AS recall_total, sum(evidence_count) AS evidence_total, sum(used_count) AS used_total, sum(harm_count) AS harm_total, max(las a661d7b26199 ALTER TABLE loom.memory_extraction_queue DELETE WHERE (org = {org:String} AND user_id = {uid:String}) AND ({erasure_mutation_tag:String} = {erasure_mutation_tag:String}) SETTINGS mutations_sync = ? a662cd95ca59 SELECT memory_key, (status != ? OR tombstoned_at IS NOT NULL OR redacted_at IS NOT NULL) AS taken_out FROM loom.memory FINAL WHERE org = {org:String} AND namespace = {ns:String} AND agent = {agent:Str a67dc6a1b98a SELECT path, start_line, end_line, content_hash FROM loom.code_chunks FINAL WHERE org = {org:String} AND namespace = {ns:String} AND path IN {paths:Array(String)} AND tombstoned_at IS NULL AND start_l @@ -1747,6 +1749,7 @@ c853d7bbd170 SELECT org, agent, count() AS calls, countIf(if(consumer = ?, args[ c85b7e038508 SELECT namespace, kind FROM loom.namespaces FINAL WHERE org = {org:String} c873c5cd3338 SELECT retention_days, dedup_threshold, hot_quota, conflict_policy FROM loom.namespace_types FINAL WHERE org = {org:String} AND namespace = {ns:String} AND memory_type = {mt:String} LIMIT ? c87677674058 SELECT DISTINCT namespace FROM loom.search_misses WHERE org = {org:String} AND (has({spellings:Array(String)}, asker) OR has({spellings:Array(String)}, lowerUTF8(user_id))) +c8870c302e09 SELECT min(t.bumped) AS first_unfolded FROM ( SELECT org, namespace, memory_id, sum(recall_count) AS recall_total, sum(evidence_count) AS evidence_total, max(last_recalled_at) AS last_recalled, max(bu c892414a0cf3 SELECT role, granted_by, status, pending_role, pending_since FROM loom.namespace_grants FINAL WHERE org = {org:String} AND namespace = {ns:String} AND subject = {sub:String} AND revoked_at IS NULL AND c8ce735d214f SELECT count() AS n FROM loom.memory_embeddings FINAL WHERE org = {org:String} AND tombstoned_at IS NULL AND namespace IN {ns:Array(String)} c8e4a254b959 SELECT count() AS n FROM loom.conversation_threads FINAL WHERE org = {org:String} AND tombstoned_at IS NULL AND user_id = {uid:String} diff --git a/tests/unit/test_usage_fold_sql.py b/tests/unit/test_usage_fold_sql.py index 9729da7e43..e422b66939 100644 --- a/tests/unit/test_usage_fold_sql.py +++ b/tests/unit/test_usage_fold_sql.py @@ -137,7 +137,7 @@ async def test_a_cold_start_resumes_at_the_namespace_still_behind(tmp_path: Path f"('{MID}', 'o', 'b', 0, 3, 0, 0, '2026-01-01 00:00:00', '2026-01-02 00:00:00')" ) hwm = datetime(2026, 2, 1, tzinfo=UTC) - got = await usage_fold._derive_cold_start_watermark(kept, hwm) + got = await usage_fold._derive_watermark(kept, hwm) assert got == datetime(2026, 1, 1, 23, 59, 59, tzinfo=UTC) @@ -164,3 +164,233 @@ async def test_the_identity_read_returns_only_the_memories_of_the_batch(tmp_path ("ka", 0), ("kd", 2), ] + + +@pytest.fixture +def fold_db(tmp_path: Path, monkeypatch: pytest.MonkeyPatch): + async def _no_acted(_ch: Any, _result: Any) -> None: + pass + + usage_fold.reset_watermark() + monkeypatch.setattr(usage_fold, "acted_pass", _no_acted) + kept = KeptCH( + tmp_path, + ddl("memory"), + ddl("memory_usage_totals"), + ddl("memory_usage_events"), + ddl("memory_reinforcement"), + ) + yield kept + usage_fold.reset_watermark() + + +async def _usage( + kept: KeptCH, + *, + mid: str = MID, + recall: int = 0, + evidence: int = 0, + at: datetime = SEEN, + org: str = "o", + namespace: str = "a", +) -> None: + await kept.insert( + "loom.memory_usage_totals", + [ + { + "org": org, + "namespace": namespace, + "memory_id": mid, + "recall_count": recall, + "evidence_count": evidence, + "last_recalled_at": at if recall else datetime(1970, 1, 1), + "bumped_at": at, + } + ], + ) + + +async def test_late_same_second_totals_are_folded_on_the_next_pass(fold_db: KeptCH) -> None: + kept = fold_db + await _memory(kept, ("o", "a", "k", MID)) + await _usage(kept, recall=1, evidence=1) + await usage_fold.run_pass(ch=kept) + assert (await _counters(kept))[("o", "a", "k")][:2] == (1, 2) + + await _usage(kept, recall=2, evidence=3) # Same bumped_at, a later visible part. + await usage_fold.run_pass(ch=kept) + assert (await _counters(kept))[("o", "a", "k")] == (3, 5, SEEN.replace(tzinfo=None)) + await usage_fold.run_pass(ch=kept) # Absolute targets never double-count. + assert (await _counters(kept))[("o", "a", "k")][:2] == (3, 5) + + +@pytest.mark.parametrize("preserve_counts", [False, True], ids=["all-counters", "last-recall-only"]) +async def test_a_stale_winning_version_is_repaired_without_new_usage( + fold_db: KeptCH, preserve_counts: bool +) -> None: + from datetime import timedelta + + kept = fold_db + await _memory(kept, ("o", "a", "k", MID), ("o", "a", "later", MID2)) + [stale] = await kept.query_all("SELECT * FROM loom.memory FINAL WHERE memory_key = 'k'") + await _usage(kept, recall=3, evidence=2) + await _usage(kept, mid=MID2, recall=1, at=SEEN + timedelta(hours=1)) + await usage_fold.run_pass(ch=kept) + assert (await _counters(kept))[("o", "a", "k")][:2] == (3, 3) + + stale["version"] += 1 + if preserve_counts: + stale.update(recall_count=3, evidence_count=3) + await kept.insert("loom.memory", [stale]) + expected_counts = (3, 3) if preserve_counts else (0, 1) + assert (await _counters(kept))[("o", "a", "k")] == (*expected_counts, None) + await usage_fold.run_pass(ch=kept) + # A key predicate can move into PREWHERE and hide unapplied patch columns. + [actual] = [ + row + for row in await kept.query_all("SELECT * FROM loom.memory FINAL") + if row["memory_key"] == "k" + ] + assert actual == { + **stale, + "recall_count": 3, + "evidence_count": 3, + "last_recalled_at": SEEN.replace(tzinfo=None), + } + + +@pytest.mark.parametrize("handover", [False, True], ids=["one-pass", "handover-between-batches"]) +async def test_timestamp_ties_larger_than_the_cap_are_not_skipped( + fold_db: KeptCH, handover: bool +) -> None: + from server.workers import pass_context + + kept = fold_db + mids = [f"6b47c9b7-0000-0000-0000-{i:012d}" for i in range(1, 6)] + await _memory(kept, *(("o", "a", mid, mid) for mid in mids)) + for mid in mids: + await _usage(kept, mid=mid, recall=1) + cfg = usage_fold.UsageFoldConfig( + max_memories_per_pass=2, pass_time_budget_seconds=0 if handover else 120 + ) + folded = batches = 0 + cursor = "" + for _ in range(3 if handover else 1): + usage_fold.reset_watermark() + ctx = pass_context.begin("usage_fold", resume_cursor=cursor) + try: + result = await usage_fold.run_pass(ch=kept, cfg=cfg) + folded += result.folded + batches += result.batches + cursor = ctx.cursor + finally: + pass_context.clear() + assert folded == 5 + assert batches == 3 + assert cursor # An explicit reset survives the supervisor's nonempty-cursor filter. + assert set((await _counters(kept)).values()) == {(1, 1, SEEN.replace(tzinfo=None))} + + +async def _supervised_fold(kept: KeptCH, cfg: Any = None) -> Any: + import time + + from server.workers import pass_context, status_store, supervisor + + saved = await status_store.resume_cursor("usage_fold", ch=kept) + state = supervisor.LoopState(cursor=saved) + ctx = pass_context.begin("usage_fold", resume_cursor=saved) + started = time.monotonic() + try: + result = await usage_fold.run_pass(ch=kept, cfg=cfg) + supervisor._record_status_row( + "usage_fold", started=started, success=True, partial=ctx, state=state + ) + await status_store.flush() + return result + finally: + pass_context.clear() + + +async def test_completed_scan_persists_its_reset_before_same_second_repairs( + fold_db: KeptCH, +) -> None: + from server.workers import status_store + + kept = fold_db + kept.create(ddl("worker_status")) + mids = [f"6b47c9b7-0000-0000-0000-{i:012d}" for i in range(1, 4)] + await _memory(kept, *(("o", "a", mid, mid) for mid in mids)) + for mid in mids: + await _usage(kept, mid=mid, recall=1) + status_store.reset_for_tests(client=kept) + try: + first = await _supervised_fold( + kept, usage_fold.UsageFoldConfig(max_memories_per_pass=2, pass_time_budget_seconds=0) + ) + assert first.folded == 2 + assert (await _supervised_fold(kept)).folded == 1 + await _usage(kept, mid=mids[0], recall=1) + usage_fold.reset_watermark() # A restarted supervisor reads the stored reset. + await _supervised_fold(kept) + assert (await _counters(kept))[("o", "a", mids[0])][:2] == (2, 1) + finally: + status_store.reset_for_tests() + + +@pytest.mark.parametrize( + "org,namespace", [("o" * 500, "a"), ("o", "n" * 500)], ids=["long-org", "long-namespace"] +) +async def test_long_identity_ties_resume_through_the_bounded_status_cursor( + fold_db: KeptCH, org: str, namespace: str +) -> None: + from server.workers import status_store + + kept = fold_db + kept.create(ddl("worker_status")) + mids = [f"6b47c9b7-0000-0000-0000-{i:012d}" for i in range(1, 6)] + await _memory(kept, *((org, namespace, mid, mid) for mid in mids)) + for mid in mids: + await _usage(kept, mid=mid, recall=1, org=org, namespace=namespace) + cfg = usage_fold.UsageFoldConfig(max_memories_per_pass=2, pass_time_budget_seconds=0) + status_store.reset_for_tests(client=kept) + try: + for _ in range(3): + usage_fold.reset_watermark() + await _supervised_fold(kept, cfg) + assert set((await _counters(kept)).values()) == {(1, 1, SEEN.replace(tzinfo=None))} + finally: + status_store.reset_for_tests() + + +@pytest.mark.parametrize("counter", ["recall", "evidence"]) +async def test_saturated_counters_do_not_keep_rewinding_a_time_limited_scan( + fold_db: KeptCH, counter: str +) -> None: + from datetime import timedelta + + from server.workers import status_store + + kept = fold_db + kept.create(ddl("worker_status")) + mids = [f"6b47c9b7-0000-0000-0000-{i:012d}" for i in range(1, 4)] + await _memory(kept, *(("o", "a", mid, mid) for mid in mids)) + for i, mid in enumerate(mids): + await _usage( + kept, + mid=mid, + recall=2**32 if i == 0 and counter == "recall" else 1, + evidence=2**32 if i == 0 and counter == "evidence" else 0, + at=SEEN + timedelta(hours=i), + ) + cfg = usage_fold.UsageFoldConfig(max_memories_per_pass=1, pass_time_budget_seconds=0) + status_store.reset_for_tests(client=kept) + try: + for _ in range(3): + usage_fold.reset_watermark() + await _supervised_fold(kept, cfg) + counters = await _counters(kept) + expected = (2**32 - 1, 1) if counter == "recall" else (1, 2**32 - 1) + assert counters[("o", "a", mids[0])][:2] == expected + assert counters[("o", "a", mids[-1])][:2] == (1, 1) + finally: + status_store.reset_for_tests() diff --git a/tests/unit/test_usage_fold_worker.py b/tests/unit/test_usage_fold_worker.py index 65925cc563..9c8465dd7f 100644 --- a/tests/unit/test_usage_fold_worker.py +++ b/tests/unit/test_usage_fold_worker.py @@ -1,7 +1,7 @@ """Usage fold-back worker — runs against a stub CH client. The stub answers queries in order: probe (query_one) → cold-start -watermark derivation (query_one, first pass only) → per batch: the +watermark reconciliation (query_one, every pass) → per batch: the pending-mutations probe (query_one, unseeded reads as 0), totals (query_all), then one identity read (query_all) per size chunk of its keys. A batch below the cap ends the pass; a full batch is followed by another @@ -196,19 +196,19 @@ async def test_noop_when_no_usage_at_all() -> None: assert ch.execute_calls == [] -async def test_noop_when_watermark_already_at_high_water() -> None: - """Second pass with no new events stops at the probe (no re-derivation).""" +async def test_unchanged_high_water_still_checks_for_regressed_counters() -> None: + """An unchanged timestamp cannot rule out a stale full-version insert.""" ch = StubCH() _seed_active_pass(ch, totals=[_totals_row()], idents=[_ident_row()]) await run_pass(ch=ch) - assert usage_fold.current_watermark() == T0.replace(tzinfo=UTC) + assert usage_fold.current_watermark() == (T0 - timedelta(seconds=1)).replace(tzinfo=UTC) ch2 = StubCH() - ch2.query_one_returns.append({"hwm": T0}) + ch2.query_one_returns += [{"hwm": T0}, {"first_unfolded": None}] result = await run_pass(ch=ch2) assert result.folded == 0 - assert len(ch2.query_one_calls) == 1 - assert ch2.query_all_calls == [] + assert len(ch2.query_one_calls) == 3 + assert len(ch2.query_all_calls) == 1 assert ch2.execute_calls == [] @@ -536,7 +536,7 @@ async def test_totals_for_erased_memory_are_skipped() -> None: assert result.folded == 0 assert result.skipped_no_memory == 1 assert ch.execute_calls == [] - assert usage_fold.current_watermark() == T0.replace(tzinfo=UTC) + assert usage_fold.current_watermark() == (T0 - timedelta(seconds=1)).replace(tzinfo=UTC) async def test_cold_start_watermark_is_derived_not_epoch() -> None: @@ -554,21 +554,21 @@ async def test_cold_start_watermark_is_derived_not_epoch() -> None: await run_pass(ch=ch) derive_sql, _ = ch.query_one_calls[1] assert "min(t.bumped) AS first_unfolded" in derive_sql - assert "t.recall_total > m.recall_count" in derive_sql - assert "(1 + t.evidence_total) > m.evidence_count" in derive_sql + assert "least(t.recall_total, 4294967295) > m.recall_count" in derive_sql + assert "(1 + least(t.evidence_total, 4294967294)) > m.evidence_count" in derive_sql # The totals query was bounded by the derived watermark (first_unfolded − 1 s). _, params = ch.query_all_calls[0] assert params["wm"] == (first_unfolded - timedelta(seconds=1)).replace(tzinfo=UTC) -async def test_cold_start_with_nothing_unfolded_jumps_to_high_water() -> None: +async def test_cold_start_with_nothing_unfolded_retains_high_water_second() -> None: ch = StubCH() ch.query_one_returns.append({"hwm": T0}) ch.query_one_returns.append({"first_unfolded": None}) result = await run_pass(ch=ch) assert result.candidates == 0 - assert ch.query_all_calls == [] - assert usage_fold.current_watermark() == T0.replace(tzinfo=UTC) + assert len(ch.query_all_calls) == 1 + assert usage_fold.current_watermark() == (T0 - timedelta(seconds=1)).replace(tzinfo=UTC) async def test_watermark_advances_to_max_bumped_of_batch() -> None: @@ -585,16 +585,15 @@ async def test_watermark_advances_to_max_bumped_of_batch() -> None: ) result = await run_pass(ch=ch) assert result.folded == 2 - assert usage_fold.current_watermark() == later.replace(tzinfo=UTC) + assert usage_fold.current_watermark() == (later - timedelta(seconds=1)).replace(tzinfo=UTC) # Filter AFTER aggregation: a row-level predicate undercounts unmerged parts. totals_sql = ch.query_all_calls[0][0] assert "HAVING bumped > {wm:DateTime}" in totals_sql assert "GROUP BY org, namespace, memory_id" in totals_sql -async def test_full_batch_steps_watermark_back_and_keeps_going() -> None: - """At the batch cap the pass takes another batch (stepping the watermark - back 1 s to re-fold tied boundary rows) until a short batch drains it.""" +async def test_full_batch_keeps_the_boundary_and_pages_after_its_last_identity() -> None: + """A full batch resumes past its last identity while retaining the boundary.""" later = T0 + timedelta(minutes=5) ch = StubCH() _seed_active_pass( @@ -606,16 +605,18 @@ async def test_full_batch_steps_watermark_back_and_keeps_going() -> None: idents=[_ident_row(), _ident_row(memory_id=MID2, memory_key="k2")], hwm=later, ) - # Second batch: the boundary row again (tie re-fold), short, so the pass ends. - ch.query_all_returns.append([_totals_row(memory_id=MID2, bumped=later)]) - ch.query_all_returns.append([_ident_row(memory_id=MID2, memory_key="k2")]) + # The next batch has no rows past the cursor. + ch.query_all_returns.append([]) result = await run_pass(cfg=UsageFoldConfig(max_memories_per_pass=2), ch=ch) - assert result.batches == 2 - assert result.candidates == 3 - assert len(ch.query_all_calls) == 4 - # Batch 2 started from later − 1 s, then the short batch pinned the watermark. + assert result.batches == 1 + assert result.candidates == 2 + assert len(ch.query_all_calls) == 3 + assert ch.query_all_calls[2][1]["after"] == ( + later.replace(tzinfo=UTC), + *usage_fold._cursor_identity(*_key(MID2)), + ) assert ch.query_all_calls[2][1]["wm"] == (later - timedelta(seconds=1)).replace(tzinfo=UTC) - assert usage_fold.current_watermark() == later.replace(tzinfo=UTC) + assert usage_fold.current_watermark() == (later - timedelta(seconds=1)).replace(tzinfo=UTC) async def test_time_budget_ends_a_pass_between_batches() -> None: @@ -862,3 +863,150 @@ async def test_the_identity_read_is_cut_by_size_too() -> None: for p in reads: for value in p.values(): assert len(format_bind_value(value, top_level=False).encode()) <= ch_params.FIELD_BYTES + + +@pytest.mark.parametrize("recall,evidence", [(2, 0), (0, 2)]) +async def test_late_usage_in_the_boundary_second_is_folded(recall: int, evidence: int) -> None: + """A second totals part becomes visible after a short batch, without a new bump.""" + ch = StubCH() + _seed_active_pass(ch, totals=[_totals_row(recall=1)], idents=[_ident_row()]) + await run_pass(ch=ch) + + late = StubCH() + _seed_active_pass( + late, + totals=[_totals_row(recall=recall, evidence=evidence)], + idents=[_ident_row()], + ) + result = await run_pass(ch=late) + assert result.folded == 1 + [(_, params)] = _updates(late, "loom.memory") + assert params["rcls"] == [recall] + assert params["evs"] == [1 + evidence] + assert late.query_all_calls[0][1]["wm"] < T0.replace(tzinfo=UTC) + assert usage_fold.current_watermark() == (T0 - timedelta(seconds=1)).replace(tzinfo=UTC) + + +async def test_a_later_stale_version_is_repaired_without_another_usage_event() -> None: + """Another replica inserts counters it read before the fold, behind the cursor.""" + later = T0 + timedelta(hours=1) + ch = StubCH() + _seed_active_pass( + ch, + totals=[_totals_row(), _totals_row(memory_id=MID2, bumped=later)], + idents=[_ident_row(), _ident_row(memory_id=MID2, memory_key="k2")], + hwm=later, + ) + await run_pass(ch=ch) + + # The earlier memory's winning version lost its folded counts. The totals + # and their high water mark are unchanged; destination comparison finds it. + repair = StubCH() + _seed_active_pass(repair, totals=[_totals_row()], idents=[_ident_row()], hwm=later) + result = await run_pass(ch=repair) + assert result.folded == 1 + assert repair.query_all_calls[0][1]["wm"] == (T0 - timedelta(seconds=1)).replace(tzinfo=UTC) + assert _updates(repair, "loom.memory")[0][1]["rcls"] == [3] + + +async def test_more_than_a_batch_of_tied_totals_all_fold() -> None: + """Ties paginate by identity, so the boundary overlap cannot starve a memory.""" + from tests.unit.ch_fake import RecordingCH + + mids = [f"6b47c9b7-0000-0000-0000-{i:012d}" for i in range(1, 6)] + + def _page(_sql: str, params: dict[str, Any]) -> list[dict[str, Any]]: + candidates = [ + _totals_row(memory_id=mid) + for mid in mids + if T0.replace(tzinfo=UTC) > params["wm"] + and (T0.replace(tzinfo=UTC), *usage_fold._cursor_identity("clickhouse", "ns", mid)) + > params.get("after", (datetime.min.replace(tzinfo=UTC), "", "", "")) + ] + return candidates[: params["lim"]] + + ch = ( + RecordingCH() + .on("max(bumped_at) AS hwm", [{"hwm": T0}]) + .on("min(t.bumped) AS first_unfolded", [{"first_unfolded": T0}]) + .on("system.mutations", [{"n": 0}]) + .on("HAVING bumped >", _page) + .on( + "toString(m.memory_id) AS mid", + lambda _sql, p: [_ident_row(memory_id=mid, memory_key=mid) for mid in p["ids"]], + ) + ) + result = await run_pass(cfg=UsageFoldConfig(max_memories_per_pass=2), ch=ch) + assert result.folded == len(mids) + assert result.batches == 3 + assert {str(row["memory_key"]) for row in ch.inserted(REINF)} == set(mids) + + +async def test_a_time_limited_tie_scan_resumes_after_handover() -> None: + from server.core.datetimes import UNIX_EPOCH + from server.workers import pass_context + + cfg = UsageFoldConfig(max_memories_per_pass=1, pass_time_budget_seconds=0) + ch = StubCH() + _seed_active_pass(ch, totals=[_totals_row()], idents=[_ident_row()]) + ctx = pass_context.begin("usage_fold") + try: + await run_pass(cfg=cfg, ch=ch) + cursor = ctx.cursor + assert cursor + finally: + pass_context.clear() + + usage_fold.reset_watermark() # Another replica resumes the stored cursor. + next_ch = StubCH() + _seed_active_pass( + next_ch, + totals=[_totals_row(memory_id=MID2)], + idents=[_ident_row(memory_id=MID2)], + ) + ctx = pass_context.begin("usage_fold", resume_cursor=cursor) + try: + await run_pass(ch=next_ch) + assert next_ch.query_all_calls[0][1]["after"] == ( + T0.replace(tzinfo=UTC), + *usage_fold._cursor_identity(*_key(MID)), + ) + assert ctx.cursor # The supervisor persists a nonempty reset for a drained scan. + ctx.resume_cursor = ctx.cursor + assert usage_fold._resume_after(T0.replace(tzinfo=UTC)) == (UNIX_EPOCH, "", "", "") + finally: + pass_context.clear() + + +@pytest.mark.parametrize( + "saved", ["broken", "null", "{}", "[1, 2, 3, 4]", '["bad", "o", "n", "m"]'] +) +def test_invalid_resume_cursor_restarts_from_reconciliation(saved: str) -> None: + from server.core.datetimes import UNIX_EPOCH + from server.workers import pass_context + + pass_context.begin("usage_fold", resume_cursor=saved) + try: + assert usage_fold._resume_after(T0.replace(tzinfo=UTC)) == (UNIX_EPOCH, "", "", "") + finally: + pass_context.clear() + + +async def test_reconciliation_rewinds_a_saved_cursor_for_an_older_stale_head() -> None: + import json + + from server.core.datetimes import UNIX_EPOCH + from server.workers import pass_context + + later = T0 + timedelta(hours=1) + cursor = json.dumps((later.isoformat(), *usage_fold._cursor_identity(*_key(MID2)))) + ch = StubCH() + _seed_active_pass(ch, totals=[_totals_row()], idents=[_ident_row()], hwm=later) + ctx = pass_context.begin("usage_fold", resume_cursor=cursor) + try: + result = await run_pass(ch=ch) + assert result.folded == 1 + assert ch.query_all_calls[0][1]["after"] == (UNIX_EPOCH, "", "", "") + assert ctx.cursor + finally: + pass_context.clear()