Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
29 commits
Select commit Hold shift + click to select a range
48fc370
Vector reads reserve weighted capacity through driver retirement
zlareb1 Nov 18, 2026
318efe8
Supplemental reads join siblings before returning failures
zlareb1 Nov 18, 2026
86e6fd4
Attribute composed vector reads and retire their children
zlareb1 Nov 18, 2026
69e7c3e
Pin vector admission defaults in the settings snapshot
zlareb1 Nov 18, 2026
c55c526
Account for resolver scans and classify vector dispatches
zlareb1 Nov 18, 2026
f3765ef
Keep internal vector refusals outside the wire error catalog
zlareb1 Nov 18, 2026
17f59cc
Refuse supplemental vector reads that would queue
zlareb1 Nov 18, 2026
e81a519
Propagate vector capacity refusals through evidence readers
zlareb1 Nov 18, 2026
0b5cdd0
Propagate vector refusals through outer read owners
zlareb1 Nov 18, 2026
66f468c
Keep capacity refusals through bisect and tracker owners
zlareb1 Nov 18, 2026
a1beb61
Reconnect metadata stays scalar and bisect retains discarded reads
zlareb1 Nov 18, 2026
6cf711b
Cancelled waiters retain replacement org accounting
zlareb1 Nov 18, 2026
f748c53
Classify reconnect metadata as scalar work
zlareb1 Nov 18, 2026
36b87bf
Keep vector admission on ranked mirror discovery
zlareb1 Nov 18, 2026
f26273b
Keep key-component refreshes outside vector admission
zlareb1 Nov 18, 2026
783468e
Compare ordered retrieval across mixed builds
zlareb1 Nov 18, 2026
5d052ae
Exercise native vector budgets and paired API timings
zlareb1 Nov 18, 2026
c057774
Retain refusal request IDs and verify native process restarts
zlareb1 Nov 18, 2026
3955249
Correlate mixed-build request timings with native work
zlareb1 Nov 19, 2026
025c36d
Throw when vector reads exhaust native execution time
zlareb1 Nov 19, 2026
0448043
Vector reads reject incomplete native results
zlareb1 Nov 19, 2026
0af6ffe
Record request stages in mixed-build timing checks
zlareb1 Nov 19, 2026
927dec2
Distinct-limited vector reads refuse partial identities
zlareb1 Nov 19, 2026
cae53a7
Mixed-version timing captures bounded native statement costs
zlareb1 Nov 19, 2026
f9f266e
Fixture timing records response publication stages
zlareb1 Nov 19, 2026
97e11ab
Prepare both latency cohorts before measuring either build
zlareb1 Nov 19, 2026
5783be6
The timing fixture records namespace key waits
zlareb1 Nov 19, 2026
c1df6ca
SCIM clock test issues the token before deactivation
zlareb1 Nov 19, 2026
bcb810c
Merge the shipped lesson test isolation repair
zlareb1 Nov 19, 2026
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -1110,7 +1110,7 @@ jobs:
# first parent of its merge commit, the target branch as the merge was made. The
# target branch's tip can carry a newer migration by the time the run starts.
- name: Mixed-version gate
run: python scripts/gates/mixed_version.py --require --base HEAD~1
run: python scripts/gates/mixed_version.py --require --base HEAD~1 --ranking-neutral
- name: Published build ordinary-event compatibility
run: python scripts/gates/mixed_version.py --require --base 1cbaf0411114512957b96ec5ea7bbb8058828543

Expand Down
117 changes: 117 additions & 0 deletions bench/retrieval_determinism.py
Original file line number Diff line number Diff line change
Expand Up @@ -28,11 +28,14 @@

import argparse
import json
import math
import os
import statistics
import sys
import time
import urllib.parse
import urllib.request
from collections.abc import Callable
from dataclasses import dataclass, field
from typing import Any

Expand Down Expand Up @@ -101,6 +104,120 @@ def to_json(self) -> dict[str, Any]:
# ----------------------------------------------------------------------------


RANKING_OPS = ("memory.search", "memory.search_trace", "memory.ask")
_RANKING_SCORES = ("distance", "similarity", "rank_score")
_RANKING_IDENTITY = (
"memory_key",
"memory_type",
"source",
"namespace",
"agent",
"user_id",
"session_id",
"memory_id",
"version",
"parent_memory_key",
)
RankingSnapshot = dict[str, dict[str, list[dict[str, Any]]]]


def _ranking_score(value: object) -> float:
assert isinstance(value, int | float) and not isinstance(value, bool), "missing numeric score"
assert math.isfinite(value), "nonfinite score"
return float(value)


def ranked_rows(payload: object, *, namespace: str, positive_key: str) -> list[dict[str, Any]]:
"""Keep every returned identity and score, without sorting or truncating rows."""
assert isinstance(payload, dict), "ranking response is not an object"
rows = payload.get("results")
assert isinstance(rows, list) and rows, "missing or empty ranking results"
out = []
for row in rows:
assert isinstance(row, dict), "ranking result is not an object"
for name in ("memory_key", "memory_type", "source"):
assert isinstance(row.get(name), str) and row[name], f"missing {name}"
assert row.get("namespace", namespace) in ("", namespace), "wrong result namespace"
identity = {name: row[name] for name in _RANKING_IDENTITY if name in row}
out.append(
{
"identity": identity,
"scores": {name: _ranking_score(row.get(name)) for name in _RANKING_SCORES},
}
)
assert out[0]["identity"]["memory_key"] == positive_key, "positive anchor is not first"
return out


def capture_rankings(
call: Callable[[str, dict[str, Any]], dict[str, Any]],
requests: dict[str, dict[str, dict[str, Any]]],
positives: dict[str, str],
*,
elapsed: dict[str, dict[str, float]] | None = None,
) -> RankingSnapshot:
"""Capture all requested public reads; request errors propagate as failures."""
assert positives and set(requests) == set(RANKING_OPS), "missing ranking operations"
snapshot: RankingSnapshot = {}
for op in RANKING_OPS:
assert set(requests[op]) == set(positives), "missing ranking queries"
snapshot[op] = {}
for query, positive_key in positives.items():
body = requests[op][query]
assert body.get("query") == query, "query does not match its request"
namespace = body.get("namespace")
assert isinstance(namespace, str) and namespace and positive_key
started = time.perf_counter()
payload = call(op, body)
duration = time.perf_counter() - started
if elapsed is not None:
elapsed.setdefault(op, {})[query] = duration
if op == "memory.ask":
assert payload.get("source") == "retrieval_only", "ask did not return retrieval"
assert payload.get("synth_calls") == 0, "ask made a synthesis call"
snapshot[op][query] = ranked_rows(
payload, namespace=namespace, positive_key=positive_key
)
return snapshot


def assert_rankings_equal(
before: RankingSnapshot, after: RankingSnapshot, *, queries: list[str]
) -> None:
"""Compare each API independently with fixed float tolerances and exact order."""
assert queries and len(set(queries)) == len(queries), "missing or repeated queries"
assert set(before) == set(after) == set(RANKING_OPS), "missing ranking operations"
for op in RANKING_OPS:
assert set(before[op]) == set(after[op]) == set(queries), "missing ranking queries"
for query in queries:
left, right = before[op][query], after[op][query]
assert left and len(left) == len(right), f"{op}: changed result count"
for a, b in zip(left, right, strict=True):
assert a["identity"] == b["identity"], f"{op}: changed identity or order"
assert set(a["scores"]) == set(b["scores"]) == set(_RANKING_SCORES)
for name in _RANKING_SCORES:
x, y = _ranking_score(a["scores"][name]), _ranking_score(b["scores"][name])
assert math.isclose(x, y, rel_tol=1e-7, abs_tol=1e-6), (
f"{op}: changed {name}: {x!r} != {y!r}"
)


def ranking_timings(elapsed: dict[str, dict[str, float]]) -> dict[str, dict[str, Any]]:
"""Small-sample timings; nearest-rank p99 is the maximum of three queries."""
assert set(elapsed) == set(RANKING_OPS), "missing timing operations"
out = {}
for op, calls in elapsed.items():
values = sorted(_ranking_score(value) for value in calls.values())
assert values and values[0] >= 0, "missing or negative elapsed time"
out[op] = {
"n": len(values),
"p50_s": statistics.median(values),
"p99_s": values[math.ceil(0.99 * len(values)) - 1],
"calls_s": calls,
}
return out


def top_keys(search_output: dict[str, Any], k: int) -> list[str]:
"""Top-k memory keys from a memory.search response, in returned order."""
hits = search_output.get("results") or []
Expand Down
75 changes: 75 additions & 0 deletions bench/scale_load.py
Original file line number Diff line number Diff line change
Expand Up @@ -415,6 +415,81 @@ def _summary(lat: list[float], errors: dict[str, int]) -> dict[str, Any]:
}


def latency_observations(samples: list[dict[str, Any]]) -> dict[str, Any]:
"""Keep every attempt beside the successful-call percentiles (small n is explicit)."""
assert samples, "no attempted calls"
for sample in samples:
ms = sample["elapsed_ms"]
assert isinstance(ms, (int, float)) and math.isfinite(ms) and ms >= 0
errors = Counter(str(s["error"]) for s in samples if s.get("error"))
lat = [float(s["elapsed_ms"]) for s in samples if not s.get("error")]
return {**_summary(lat, dict(errors)), "attempted": len(samples), "samples": samples}


def committed_latency_rows(
op: str, result: dict[str, Any], entries: list[dict[str, Any]]
) -> dict[str, dict[str, Any]]:
"""A normal-fixture refusal/merge is evidence, never an invented committed row."""
if op == "memory.set":
assert len(entries) == 1
assert result.get("noop") is False and result.get("status") == "active"
assert not result.get("duplicate_of") and not result.get("merged_into_key")
rows = [result]
else:
assert op == "memory.bulk_set"
assert _outcomes(op, result) == ["written"] * len(entries)
for key in (
"merged",
"duplicates",
"conflicted",
"rejected_pinned",
"identity_duplicates",
"skipped_pinned",
"skipped_locked",
"skipped_blacklisted",
"retyped",
):
assert not result.get(key), f"fixture write: {key}"
rows = result.get("written")
assert isinstance(rows, list) and len(rows) == len(entries)
expected = {e["memory_key"]: e["value"] for e in entries}
assert len(expected) == len(entries)
assert [r.get("memory_key") for r in rows] == [e["memory_key"] for e in entries]
out = {}
for row in rows:
key = row["memory_key"]
assert row.get("conflict_verdict") == "clear", f"{key}: not checked clear"
assert isinstance(row.get("version"), int) and row["version"] > 0
assert row.get("memory_id"), f"{key}: no committed id"
out[key] = {
"value": expected[key],
"version": row["version"],
"memory_id": row["memory_id"],
}
return out


def assert_latency_current(row: dict[str, Any], expected: dict[str, Any]) -> None:
assert row.get("found") is True, "acknowledged row missing from point read"
for field in ("value", "version", "memory_id"):
assert row.get(field) == expected[field], f"point read differs: {field}"
assert row.get("status") == "active"


def assert_latency_visible(result: dict[str, Any], key: str, expected: dict[str, Any]) -> None:
rows = result.get("results")
assert isinstance(rows, list), "missing search results"
matches = [r for r in rows if r.get("memory_key") == key]
assert len(matches) == 1, "acknowledged row missing/duplicated on FIRST search"
assert matches[0].get("version") == expected["version"], "stale first search"
assert matches[0].get("status") == "active", "nonactive first search"


def assert_latency_worker(pid: int, parent_pid: int, ppid: int, pgid: int) -> None:
assert pid > 0 and parent_pid > 0 and pid != parent_pid
assert ppid == parent_pid and pgid == parent_pid, "health PID not an owned worker"


def sample_keys(ch: Any, shape: str, n: int) -> list[dict[str, Any]]:
"""`n` existing heads to read, spread over the load (largest tenant for single)."""
where = (
Expand Down
55 changes: 55 additions & 0 deletions docs/architecture/clickhouse.md
Original file line number Diff line number Diff line change
Expand Up @@ -1028,3 +1028,58 @@ rows. When any replica is older, every Loom session turns the cache off when it
- [Migrations](../development/migrations.md): writing and testing a migration.
- [Troubleshooting](../operations/troubleshooting.md): too many parts, slow inserts.
- [Performance tuning](../guides/performance-tuning.md): traffic classes, vector knobs.

## Vector read admission

Vector dispatches carry their target org, authorized namespaces and probe weight.
One query vector against one stored axis costs one unit; a value/key read or entity
boost costs two, and a six-probe conflict statement costs six. These units are
independent of connection slots. SQL, tenant predicates and ranking stay in their
owning readers.

Each process admits 16 weighted units per org and queues at most 256 more by
default. Its total population is at most twice the configured waiting limit,
including org waits, connection waits, execution, retries and local retirement.
Reservation precedes any org or waiter allocation. Atomic FIFO grants prevent
partial acquisition. Lower live limits refuse new reservations while existing
charges drain; an overweight queued statement refuses instead of blocking its
successors. Disabling the existing connection gate refuses new vector work.

Sub-query and set-expansion phrases share a completion-sensitive duplicate set.
Their vector statements take an immediate atomic grant or refuse the complete read
with `VECTOR_CAPACITY_REFUSED` and reason `org_queue`; they never join an org wait
queue or pass an older waiter. Owned siblings finish cancellation before the refusal
returns. Other vector owners retain bounded FIFO waits. The policy is inherited by
child tasks independently of statement attribution and resets when the phrase owner
exits. Admitted work still holds its charge through connection waits, retries and
local retirement.

With no competing work, two phrases over one namespace can require eight weighted
units; two namespaces can require sixteen. Actual overlap determines admission,
so a nine-namespace fact-only read can succeed at the default sixteen-unit ceiling.
It is not refused from input counts alone. Wide shapes, including thirty-three
namespaces and eight phrases, can refuse even at the default ceiling. A two-unit
ceiling can refuse overlapping weighted statements. These are explicit capacity
failures, not partial successful rankings. Request frequency and representative
refusal rates require deployment qualification.

The admitted and queued unit bounds do not bound eager task objects created by
upstream producers. Supplemental phrases retain eager scheduling and have no
four-task producer window. There is no established composed maximum fanout or
process-memory bound from these admission limits alone.

Every vector attempt has finite time and memory budgets and a native throw ceiling
of one million read rows by default. Versions, joins and several namespaces can
reach that ceiling before one million logical keys. Admission and native refusals
propagate through optional read legs rather than returning a different ranking.
A selected conflict check refuses before commit. Optional value linking after a
commit preserves those versions and records the refused linking in their existing
result guidance and degradation record.

The existing streaming slots and upstream tracker credential connections remain
separate from ordinary pooled connections. Their vector work shares the same
weighted population; permission reads consuming a stream do not inherit its vector
attribution. Local cleanup does not prove remote cancellation or a fleet-wide bound.
Native retry amplification, maximum supported shape completion and competing-org
progress require deployment qualification. Placement evidence alone does not
report this read capacity, and the full capacity report remains unfinished.
3 changes: 3 additions & 0 deletions docs/reference/configuration.md
Original file line number Diff line number Diff line change
Expand Up @@ -160,6 +160,9 @@ ClickHouse connection, query shaping and vector storage.
| `LOOM_CH_LOG_COMMENT_STATEMENT_DIGEST` | `bool` | `False` | | Append `sql:<shape12> <first 200 chars>` of each statement to its `log_comment` ("<op> <request_id>"), so query_log rows group by shape. A statement that carries a secret, or one run as the analyst, gets its shape alone. Off by default: readers that split log_comment on spaces gain tokens. |
| `LOOM_CH_CAPTURE_STATEMENTS` | `str` | `''` | | A file path: append one JSON line (`digest`, `sql`) per executed statement, to collect the shapes an acceptance run executes for the analyzer corpus. '' = off; never set it in production. |
| `LOOM_CH_POOL_MAXSIZE` | `int` | `64` | | HTTP connection pool size: request concurrency plus 3 x the batched-insert bound (inserts a second x wait / 1000; docs/architecture/clickhouse.md#batched-inserts). |
| `LOOM_VECTOR_SCAN_MAX_CONCURRENT_PER_ORG` | `int` | `16` | live | Weighted vector work admitted per org in this process; two axes need two units. |
| `LOOM_VECTOR_SCAN_MAX_WAITERS_PER_ORG` | `int` | `256` | live | Weighted org wait units; twice this also bounds the whole process population. |
| `LOOM_VECTOR_SCAN_MAX_ROWS_TO_READ` | `int` | `1000000` | | Native throw ceiling per vector statement, including joins and row versions. |
| `LOOM_CH_KEEPALIVE_TIMEOUT` | `float` | `5.0` | | Seconds an idle pooled connection is kept. Must stay BELOW the Cloud load balancer's idle cutoff (8 s) so loom closes first: a peer-closed pooled TLS connection is never reaped under uvloop (`LOOM_UVICORN_LOOP=uvloop`). |
| `LOOM_CH_READ_YOUR_WRITES` | `bool` | `True` | | A point lookup that MISSES right after a write (get after set, verify, update_by_id, delete) re-reads once with select_sequential_consistency=1. Costs only on a miss; a no-op on single-node MergeTree. |
| `LOOM_CH_READ_YOUR_WRITES_WINDOW_S` | `float` | `20.0` | | memory.search on a namespace this process wrote within this many seconds reads with select_sequential_consistency from the start. 0 = off (the empty-pool retry remains the backstop). |
Expand Down
4 changes: 4 additions & 0 deletions docs/reference/errors.md
Original file line number Diff line number Diff line change
Expand Up @@ -964,6 +964,10 @@ Raised in `server/memory/app/apply.py`, `server/memory/app/lock.py`, `server/mem

HTTP 422, raised as `server.core.errors.ValidationError`. Category `input`, retryable: no. What to do: [common errors](#common-errors-and-what-to-do).

#### `VECTOR_CAPACITY_REFUSED`

HTTP 429, raised as `server.core.errors.VectorAdmissionError`. 429 — complete vector work exceeds a weighted queue or native budget. Category `capacity`, retryable: yes.

#### `VECTOR_INPUT_TOO_LARGE`

HTTP 422, raised as `server.core.errors.VectorInputTooLargeError`. 422 — an entity filter exceeds 64 raw names, or its canonical aliases or summary seeds exceed 256 distinct names. Category `input`, retryable: no.
Expand Down
11 changes: 11 additions & 0 deletions loom.example.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,17 @@ settings:
# [restart]
# ch_pool_maxsize: 64

# Weighted vector work admitted per org in this process; two axes need two units.
# [live]
# vector_scan_max_concurrent_per_org: 16

# Weighted org wait units; twice this also bounds the whole process population. [live]
# vector_scan_max_waiters_per_org: 256

# Native throw ceiling per vector statement, including joins and row versions.
# [restart]
# vector_scan_max_rows_to_read: 1000000

# Seconds an idle pooled connection is kept. Must stay BELOW the Cloud load balancer's
# idle cutoff (8 s) so loom closes first: a peer-closed pooled TLS connection is never
# reaped under uvloop (`LOOM_UVICORN_LOOP=uvloop`). [restart]
Expand Down
18 changes: 18 additions & 0 deletions schemas/loom-settings.schema.json
Original file line number Diff line number Diff line change
Expand Up @@ -7258,6 +7258,24 @@
"type": "integer",
"description": "Seconds between re-reads of which vector columns are coded and which coded tables carry patch parts. A write that embeds reads again on its own when the reading is older than a few seconds (codes_state.FRESH_READING_S)."
},
"vector_scan_max_concurrent_per_org": {
"default": 16,
"minimum": 2,
"type": "integer",
"description": "Weighted vector work admitted per org in this process; two axes need two units."
},
"vector_scan_max_rows_to_read": {
"default": 1000000,
"exclusiveMinimum": 0,
"type": "integer",
"description": "Native throw ceiling per vector statement, including joins and row versions."
},
"vector_scan_max_waiters_per_org": {
"default": 256,
"exclusiveMinimum": 0,
"type": "integer",
"description": "Weighted org wait units; twice this also bounds the whole process population."
},
"vector_search_quantized_codes": {
"default": true,
"type": "boolean",
Expand Down
Loading