From 48fc370c8518648b9f76dcdd512b0b3e49d7e5d4 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Wed, 18 Nov 2026 04:02:24 +0000 Subject: [PATCH 01/28] Vector reads reserve weighted capacity through driver retirement Attribute vector dispatches explicitly, bound org and process queues, and preserve capacity refusals across composed reads. Keep existing connection pools and apply finite native query limits. Signed-off-by: zlareb1 --- docs/architecture/clickhouse.md | 33 ++ docs/reference/configuration.md | 3 + docs/reference/errors.md | 4 + loom.example.yaml | 11 + schemas/loom-settings.schema.json | 18 ++ sdk/python/loom/__init__.py | 2 + sdk/python/loom/errors.py | 9 +- sdk/typescript/src/errors.ts | 4 + sdk/typescript/src/index.ts | 1 + server/code/app/answers_cache.py | 20 +- server/code/app/search/cards.py | 5 + server/code/app/search/hyde.py | 3 + server/code/app/search/track.py | 3 + server/code/app/search/vector.py | 3 + server/code/app/search_fanout.py | 44 ++- server/code/app/search_legs.py | 38 ++- server/code/store/index/answers.py | 8 +- server/code/store/index/files.py | 87 ++--- server/code/store/index/search.py | 176 +++++----- server/core/ch.py | 162 +++++++++- server/core/errors.py | 48 ++- server/core/vectors/admission.py | 244 ++++++++++++++ server/identity/app/namespace_catalog.py | 20 +- server/knowledge/app/conversation_search.py | 28 +- server/knowledge/app/conversations.py | 4 +- server/knowledge/app/repo_docs.py | 51 ++- server/knowledge/app/search.py | 10 +- server/knowledge/app/search_legs.py | 15 +- server/knowledge/app/source_search.py | 13 +- server/knowledge/store/source_candidates.py | 8 +- server/memory/app/ask/pool.py | 6 +- server/memory/app/ask/retrieval_legs.py | 8 +- server/memory/app/bulk_set.py | 28 +- server/memory/app/conflicts.py | 83 ++--- server/memory/app/loom_search/fanout.py | 78 +++-- .../memory/app/loom_search/repo_decisions.py | 10 +- server/memory/app/loom_search/tracker_leg.py | 8 +- server/memory/app/profile_search.py | 22 +- server/memory/app/retrieval/chunks.py | 9 +- server/memory/app/retrieval/derived.py | 34 +- server/memory/app/retrieval/fact_sql.py | 12 +- server/memory/app/retrieval/pool.py | 7 +- server/memory/app/retrieval/retry.py | 11 +- server/memory/app/retrieval/search.py | 20 +- server/memory/app/retrieval/side_legs.py | 16 +- server/memory/app/retrieval/summaries.py | 12 +- server/memory/app/search/anchor_pool.py | 9 +- server/memory/app/search/ann_entity_boosts.py | 25 +- server/memory/app/search/assemble.py | 7 + server/memory/app/search/computed.py | 7 +- server/memory/app/search/consistency.py | 6 +- server/memory/app/search/digest_descent.py | 9 +- server/memory/app/search/fallbacks.py | 4 +- server/memory/app/search/key_match.py | 9 +- server/memory/app/search/supplemental.py | 3 + server/memory/app/search/vector_legs.py | 3 + server/memory/app/session_outcomes.py | 30 +- server/memory/app/value_updates.py | 86 ++--- server/memory/store/conflict_resolver.py | 6 +- server/memory/store/lookups.py | 107 +++--- server/settings/catalog.py | 6 + server/tracker/app/dupcheck.py | 103 ++++-- server/tracker/app/handlers.py | 19 +- server/tracker/app/router.py | 15 +- server/tracker/app/similar.py | 101 ++++-- server/tracker/store/embed.py | 24 +- server/tracker/store/source/base.py | 38 ++- server/tracker/store/source/qa_intel.py | 8 +- server/tracker/store/source/similarity.py | 1 + tests/unit/snapshots/live_settings.json | 2 + .../unit/test_vector_admission_boundaries.py | 124 +++++++ .../unit/test_vector_capacity_composition.py | 176 ++++++++++ tests/unit/test_vector_read_admission.py | 306 ++++++++++++++++++ 73 files changed, 2145 insertions(+), 528 deletions(-) create mode 100644 server/core/vectors/admission.py create mode 100644 tests/unit/test_vector_admission_boundaries.py create mode 100644 tests/unit/test_vector_capacity_composition.py create mode 100644 tests/unit/test_vector_read_admission.py diff --git a/docs/architecture/clickhouse.md b/docs/architecture/clickhouse.md index 60c6c8456f..ea5642c732 100644 --- a/docs/architecture/clickhouse.md +++ b/docs/architecture/clickhouse.md @@ -1021,3 +1021,36 @@ rows. When any replica is older, every Loom session turns the cache off when it - [Migrations](../development/migrations.md): writing and testing a migration. - [Troubleshooting](../operations/troubleshooting.md): too many parts, slow inserts. - [Performance tuning](../guides/performance-tuning.md): traffic classes, vector knobs. + + +## Vector read admission + +Vector dispatches carry their target org, authorized namespaces and probe weight. +One query vector against one stored axis costs one unit; a value/key read or entity +boost costs two, and a six-probe conflict statement costs six. These units are +independent of connection slots. SQL, tenant predicates and ranking stay in their +owning readers. + +Each process admits 16 weighted units per org and queues at most 256 more by +default. Its total population is at most twice the configured waiting limit, +including org waits, connection waits, execution, retries and local retirement. +Reservation precedes any org or waiter allocation. Atomic FIFO grants prevent +partial acquisition. Lower live limits refuse new reservations while existing +charges drain; an overweight queued statement refuses instead of blocking its +successors. Disabling the existing connection gate refuses new vector work. + +Every vector attempt has finite time and memory budgets and a native throw ceiling +of one million read rows by default. Versions, joins and several namespaces can +reach that ceiling before one million logical keys. Admission and native refusals +propagate through optional read legs rather than returning a different ranking. +A selected conflict check refuses before commit. Optional value linking after a +commit preserves those versions and records the refused linking in their existing +result guidance and degradation record. + +The existing streaming slots and upstream tracker credential connections remain +separate from ordinary pooled connections. Their vector work shares the same +weighted population; permission reads consuming a stream do not inherit its vector +attribution. Local cleanup does not prove remote cancellation or a fleet-wide bound. +Native retry amplification, maximum supported shape completion and competing-org +progress require deployment qualification. Placement evidence alone does not +report this read capacity, and the full capacity report remains unfinished. diff --git a/docs/reference/configuration.md b/docs/reference/configuration.md index 097402ef89..211eea8f3d 100644 --- a/docs/reference/configuration.md +++ b/docs/reference/configuration.md @@ -159,6 +159,9 @@ ClickHouse connection, query shaping and vector storage. | `LOOM_CH_LOG_COMMENT_STATEMENT_DIGEST` | `bool` | `False` | | Append `sql: ` of each statement to its `log_comment` (" "), so query_log rows group by shape. A statement that carries a secret, or one run as the analyst, gets its shape alone. Off by default: readers that split log_comment on spaces gain tokens. | | `LOOM_CH_CAPTURE_STATEMENTS` | `str` | `''` | | A file path: append one JSON line (`digest`, `sql`) per executed statement, to collect the shapes an acceptance run executes for the analyzer corpus. '' = off; never set it in production. | | `LOOM_CH_POOL_MAXSIZE` | `int` | `64` | | HTTP connection pool size: request concurrency plus 3 x the batched-insert bound (inserts a second x wait / 1000; docs/architecture/clickhouse.md#batched-inserts). | +| `LOOM_VECTOR_SCAN_MAX_CONCURRENT_PER_ORG` | `int` | `16` | live | Weighted vector work admitted per org in this process; two axes need two units. | +| `LOOM_VECTOR_SCAN_MAX_WAITERS_PER_ORG` | `int` | `256` | live | Weighted org wait units; twice this also bounds the whole process population. | +| `LOOM_VECTOR_SCAN_MAX_ROWS_TO_READ` | `int` | `1000000` | | Native throw ceiling per vector statement, including joins and row versions. | | `LOOM_CH_KEEPALIVE_TIMEOUT` | `float` | `5.0` | | Seconds an idle pooled connection is kept. Must stay BELOW the Cloud load balancer's idle cutoff (8 s) so loom closes first: a peer-closed pooled TLS connection is never reaped under uvloop (`LOOM_UVICORN_LOOP=uvloop`). | | `LOOM_CH_READ_YOUR_WRITES` | `bool` | `True` | | A point lookup that MISSES right after a write (get after set, verify, update_by_id, delete) re-reads once with select_sequential_consistency=1. Costs only on a miss; a no-op on single-node MergeTree. | | `LOOM_CH_READ_YOUR_WRITES_WINDOW_S` | `float` | `20.0` | | memory.search on a namespace this process wrote within this many seconds reads with select_sequential_consistency from the start. 0 = off (the empty-pool retry remains the backstop). | diff --git a/docs/reference/errors.md b/docs/reference/errors.md index 2492b12cf3..48e504c7e4 100644 --- a/docs/reference/errors.md +++ b/docs/reference/errors.md @@ -964,6 +964,10 @@ Raised in `server/memory/app/apply.py`, `server/memory/app/lock.py`, `server/mem HTTP 422, raised as `server.core.errors.ValidationError`. Category `input`, retryable: no. What to do: [common errors](#common-errors-and-what-to-do). +#### `VECTOR_CAPACITY_REFUSED` + +HTTP 429, raised as `server.core.errors.VectorAdmissionError`. 429 — complete vector work exceeds a weighted queue or native budget. Category `capacity`, retryable: yes. + #### `VECTOR_INPUT_TOO_LARGE` HTTP 422, raised as `server.core.errors.VectorInputTooLargeError`. 422 — an entity filter exceeds 64 raw names, or its canonical aliases or summary seeds exceed 256 distinct names. Category `input`, retryable: no. diff --git a/loom.example.yaml b/loom.example.yaml index 6788dcfdec..23ee2fdb73 100644 --- a/loom.example.yaml +++ b/loom.example.yaml @@ -81,6 +81,17 @@ settings: # [restart] # ch_pool_maxsize: 64 + # Weighted vector work admitted per org in this process; two axes need two units. + # [live] + # vector_scan_max_concurrent_per_org: 16 + + # Weighted org wait units; twice this also bounds the whole process population. [live] + # vector_scan_max_waiters_per_org: 256 + + # Native throw ceiling per vector statement, including joins and row versions. + # [restart] + # vector_scan_max_rows_to_read: 1000000 + # Seconds an idle pooled connection is kept. Must stay BELOW the Cloud load balancer's # idle cutoff (8 s) so loom closes first: a peer-closed pooled TLS connection is never # reaped under uvloop (`LOOM_UVICORN_LOOP=uvloop`). [restart] diff --git a/schemas/loom-settings.schema.json b/schemas/loom-settings.schema.json index adec0ca6a8..71cc5cea67 100644 --- a/schemas/loom-settings.schema.json +++ b/schemas/loom-settings.schema.json @@ -7181,6 +7181,24 @@ "type": "integer", "description": "Seconds between re-reads of which vector columns are coded and which coded tables carry patch parts. A write that embeds reads again on its own when the reading is older than a few seconds (codes_state.FRESH_READING_S)." }, + "vector_scan_max_concurrent_per_org": { + "default": 16, + "minimum": 2, + "type": "integer", + "description": "Weighted vector work admitted per org in this process; two axes need two units." + }, + "vector_scan_max_rows_to_read": { + "default": 1000000, + "exclusiveMinimum": 0, + "type": "integer", + "description": "Native throw ceiling per vector statement, including joins and row versions." + }, + "vector_scan_max_waiters_per_org": { + "default": 256, + "exclusiveMinimum": 0, + "type": "integer", + "description": "Weighted org wait units; twice this also bounds the whole process population." + }, "vector_search_quantized_codes": { "default": true, "type": "boolean", diff --git a/sdk/python/loom/__init__.py b/sdk/python/loom/__init__.py index d45ae805a2..356de399c0 100644 --- a/sdk/python/loom/__init__.py +++ b/sdk/python/loom/__init__.py @@ -36,6 +36,7 @@ PreconditionFailed, RateLimited, ValidationError, + VectorCapacityRefused, WrongCell, ) from loom.retry import DEFAULT_RETRY_POLICY, RetryPolicy @@ -77,6 +78,7 @@ "SyncAgent", "TokenClaims", "ValidationError", + "VectorCapacityRefused", "WrongCell", "__version__", "decode_token_claims", diff --git a/sdk/python/loom/errors.py b/sdk/python/loom/errors.py index 818dadcc9f..ca0ac299db 100644 --- a/sdk/python/loom/errors.py +++ b/sdk/python/loom/errors.py @@ -104,6 +104,12 @@ def __init__(self, message: str = "", retry_after_s: float = DEFAULT_RETRY_AFTER self.retry_after_s = retry_after_s +class VectorCapacityRefused(RateLimited): + """A complete vector read exceeded admission or native budgets; no subset was served.""" + + code = "VECTOR_CAPACITY_REFUSED" + + class Overloaded(BackendUnavailable): """503 OVERLOADED — the server was too busy: every pooled ClickHouse connection of the serving process was busy for longer than its slot wait, or a lock or claim @@ -252,6 +258,7 @@ class WrongCell(PreconditionFailed): "AUTH_INSUFFICIENT": AuthInsufficient, "RESOURCE_NOT_FOUND": NotFound, "RATE_LIMITED": RateLimited, + "VECTOR_CAPACITY_REFUSED": VectorCapacityRefused, "BACKEND_UNAVAILABLE": BackendUnavailable, "OVERLOADED": Overloaded, "BACKEND_TIMEOUT": BackendTimeout, @@ -305,7 +312,7 @@ def from_response( message = str(payload)[:500] status = int(status or 0) cls = ERROR_CODES.get(code) or _STATUS_CLASSES.get(status, LoomError) - if status == 429: + if status == 429 and not issubclass(cls, RateLimited): cls = RateLimited elif status >= 500 and not issubclass(cls, BackendUnavailable): cls = BackendUnavailable diff --git a/sdk/typescript/src/errors.ts b/sdk/typescript/src/errors.ts index f2e3845e7f..6a3ee13815 100644 --- a/sdk/typescript/src/errors.ts +++ b/sdk/typescript/src/errors.ts @@ -160,6 +160,8 @@ export class RateLimited extends LoomError { this.retryAfterS = retryAfterS; } } +/** 429 VECTOR_CAPACITY_REFUSED: the complete vector read exceeded admission or native budgets. */ +export class VectorCapacityRefused extends RateLimited {} /** 5xx after the transport's retries, or a connection that never got a response. */ export class BackendUnavailable extends LoomError {} /** @@ -203,6 +205,8 @@ export function errorFromResponse( const code = String(detail.code ?? ""); const message = String(detail.message ?? "") || `${op}: HTTP ${status}`; const opts: LoomErrorOptions = { status, code, requestId, detail, op }; + if (status === 429 && code === "VECTOR_CAPACITY_REFUSED") + return new VectorCapacityRefused(message, retryAfterS, opts); if (status === 429) return new RateLimited(message, retryAfterS, opts); if (status === 400 || status === 422) return new ValidationError(message, opts); if (status === 401 || status === 403) return new AuthError(message, opts); diff --git a/sdk/typescript/src/index.ts b/sdk/typescript/src/index.ts index 9d9b3347fd..ac23b7c75a 100644 --- a/sdk/typescript/src/index.ts +++ b/sdk/typescript/src/index.ts @@ -50,6 +50,7 @@ export { RateLimited, TimeoutError, ValidationError, + VectorCapacityRefused, WrongCell, } from "./errors.js"; export type { FileData, IngestFileInput, KnowledgeFiles, Upload, UploadOptions, WaitOptions } from "./files.js"; diff --git a/server/code/app/answers_cache.py b/server/code/app/answers_cache.py index 253e49c25f..f021e1a196 100644 --- a/server/code/app/answers_cache.py +++ b/server/code/app/answers_cache.py @@ -4,12 +4,12 @@ from __future__ import annotations -import asyncio import json import logging import re import time import uuid +from collections.abc import Awaitable from dataclasses import dataclass from typing import Any @@ -22,7 +22,9 @@ normalize_question, ) from server.code.store.index import CodeRepo, build_row +from server.core.errors import VectorCapacityError from server.core.shielded import shielded +from server.core.vectors.private_first import each from server.settings import settings logger = logging.getLogger(__name__) @@ -97,11 +99,19 @@ class _Lookup: async def _serve_from_cache(ctx: _AskCtx) -> _Lookup: """Own namespace's semantic entry, else the exact question's cached abstain, else the release branch's trunk entry. Fails open to a miss.""" + + async def _read(call: Awaitable[Any]) -> Any: + return await call + try: - (outcome, row, sim), negative = await asyncio.gather( - _semantic(ctx, ctx.ns, verify_ns=ctx.ns, own=True), - _negative(ctx), + (outcome, row, sim), negative = await each(_read)( + [ + _semantic(ctx, ctx.ns, verify_ns=ctx.ns, own=True), + _negative(ctx), + ] ) + except VectorCapacityError: + raise except Exception as e: # noqa: BLE001 — a cache read never sinks the ask logger.debug("code.ask cache read failed: %s", e) return _Lookup("miss") @@ -250,6 +260,8 @@ async def _reuse_from_trunk(ctx: _AskCtx) -> _Lookup | None: return None try: outcome, row, sim = await _semantic(ctx, trunk, verify_ns=ctx.ns, own=False) + except VectorCapacityError: + raise except Exception as e: # noqa: BLE001 — the trunk cache is an optimization: a miss logger.debug("code.ask trunk cache read failed: %s", e) return None diff --git a/server/code/app/search/cards.py b/server/code/app/search/cards.py index 18e46200e5..2324b50efb 100644 --- a/server/code/app/search/cards.py +++ b/server/code/app/search/cards.py @@ -17,6 +17,7 @@ from server.code.domain.index import query_trim from server.code.domain.search_scope import SearchScope from server.code.store.index import CodeRepo +from server.core.errors import VectorCapacityError from server.settings import settings from server.settings.budgets import llm_max_tokens @@ -94,6 +95,8 @@ async def _card_leg_rows( int(r.get("start_line") or 0), ) ) + except VectorCapacityError: + raise except Exception as e: # noqa: BLE001 — a vote, never a gate logger.warning("code.search card leg failed: %s", e) work["card_leg_failed"] = 1 @@ -226,6 +229,8 @@ async def _source_cards() -> list[dict[str, Any]]: pool=pool, skip_tests=True, ) + except VectorCapacityError: + raise except Exception as e: # noqa: BLE001 — a vote, never a gate logger.warning("code.search card term read failed: %s", e) return [] diff --git a/server/code/app/search/hyde.py b/server/code/app/search/hyde.py index af7ea06b36..976572013a 100644 --- a/server/code/app/search/hyde.py +++ b/server/code/app/search/hyde.py @@ -11,6 +11,7 @@ from server.code.app.spans import code_span, span_attrs from server.code.domain.search_scope import SearchScope from server.code.store.index import CodeRepo +from server.core.errors import VectorCapacityError from server.settings import settings logger = logging.getLogger(__name__) @@ -96,6 +97,8 @@ async def _hyde_leg_rows( work["hyde_ms"] = int((time.perf_counter() - t0) * 1000) span_attrs(sp, results=len(rows)) return rows + except VectorCapacityError: + raise except Exception as e: # noqa: BLE001 — a vote, never a gate logger.warning("code.search hyde leg failed: %s", e) work["hyde_leg_failed"] = 1 diff --git a/server/code/app/search/track.py b/server/code/app/search/track.py index 64810edccd..525a90f795 100644 --- a/server/code/app/search/track.py +++ b/server/code/app/search/track.py @@ -10,6 +10,7 @@ from server.code.app.search.state import _POOL, _model_name from server.code.app.spans import code_span, span_attrs from server.code.store.index import CodeRepo +from server.core.errors import VectorCapacityError logger = logging.getLogger(__name__) @@ -79,6 +80,8 @@ async def _track( if found else [] ) + except VectorCapacityError: + raise except Exception as e: # noqa: BLE001 — a diagnostic never sinks a search logger.warning("code.search track_path read failed: %s", e) return {"path": path, "verdict": f"track read failed: {type(e).__name__}"} diff --git a/server/code/app/search/vector.py b/server/code/app/search/vector.py index 983b8fd009..7645d79442 100644 --- a/server/code/app/search/vector.py +++ b/server/code/app/search/vector.py @@ -12,6 +12,7 @@ from server.code.app.spans import code_span, span_attrs from server.code.domain.search_scope import SearchScope from server.code.store.index import CodeRepo +from server.core.errors import VectorCapacityError from server.settings import settings logger = logging.getLogger(__name__) @@ -155,6 +156,8 @@ async def _vector_leg_rows( # a second model's query vector cannot score the # served model's rows (track_path reads those) qvec = None + except VectorCapacityError: + raise except Exception as e: # noqa: BLE001 — lexical-only degrade logger.warning("code.search vector leg failed: %s", e) work["vector_leg_failed"] = 1 diff --git a/server/code/app/search_fanout.py b/server/code/app/search_fanout.py index 64b4bf56e3..5fbdab2a70 100644 --- a/server/code/app/search_fanout.py +++ b/server/code/app/search_fanout.py @@ -26,6 +26,8 @@ from typing import TYPE_CHECKING, Any, Literal from server.code.api.models_search import CodeSearchInput, CodeSearchOutput +from server.core.errors import VectorCapacityError +from server.core.shielded import to_its_end from server.settings import settings if TYPE_CHECKING: @@ -165,23 +167,39 @@ async def run_all( for ns in namespaces: sub = inp.model_copy(update={"namespace": ns, "namespaces": []}) tasks[ns] = asyncio.ensure_future(_timed(ns, single(sub))) + + async def stopped() -> None: + await asyncio.gather(*tasks.values(), return_exceptions=True) + + def capacity_failure() -> None: + for task in tasks.values(): + if task.done() and not task.cancelled(): + error = task.exception() + if isinstance(error, VectorCapacityError): + raise error + try: done, pending = await asyncio.wait(tasks.values(), timeout=max(0.001, timeout_ms / 1000.0)) + capacity_failure() + for task in pending: + task.cancel() + if pending: + await to_its_end(stopped(), name="code.search.fanout.deadline.cleanup") + capacity_failure() + answered: Answered = {} + for task in done: + ns, out, ms = task.result() + answered[ns] = (out, ms) + dropped = [ns for ns, task in tasks.items() if task in pending] + return answered, dropped except BaseException: - # the caller was cancelled: `wait` leaves its tasks running - for t in tasks.values(): - t.cancel() + for task in tasks.values(): + task.cancel() + try: + await to_its_end(stopped(), name="code.search.fanout.cleanup") + except asyncio.CancelledError: + pass # Preserve the first failure after every owned namespace stops. raise - for t in pending: - t.cancel() - if pending: - await asyncio.gather(*pending, return_exceptions=True) - answered: Answered = {} - for t in done: - ns, out, ms = t.result() - answered[ns] = (out, ms) - dropped = [ns for ns, t in tasks.items() if t in pending] - return answered, dropped def interleave(answered: Answered, namespaces: list[str], top_k: int) -> list[CodeUnit]: diff --git a/server/code/app/search_legs.py b/server/code/app/search_legs.py index 46fef7de10..ce9372550d 100644 --- a/server/code/app/search_legs.py +++ b/server/code/app/search_legs.py @@ -29,12 +29,13 @@ from __future__ import annotations import asyncio -import contextlib import time from collections.abc import Awaitable from typing import Any +from server.core.errors import VectorCapacityError from server.core.llm_free import reads_llm_free +from server.core.shielded import to_its_end from server.settings import settings # The legs whose rows the answer cannot do without; the deadline waits for @@ -103,10 +104,16 @@ async def run_legs( legs, tasks, result=result, work=work, deadline_ms=deadline_ms, floor_ms=floor_ms ) except BaseException: - # the caller was cancelled, or a leg raised: no sibling leg keeps - # reading after the request has gone - for t in tasks.values(): - t.cancel() + for task in tasks.values(): + task.cancel() + + async def stopped() -> None: + await asyncio.gather(*tasks.values(), return_exceptions=True) + + try: + await to_its_end(stopped(), name="code.search.legs.cleanup") + except asyncio.CancelledError: + pass # Preserve the first failure after every owned leg stops. raise @@ -136,13 +143,22 @@ def _left(cap_s: float | None = None) -> float | None: return rem return cap_s if rem is None else min(rem, cap_s) + def capacity_failure() -> None: + for task in tasks.values(): + if task.done() and not task.cancelled(): + error = task.exception() + if isinstance(error, VectorCapacityError): + raise error + plain = [tasks[n] for n in names if n not in LLM_LEGS] model = [tasks[n] for n in names if n in LLM_LEGS] if plain: await asyncio.wait(plain, timeout=_left()) + capacity_failure() pending_model = [t for t in model if not t.done()] if pending_model: await asyncio.wait(pending_model, timeout=_left(grace / 1000.0 if grace else None)) + capacity_failure() core = [tasks[n] for n in names if n in CORE_LEGS] if core and not any(t.done() for t in core): # the floor: an answer from at least one content leg, inside the budget @@ -151,12 +167,18 @@ def _left(cap_s: float | None = None) -> float | None: return_when=asyncio.FIRST_COMPLETED, timeout=None if floor_ms is None else floor_ms / 1000.0, ) + capacity_failure() dropped = [n for n in names if not tasks[n].done()] for n in dropped: tasks[n].cancel() - for n in dropped: - with contextlib.suppress(asyncio.CancelledError, Exception): - await tasks[n] + if dropped: + + async def stopped() -> None: + await asyncio.gather(*(tasks[n] for n in dropped), return_exceptions=True) + + await to_its_end(stopped(), name="code.search.deadline.cleanup") + # A leg can refuse capacity while its deadline cancellation is settling. + capacity_failure() if dropped: result.legs_timed_out = dropped result.epistemic = "partial" diff --git a/server/code/store/index/answers.py b/server/code/store/index/answers.py index ad8bffdc20..d54edc16a1 100644 --- a/server/code/store/index/answers.py +++ b/server/code/store/index/answers.py @@ -8,6 +8,7 @@ from typing import Any from server.code.store.index.base import CodeRepoBase +from server.core.vectors.admission import vector_work def _candidate_sql() -> str: @@ -42,9 +43,10 @@ async def nearest_answers( ) -> list[dict[str, Any]]: """The `k` live answer entries of (org, ns, model) nearest to `qvec`, nearest first, with their `distance` and `age_s`.""" - rows = await self._ch.query_all( - _candidate_sql(), {"org": org, "ns": ns, "model": model, "q": qvec, "k": k} - ) + with vector_work(org, ns): + rows = await self._ch.query_all( + _candidate_sql(), {"org": org, "ns": ns, "model": model, "q": qvec, "k": k} + ) return list(rows or []) async def negative_answer( diff --git a/server/code/store/index/files.py b/server/code/store/index/files.py index a8a6e061bf..05279c41d6 100644 --- a/server/code/store/index/files.py +++ b/server/code/store/index/files.py @@ -9,6 +9,7 @@ from server.code.store.index.base import CodeRepoBase from server.code.store.index.sql import _batched, _gather_batched, _hex +from server.core.vectors.admission import vector_work class FilesMixin(CodeRepoBase): @@ -491,36 +492,37 @@ async def track_path_chunks( if query_vec else "toFloat64(-1.0)" ) - return await self._ch.query_all( - f""" - SELECT chunk_id, path, scope, kind, start_line, end_line, embedding_dim, - arraySum( - arrayMap( - (tok, idf) -> idf * countSubstrings( - concat(' ', lexemes, ' '), concat(' ', tok, ' ') - ), - {{tokens:Array(String)}}, {{idfs:Array(Float32)}} - ) - ) AS lex_score, - {vec_expr} AS distance - FROM loom.code_chunks FINAL - WHERE org = {{org:String}} AND namespace = {{ns:String}} - AND embedding_model = {{model:String}} AND path IN {{paths:Array(String)}} - AND tombstoned_at IS NULL - ORDER BY path, start_line - LIMIT {{limit:UInt32}} - """, - { - "org": org, - "ns": ns, - "model": model, - "paths": paths, - "tokens": tokens, - "idfs": idfs, - "limit": limit, - **({"q": query_vec} if query_vec else {}), - }, - ) + with vector_work(org, ns, 1 if query_vec else 0): + return await self._ch.query_all( + f""" + SELECT chunk_id, path, scope, kind, start_line, end_line, embedding_dim, + arraySum( + arrayMap( + (tok, idf) -> idf * countSubstrings( + concat(' ', lexemes, ' '), concat(' ', tok, ' ') + ), + {{tokens:Array(String)}}, {{idfs:Array(Float32)}} + ) + ) AS lex_score, + {vec_expr} AS distance + FROM loom.code_chunks FINAL + WHERE org = {{org:String}} AND namespace = {{ns:String}} + AND embedding_model = {{model:String}} AND path IN {{paths:Array(String)}} + AND tombstoned_at IS NULL + ORDER BY path, start_line + LIMIT {{limit:UInt32}} + """, + { + "org": org, + "ns": ns, + "model": model, + "paths": paths, + "tokens": tokens, + "idfs": idfs, + "limit": limit, + **({"q": query_vec} if query_vec else {}), + }, + ) async def best_chunks_for_paths( self, @@ -539,8 +541,9 @@ async def best_chunks_for_paths( this size.""" if not paths: return [] - return await self._ch.query_all( - """ + with vector_work(org, ns): + return await self._ch.query_all( + """ SELECT chunk_id, path, scope, kind, start_line, end_line, content, symbol_ids, context_description, cosineDistance(embedding, {q:Array(Float32)}) AS distance @@ -552,16 +555,16 @@ async def best_chunks_for_paths( LIMIT {per_path:UInt32} BY path LIMIT {lim:UInt32} """, - { - "org": org, - "ns": ns, - "model": model, - "q": query_vec, - "paths": sorted(set(paths)), - "per_path": int(per_path), - "lim": int(per_path) * len(set(paths)), - }, - ) + { + "org": org, + "ns": ns, + "model": model, + "q": query_vec, + "paths": sorted(set(paths)), + "per_path": int(per_path), + "lim": int(per_path) * len(set(paths)), + }, + ) async def chunks_for_paths( self, diff --git a/server/code/store/index/search.py b/server/code/store/index/search.py index cf5f70df48..1ea337087e 100644 --- a/server/code/store/index/search.py +++ b/server/code/store/index/search.py @@ -6,6 +6,7 @@ from __future__ import annotations import asyncio +from collections.abc import Awaitable from datetime import datetime from typing import Any @@ -14,6 +15,8 @@ from server.code.store.index.base import CodeRepoBase from server.code.store.index.sql import _scope_sql, skip_tests_sql from server.core.datetimes import naive_utc_now as _now +from server.core.vectors.admission import vector_work +from server.core.vectors.private_first import each class SearchMixin(CodeRepoBase): @@ -120,28 +123,29 @@ async def vector_search_cards( # Source files only — two thirds of the ClickHouse # cards are tests, and for a behaviour question they are the nearest where_prefix += skip_tests_sql("scope_path") - return await self._ch.query_all( - f""" - SELECT scope_path AS path, summary, pagerank, - cosineDistance(embedding, {{q:Array(Float32)}}) AS distance - FROM loom.code_summaries FINAL - WHERE org = {{org:String}} AND namespace = {{ns:String}} - AND embedding_model = {{model:String}} AND scope_type = 'file' - AND embedding_dim > 0 AND tombstoned_at IS NULL AND summary != '' - {where_prefix} - ORDER BY distance ASC - LIMIT {{pool:UInt32}} - {cards_settings} - """, - { - "org": org, - "ns": ns, - "model": model, - "q": query_vec, - "pool": pool, - **({"prefix": path_prefix} if path_prefix else {}), - }, - ) + with vector_work(org, ns): + return await self._ch.query_all( + f""" + SELECT scope_path AS path, summary, pagerank, + cosineDistance(embedding, {{q:Array(Float32)}}) AS distance + FROM loom.code_summaries FINAL + WHERE org = {{org:String}} AND namespace = {{ns:String}} + AND embedding_model = {{model:String}} AND scope_type = 'file' + AND embedding_dim > 0 AND tombstoned_at IS NULL AND summary != '' + {where_prefix} + ORDER BY distance ASC + LIMIT {{pool:UInt32}} + {cards_settings} + """, + { + "org": org, + "ns": ns, + "model": model, + "q": query_vec, + "pool": pool, + **({"prefix": path_prefix} if path_prefix else {}), + }, + ) async def vector_search_chunks( self, @@ -176,31 +180,32 @@ async def vector_search_chunks( # maps to (the coded column is one width per table); the served # label resolves to loom.code_chunks. table = code_vector_table(model) - rows = await self._ch.query_all( - f""" - SELECT chunk_id, path, scope, kind, start_line, end_line, - content, symbol_ids, context_description, - cosineDistance(embedding, {{q:Array(Float32)}}) AS distance - FROM {table} FINAL - WHERE org = {{org:String}} AND namespace = {{ns:String}} - AND embedding_model = {{model:String}} AND embedding_dim > 0 - AND tombstoned_at IS NULL - {where_prefix} - {scope_sql} - ORDER BY distance ASC - LIMIT {{pool:UInt32}} - {vector_search_settings(table=table)} - """, - { - "org": org, - "ns": ns, - "model": model, - "q": query_vec, - "pool": pool, - **({"prefix": path_prefix} if path_prefix else {}), - **scope_params, - }, - ) + with vector_work(org, ns): + rows = await self._ch.query_all( + f""" + SELECT chunk_id, path, scope, kind, start_line, end_line, + content, symbol_ids, context_description, + cosineDistance(embedding, {{q:Array(Float32)}}) AS distance + FROM {table} FINAL + WHERE org = {{org:String}} AND namespace = {{ns:String}} + AND embedding_model = {{model:String}} AND embedding_dim > 0 + AND tombstoned_at IS NULL + {where_prefix} + {scope_sql} + ORDER BY distance ASC + LIMIT {{pool:UInt32}} + {vector_search_settings(table=table)} + """, + { + "org": org, + "ns": ns, + "model": model, + "q": query_vec, + "pool": pool, + **({"prefix": path_prefix} if path_prefix else {}), + **scope_params, + }, + ) return rows async def vector_shortlist_ids( @@ -224,30 +229,31 @@ async def vector_shortlist_ids( where_prefix = "AND startsWith(path, {prefix:String})" if path_prefix else "" scope_sql, scope_params = _scope_sql(scope) table = code_vector_table(model) - return await self._ch.query_all( - f""" - SELECT chunk_id, path, - cosineDistance(embedding, {{q:Array(Float32)}}) AS distance - FROM {table} FINAL - WHERE org = {{org:String}} AND namespace = {{ns:String}} - AND embedding_model = {{model:String}} AND embedding_dim > 0 - AND tombstoned_at IS NULL - {where_prefix} - {scope_sql} - ORDER BY distance ASC - LIMIT {{n:UInt32}} - {vector_search_settings(table=table)} - """, - { - "org": org, - "ns": ns, - "model": model, - "q": query_vec, - "n": n, - **({"prefix": path_prefix} if path_prefix else {}), - **scope_params, - }, - ) + with vector_work(org, ns): + return await self._ch.query_all( + f""" + SELECT chunk_id, path, + cosineDistance(embedding, {{q:Array(Float32)}}) AS distance + FROM {table} FINAL + WHERE org = {{org:String}} AND namespace = {{ns:String}} + AND embedding_model = {{model:String}} AND embedding_dim > 0 + AND tombstoned_at IS NULL + {where_prefix} + {scope_sql} + ORDER BY distance ASC + LIMIT {{n:UInt32}} + {vector_search_settings(table=table)} + """, + { + "org": org, + "ns": ns, + "model": model, + "q": query_vec, + "n": n, + **({"prefix": path_prefix} if path_prefix else {}), + **scope_params, + }, + ) async def exact_rerank_by_ids( self, @@ -289,9 +295,11 @@ async def exact_rerank_by_ids( ident = """org = {org:String} AND namespace = {ns:String} AND embedding_model = {model:String} AND path IN {paths:Array(String)} AND chunk_id IN {ids:Array(UUID)}""" - rows, tombs = await asyncio.gather( - self._ch.query_all( - f""" + + async def _ranked() -> list[dict[str, Any]]: + with vector_work(org, ns): + return await self._ch.query_all( + f""" SELECT chunk_id, path, scope, kind, start_line, end_line, content, symbol_ids, context_description, distance FROM ( @@ -307,18 +315,26 @@ async def exact_rerank_by_ids( ORDER BY distance ASC LIMIT {{lim:UInt32}} """, - params, - ), - self._ch.query_all( - f""" + params, + ) + + async def _read(call: Awaitable[list[dict[str, Any]]]) -> list[dict[str, Any]]: + return await call + + rows, tombs = await each(_read)( + [ + _ranked(), + self._ch.query_all( + f""" SELECT path, chunk_id, argMax(tombstoned_at IS NULL, version) AS is_live FROM loom.code_chunks WHERE {ident} GROUP BY path, chunk_id HAVING is_live = 1 """, - params, - ), + params, + ), + ] ) # the keys whose newest version is live (the null-safe idiom: a # re-inserted key reads live, not its old tombstone) diff --git a/server/core/ch.py b/server/core/ch.py index 15170e721f..4bb327de75 100644 --- a/server/core/ch.py +++ b/server/core/ch.py @@ -53,10 +53,18 @@ from server.core.ch_statement_text import capture_statement, statement_digest, statement_shape from server.core.ch_target import connection_kwargs from server.core.ch_writes import dedup_settings, insert_dedup_token -from server.core.errors import OverloadedError +from server.core.errors import OverloadedError, VectorOverloadedError +from server.core.shielded import to_its_end from server.core.sql_comments import LEADING_COMMENTS from server.core.sql_literal import sql_string from server.core.traffic_context import traffic_class_var +from server.core.vectors.admission import ( + admit, + current_work, + native_settings, + raise_native, + vector_work, +) from server.settings import settings log = logging.getLogger(__name__) @@ -578,6 +586,8 @@ class also carries that class's settings from # collide with them. out.pop("max_execution_time", None) out.update(analyst[2]) + if current_work() is not None: + native_settings(out, timeout=vector_timeout_s()) return _with_request_query_id(out) tclass = traffic_class_var.get()[0] if tclass and is_read_statement(sql): @@ -592,9 +602,42 @@ class also carries that class's settings from use_query_cache=0, use_query_condition_cache=0, ) + if current_work() is not None: + native_settings(out, timeout=vector_timeout_s()) return _with_request_query_id(out) +def vector_timeout_s() -> float: + """A vector attempt always has a finite deadline, even with class limits off.""" + limit = class_timeout_s("read") + ceiling = float(settings.ch_caller_shaped_max_execution_s) + if not math.isfinite(ceiling) or ceiling <= 0: + from server.core.errors import VectorAdmissionError + + raise VectorAdmissionError("vector native deadline is unavailable", reason="configuration") + return min(limit, ceiling) if math.isfinite(limit) and limit > 0 else ceiling + + +async def _close_vector_stream(stream: Any, failure: BaseException | None) -> None: + """Finish driver close without replacing the consumer's first failure.""" + try: + await to_its_end( + stream.__aexit__( + type(failure) if failure is not None else None, + failure, + failure.__traceback__ if failure is not None else None, + ), + name="vector-stream-close", + ) + except asyncio.CancelledError: + if failure is None: + raise + except Exception as close_error: + if failure is None: + raise + log.warning("vector stream cleanup failed (%s)", type(close_error).__name__) + + def _untagged_caller() -> str: """`untagged:.` for the nearest server frame above the ClickHouse plumbing, or '' when the call comes from outside `server`. @@ -944,6 +987,10 @@ async def open_channel() -> None: finally: self._stream_slots.release() + def vector_pool_ready(self) -> bool: + """The bounded pool gate exists; checking it allocates and acquires nothing.""" + return self.__dict__.get("_slots") is not None + @property def connected(self) -> bool: return self._client is not None @@ -1113,7 +1160,8 @@ async def _bounded(self, op_name: str, cls: str, fn: Callable[[], Awaitable[Any] global _POOL_TIMEOUTS_TOTAL gen = int(self._generation) self._inflight[gen] = self._inflight.get(gen, 0) + 1 - limit = class_timeout_s(cls) + limit = vector_timeout_s() if current_work() is not None else class_timeout_s(cls) + failure: BaseException | None = None try: if limit <= 0: return await fn() @@ -1132,14 +1180,32 @@ async def _bounded(self, op_name: str, cls: str, fn: Callable[[], Awaitable[Any] cls, limit, ) + if current_work() is not None: + from server.core.errors import VectorAdmissionError + + raise VectorAdmissionError( + "the complete vector read exceeded its deadline", + reason="deadline", + retry_after_s=1, + ) from None raise BackendTimeoutError( f"ClickHouse {cls} statement exceeded {limit:g} s", statement_class=cls, timeout_s=limit, ) from None + except BaseException as error: + failure = error + raise finally: self._inflight[gen] = self._inflight.get(gen, 1) - 1 - await self._retire_if_done(gen) + if current_work() is None: + await self._retire_if_done(gen) + else: + try: + await to_its_end(self._retire_if_done(gen), name="vector-generation-retirement") + except asyncio.CancelledError: + if failure is None: + raise async def _acquire_slot(self, op_name: str) -> bool: """Take one pool slot or raise 503 OVERLOADED after `ch_pool_wait_ms`. @@ -1243,6 +1309,7 @@ async def _retry_ladder(self, op_name: str, fn, *, cls: str): # type: ignore[no try: return await self._attempts(op_name, fn, cls=cls) except Exception as e: + raise_native(e) if is_shared_memory_limit(e): raise _short_of_memory(op_name, e) from e raise @@ -1252,6 +1319,7 @@ async def _attempts(self, op_name: str, fn, *, cls: str): # type: ignore[no-unt try: return await self._bounded(op_name, cls, fn) except Exception as e: + raise_native(e) if is_transient_network_error(e): log.warning( "ch %s: transient %s, reconnecting and retrying", @@ -1262,6 +1330,7 @@ async def _attempts(self, op_name: str, fn, *, cls: str): # type: ignore[no-unt await self._reconnect(seen) return await self._bounded(op_name, cls, fn) except Exception as retry_error: + raise_native(retry_error) if not is_transient_network_error(retry_error): raise from server.core.errors import BackendUnavailableError @@ -1287,6 +1356,7 @@ async def _attempts(self, op_name: str, fn, *, cls: str): # type: ignore[no-unt try: return await self._bounded(op_name, cls, fn) except Exception as e2: + raise_native(e2) if not is_replica_bound_error(e2): raise last = e2 @@ -1306,6 +1376,19 @@ async def query_one( return dict(zip(result.column_names, rows[0], strict=False)) async def _query_fail_open(self, op_name: str, sql: str, params: dict[str, Any] | None) -> Any: + if current_work() is None: + return await self._query_attempts(op_name, sql, params) + async with admit(pool_ready=self.vector_pool_ready(), wait_s=vector_timeout_s()): + try: + return await self._query_attempts(op_name, sql, params) + except OverloadedError as error: + raise VectorOverloadedError( + "the bounded ClickHouse gate is busy", + reason="pool", + retry_after_s=1, + ) from error + + async def _query_attempts(self, op_name: str, sql: str, params: dict[str, Any] | None) -> Any: """Run a SELECT with the standard retry; if the server rejects the quantized-codes setting (a replica that does not know it), turn the setting off for this process and run the query once @@ -1322,7 +1405,8 @@ async def _q(text: str) -> Any: note_result_summary(res) return res - await self._guard_schema(op_name, sql, "read") + with vector_work("", (), 0): + await self._guard_schema(op_name, sql, "read") try: return await self._with_retry(op_name, lambda: _q(sql), cls="read") except Exception as e: @@ -1390,6 +1474,36 @@ async def raw_stream( params: dict[str, Any] | None = None, *, settings: dict[str, Any] | None = None, + ) -> AsyncIterator[AsyncIterator[bytes]]: + if current_work() is None: + async with self._raw_stream_query(sql, params, settings=settings) as body: + yield body + return + ready = self.vector_pool_ready() and self.__dict__.get("_stream_slots") is not None + async with admit(pool_ready=ready, wait_s=vector_timeout_s()): + try: + async with self._raw_stream_query(sql, params, settings=settings) as body: + # Permission checks consuming a batch are separate scalar reads. + # The stream keeps its reservation while their attribution is clear. + with vector_work("", (), 0): + yield body + except OverloadedError as error: + raise VectorOverloadedError( + "the bounded streaming gate is busy", + reason="pool", + retry_after_s=1, + ) from error + except Exception as error: + raise_native(error) + raise + + @asynccontextmanager + async def _raw_stream_query( + self, + sql: str, + params: dict[str, Any] | None = None, + *, + settings: dict[str, Any] | None = None, ) -> AsyncIterator[AsyncIterator[bytes]]: """A SELECT's output as raw byte chunks (`FORMAT RawBLOB` for one String value), for a body too large to buffer. Opening is retried like any read; the stream holds one @@ -1418,17 +1532,43 @@ async def _open() -> Any: return stream self._ensure_state() - await self._guard_schema("raw_stream", sql, "read") + with vector_work("", (), 0): + await self._guard_schema("raw_stream", sql, "read") await acquire_slot(self._stream_slots, "streaming read") + failure: BaseException | None = None try: stream = await self._retry_ladder("raw_stream", _open, cls="read") - async with stream: - yield stream + if current_work() is None: + async with stream: + yield stream + else: + await stream.__aenter__() + try: + yield stream + except BaseException as error: + failure = error + raise + finally: + await _close_vector_stream(stream, failure) + except BaseException as error: + failure = error + raise finally: - for gen in held: - self._inflight[gen] = self._inflight.get(gen, 1) - 1 - await self._retire_if_done(gen) - self._stream_slots.release() + try: + for gen in held: + self._inflight[gen] = self._inflight.get(gen, 1) - 1 + if current_work() is None: + await self._retire_if_done(gen) + else: + try: + await to_its_end( + self._retire_if_done(gen), name="vector-stream-retirement" + ) + except asyncio.CancelledError: + if failure is None: + raise + finally: + self._stream_slots.release() async def execute( self, diff --git a/server/core/errors.py b/server/core/errors.py index 6458c7c8b7..5416ab6964 100644 --- a/server/core/errors.py +++ b/server/core/errors.py @@ -43,7 +43,42 @@ class ValidationError(LoomServerError): status_code = 422 -class VectorInputTooLargeError(ValidationError): +VectorCapacityReason = Literal[ + "attribution", + "configuration", + "weight", + "population", + "org_queue", + "org_deadline", + "pool", + "rows", + "memory", + "deadline", +] + + +class VectorCapacityError(LoomServerError): + """Common refusal family: a read must not return a ranking subset.""" + + +class VectorAdmissionError(VectorCapacityError): + """429 — complete vector work exceeds a weighted queue or native budget. + `reason` names the bounded refusal; retry after `retry_after_s`, or narrow the read.""" + + code = "VECTOR_CAPACITY_REFUSED" + status_code = 429 + + def __init__( + self, + message: str, + *, + reason: VectorCapacityReason, + retry_after_s: int = 1, + ) -> None: + super().__init__(message, reason=reason, retry_after_s=retry_after_s) + + +class VectorInputTooLargeError(ValidationError, VectorCapacityError): """422 — an entity filter exceeds 64 raw names, or its canonical aliases or summary seeds exceed 256 distinct names. Narrow the query or filter; entity names are never truncated.""" @@ -276,6 +311,16 @@ def __init__(self, message: str = "", *, retry_after_s: int = 1) -> None: super().__init__(message, retry_after_s=retry_after_s) +class VectorOverloadedError(OverloadedError, VectorCapacityError): + """An existing process overload that must survive optional vector-read catches.""" + + def __init__( + self, message: str, *, reason: Literal["pool", "population"], retry_after_s: int = 1 + ) -> None: + super().__init__(message, retry_after_s=retry_after_s) + self.extra["reason"] = reason + + class OverrideBusyError(OverloadedError): """OVERLOADED because another web process was updating the same runtime override; the admin route waits it out, and nothing changed.""" @@ -556,6 +601,7 @@ def __init__(self, message: str = "", *, retry_after_s: int = 5) -> None: # that is not idempotent a handler's answer may follow a write (OVERLOADED too: each # statement waits for a pool slot), so there it is not retryable (`handler_failure_detail`). "OVERLOADED": ("capacity", True), + "VECTOR_CAPACITY_REFUSED": ("capacity", True), "GRANT_BUSY": ("capacity", True), "ERASURE_RUNNING": ("capacity", True), "IDEMPOTENT_IN_PROGRESS": ("capacity", True), diff --git a/server/core/vectors/admission.py b/server/core/vectors/admission.py new file mode 100644 index 0000000000..23a608e22a --- /dev/null +++ b/server/core/vectors/admission.py @@ -0,0 +1,244 @@ +"""Weighted local admission for explicitly attributed vector statements.""" + +from __future__ import annotations + +import asyncio +import math +from collections import deque +from collections.abc import AsyncIterator, Iterator +from contextlib import asynccontextmanager, contextmanager +from contextvars import ContextVar +from dataclasses import dataclass, field +from typing import Any + +from server.core.ch_errors import error_code +from server.core.errors import ( + VectorAdmissionError, + VectorCapacityError, + VectorCapacityReason, + VectorOverloadedError, +) +from server.settings import settings + + +@dataclass(frozen=True) +class VectorWork: + org: str + namespaces: tuple[str, ...] + weight: int + + +_work: ContextVar[VectorWork | None] = ContextVar("vector_statement_work", default=None) + + +@contextmanager +def vector_work( + org: str, namespaces: str | tuple[str, ...] | list[str], weight: int = 1 +) -> Iterator[None]: + """Attribute only the actual vector dispatch; this scope holds no credit.""" + scope = (namespaces,) if isinstance(namespaces, str) else tuple(namespaces) + if weight == 0: + token = _work.set(None) + try: + yield + finally: + _work.reset(token) + return + if not org or not scope or not all(scope) or weight < 1: + raise VectorAdmissionError("vector work has no valid target scope", reason="attribution") + token = _work.set(VectorWork(org, scope, weight)) + try: + yield + finally: + _work.reset(token) + + +def current_work() -> VectorWork | None: + return _work.get() + + +def _limits() -> tuple[int, int]: + c = int(settings.vector_scan_max_concurrent_per_org) + w = int(settings.vector_scan_max_waiters_per_org) + if c < 2 or w < 1: + raise VectorAdmissionError( + "vector admission is not configured safely", reason="configuration" + ) + return c, w + + +def batch_size(configured: int) -> int: + """A planned multi-probe statement fits the current weighted org ceiling.""" + c, _ = _limits() + return min(max(1, configured), c) + + +@dataclass +class _Waiter: + weight: int + ready: asyncio.Future[None] + granted: bool = False + + +@dataclass +class _Org: + held: int = 0 + waiting: int = 0 + queue: deque[_Waiter] = field(default_factory=deque) + + +class _Admission: + def __init__(self) -> None: + self.population = 0 + self.orgs: dict[str, _Org] = {} + + def _advance(self, org: _Org) -> None: + try: + c, _ = _limits() + except VectorCapacityError: + # Invalid live configuration stops grants while old charges drain. + return + while org.queue: + head = org.queue[0] + if head.ready.done(): + org.queue.popleft() + org.waiting -= head.weight + continue + if head.weight > c: + org.queue.popleft() + org.waiting -= head.weight + head.ready.set_exception( + VectorAdmissionError( + "vector statement exceeds the org work ceiling", + reason="weight", + retry_after_s=1, + ) + ) + continue + if org.held + head.weight > c: + break + org.queue.popleft() + org.waiting -= head.weight + org.held += head.weight + head.granted = True + head.ready.set_result(None) + + async def enter(self, work: VectorWork, wait_s: float) -> tuple[_Org, _Waiter | None]: + c, w = _limits() + if work.weight > c: + raise VectorAdmissionError( + "vector statement exceeds the org work ceiling", reason="weight", retry_after_s=1 + ) + if self.population + work.weight > 2 * w: + raise VectorOverloadedError( + "this process is at its vector work ceiling", reason="population", retry_after_s=1 + ) + # No org entry, future or suspension exists before this reservation. + self.population += work.weight + org: _Org | None = None + waiter: _Waiter | None = None + try: + org = self.orgs.setdefault(work.org, _Org()) + self._advance(org) + if not org.queue and org.held + work.weight <= c: + org.held += work.weight + return org, None + if org.waiting + work.weight > w: + raise VectorAdmissionError( + "the org vector queue is full", reason="org_queue", retry_after_s=1 + ) + waiter = _Waiter(work.weight, asyncio.get_running_loop().create_future()) + org.queue.append(waiter) + org.waiting += work.weight + try: + async with asyncio.timeout(wait_s): + await waiter.ready + except TimeoutError: + raise VectorAdmissionError( + "the org vector wait expired", reason="org_deadline", retry_after_s=1 + ) from None + return org, waiter + except BaseException: + self.population -= work.weight + if org is not None: + if waiter is not None: + if waiter.granted: + org.held -= work.weight + elif waiter in org.queue: + org.queue.remove(waiter) + org.waiting -= work.weight + self._advance(org) + self._forget(work.org, org) + raise + + def _forget(self, name: str, org: _Org) -> None: + if not org.held and not org.queue: + self.orgs.pop(name, None) + + def leave(self, work: VectorWork, org: _Org) -> None: + org.held -= work.weight + self.population -= work.weight + self._advance(org) + self._forget(work.org, org) + + +_admission = _Admission() + + +@asynccontextmanager +async def admit(*, pool_ready: bool, wait_s: float) -> AsyncIterator[None]: + work = current_work() + if work is None: + yield + return + if not pool_ready or int(settings.ch_pool_maxsize) <= 0 or int(settings.ch_pool_wait_ms) <= 0: + raise VectorOverloadedError( + "the bounded ClickHouse gate is unavailable", reason="pool", retry_after_s=1 + ) + org, _ = await _admission.enter(work, wait_s) + try: + yield + finally: + _admission.leave(work, org) + + +def native_settings(out: dict[str, Any], *, timeout: float) -> dict[str, Any]: + if current_work() is None: + return out + caps = { + "max_execution_time": timeout, + "max_memory_usage": float(settings.ch_caller_shaped_max_memory_bytes), + "max_rows_to_read": float(settings.vector_scan_max_rows_to_read), + } + for key, cap in caps.items(): + if not math.isfinite(cap) or cap <= 0: + raise VectorAdmissionError( + "vector native limits are unavailable", reason="configuration" + ) + try: + previous = float(out.get(key, 0)) + except (ValueError, TypeError): + previous = 0 + bound = min(previous, cap) if math.isfinite(previous) and previous > 0 else cap + out[key] = int(bound) if bound.is_integer() else bound + out["read_overflow_mode"] = "throw" + return out + + +def raise_native(error: Exception) -> None: + if current_work() is None or isinstance(error, VectorCapacityError): + return + code = error_code(error) + if code is None: + return + reasons: dict[int, VectorCapacityReason] = { + 158: "rows", + 307: "rows", + 241: "memory", + 159: "deadline", + } + reason = reasons.get(code) + if reason: + raise VectorAdmissionError( + "the complete vector read exceeded its native budget", reason=reason, retry_after_s=1 + ) from error diff --git a/server/identity/app/namespace_catalog.py b/server/identity/app/namespace_catalog.py index da34c9c980..8336777041 100644 --- a/server/identity/app/namespace_catalog.py +++ b/server/identity/app/namespace_catalog.py @@ -10,6 +10,7 @@ from typing import Any from server.core.ch import get_client +from server.core.vectors.admission import vector_work from server.identity.api.models_namespace_catalog import ( NamespaceListCategoriesInput, NamespaceListCategoriesOutput, @@ -359,8 +360,9 @@ async def handle_namespace_search_categories( except Exception: # noqa: BLE001 — embed failure answers no categories return NamespaceSearchCategoriesOutput(results=[]) - rows = await get_client().query_all( - """ + with vector_work(input.org, input.namespace): + rows = await get_client().query_all( + """ SELECT category_id, name, @@ -375,13 +377,13 @@ async def handle_namespace_search_categories( ORDER BY distance ASC LIMIT {top_k:UInt32} """, - params={ - "org": input.org, - "ns": input.namespace, - "q": q_vec, - "top_k": int(input.top_k), - }, - ) + params={ + "org": input.org, + "ns": input.namespace, + "q": q_vec, + "top_k": int(input.top_k), + }, + ) return NamespaceSearchCategoriesOutput( results=[ _CategoryHit( diff --git a/server/knowledge/app/conversation_search.py b/server/knowledge/app/conversation_search.py index 7a5243dd8d..2f4028b6ba 100644 --- a/server/knowledge/app/conversation_search.py +++ b/server/knowledge/app/conversation_search.py @@ -14,6 +14,7 @@ from typing import Any, Protocol from server.core.repo_base import _default_client +from server.core.vectors.admission import vector_work from server.core.vectors.search_settings import vector_search_settings from server.identity.domain.read_scope import ReadFilters, scope_where from server.knowledge.domain.conversations import ConversationRole @@ -114,19 +115,20 @@ async def vector_search( # ASC, ts DESC`) disqualifies the query from CH's vector-search rewrite, so # the recency tie-break runs in Python below, where it costs nothing (top_k # rows, already distance-ordered by CH, and Python's sort is stable). - rows = await ch.query_all( - f""" - SELECT thread_id, msg_id, role, - content_excerpt AS embedded_text, ts, - cosineDistance(embedding, {{q:Array(Float32)}}) AS distance - FROM loom.conversation_embeddings FINAL - WHERE {" AND ".join(where)} - ORDER BY distance ASC - LIMIT {{lim:UInt32}} - {vector_search_settings(table="loom.conversation_embeddings")} - """, - params=params, - ) + with vector_work(q.org, q.namespace): + rows = await ch.query_all( + f""" + SELECT thread_id, msg_id, role, + content_excerpt AS embedded_text, ts, + cosineDistance(embedding, {{q:Array(Float32)}}) AS distance + FROM loom.conversation_embeddings FINAL + WHERE {" AND ".join(where)} + ORDER BY distance ASC + LIMIT {{lim:UInt32}} + {vector_search_settings(table="loom.conversation_embeddings")} + """, + params=params, + ) # The whole embedded text is read so the sensitivity check covers all of it. A turn that # reads as high gets no embedding row; this holds back a row an earlier build # wrote for one. diff --git a/server/knowledge/app/conversations.py b/server/knowledge/app/conversations.py index 1adffae439..34ffc0891b 100644 --- a/server/knowledge/app/conversations.py +++ b/server/knowledge/app/conversations.py @@ -17,6 +17,7 @@ from server.core.datetimes import naive_utc_now from server.core.errors import AuthInsufficientError from server.core.naming import SENTINEL, named +from server.core.vectors.private_first import each from server.embedding import get_embedder from server.identity.app.conversation_logging import set_conversation_logging from server.identity.app.request_scope import request_scope @@ -906,7 +907,6 @@ async def _search_conversations( distance with recency tiebreak. Falls back to substring search when the embedder is unavailable so the endpoint never 5xx's. """ - import asyncio from server.settings import settings @@ -947,7 +947,7 @@ async def _search_one(ns: str) -> tuple[str, list[ConversationHit]]: ) ) - results_per_ns = await asyncio.gather(*[_search_one(ns) for ns in namespaces]) + results_per_ns = await each(_search_one)(namespaces) # Carry similarity so the cross-namespace merge ranks by cosine, not # insertion order. scored: list[tuple[float, _ConversationHit]] = [] diff --git a/server/knowledge/app/repo_docs.py b/server/knowledge/app/repo_docs.py index b6019aee18..213cb7354a 100644 --- a/server/knowledge/app/repo_docs.py +++ b/server/knowledge/app/repo_docs.py @@ -6,8 +6,8 @@ Under a code scope the repository's documents join `loom_search`'s knowledge section, and `code.context_pack` returns them as its `docs` section. Every -read passes the gate for the current principal first and is bounded; a -failure answers nothing, never an error — the leg is additive. +read passes the gate for the current principal first and is bounded; an +ordinary failure answers nothing; capacity refusals propagate. """ from __future__ import annotations @@ -19,8 +19,9 @@ from contextlib import nullcontext from typing import TYPE_CHECKING, TypedDict -from server.core.background import gather_or_cancel from server.core.diagnostics import private_work +from server.core.errors import VectorCapacityError +from server.core.shielded import to_its_end from server.knowledge.api.models_knowledge import ( KnowledgeListInput, KnowledgeSearchInput, @@ -113,6 +114,8 @@ async def _renew_extra(self, *, ordinary: bool = False) -> None: await extra.ordinary() else: await extra.current() + except VectorCapacityError: + raise except Exception: # noqa: BLE001 — an additive renewal failure leaves the base search intact self._lost |= extra.lost self.extra = None @@ -129,7 +132,24 @@ def _drop_expired_extra(self) -> None: async def _ordinary(self) -> KnowledgeSearchOutput: self._lost = True - await gather_or_cancel(self.base.ordinary(), self._renew_extra(ordinary=True)) + tasks = [ + asyncio.create_task(self.base.ordinary()), + asyncio.create_task(self._renew_extra(ordinary=True)), + ] + try: + await asyncio.gather(*tasks) + except BaseException: + for task in tasks: + task.cancel() + + async def stopped() -> None: + await asyncio.gather(*tasks, return_exceptions=True) + + try: + await to_its_end(stopped(), name="knowledge.repo.ordinary.cleanup") + except asyncio.CancelledError: + pass # Preserve the first failure after both reads stop. + raise self._drop_expired_extra() return self.output @@ -224,6 +244,8 @@ async def search_repo_docs(request: RepoSearchRequest, plan: ReadPlan) -> Checke raise TimeoutError() async with asyncio.timeout_at(request.extra_deadline): return await run_checked_search(request.extra, plan) + except VectorCapacityError: + raise except Exception: # noqa: BLE001 — a failed additive search cannot expose private details log.info("repo docs search did not answer") return None @@ -235,9 +257,24 @@ async def run_repo_search(request: RepoSearchRequest, plan: ReadPlan) -> RepoSea if any(child.input.org != plan.org for child in (request.base, request.extra) if child): raise ValueError("document read scope changed") with private_work() if plan.marked else nullcontext(): - base, extra = await gather_or_cancel( - run_checked_search(request.base, plan), search_repo_docs(request, plan) - ) + tasks = [ + asyncio.create_task(run_checked_search(request.base, plan)), + asyncio.create_task(search_repo_docs(request, plan)), + ] + try: + base, extra = await asyncio.gather(*tasks) + except BaseException: + for task in tasks: + task.cancel() + + async def stopped() -> None: + await asyncio.gather(*tasks, return_exceptions=True) + + try: + await to_its_end(stopped(), name="knowledge.repo.search.cleanup") + except asyncio.CancelledError: + pass # Preserve the first failure after both searches stop. + raise assert base is not None return RepoSearch(request, base, extra) diff --git a/server/knowledge/app/search.py b/server/knowledge/app/search.py index 194c16500d..db723874dc 100644 --- a/server/knowledge/app/search.py +++ b/server/knowledge/app/search.py @@ -12,6 +12,7 @@ from server.core import bm25 from server.core.repo_base import _default_client +from server.core.vectors.admission import vector_work from server.core.vectors.search_settings import vector_search_settings from server.knowledge.domain.chunk_metadata import read_structure from server.knowledge.domain.live_chunks import live_chunk_sql @@ -199,7 +200,8 @@ async def hybrid_search( LIMIT {{top_k:UInt32}} {vector_search_settings(table="loom.chunks")} """ - rows = await ch.query_all(sql, params=params) + with vector_work(q.org, q.namespaces): + rows = await ch.query_all(sql, params=params) return [ ChunkHit( namespace=r["namespace"], @@ -342,7 +344,8 @@ async def lexical_search( LIMIT {{top_k:UInt32}} SETTINGS query_plan_direct_read_from_text_index = 0 """ - rows = await ch.query_all(sql, params=params) + with vector_work(q.org, q.namespaces): + rows = await ch.query_all(sql, params=params) return [ ChunkHit( namespace=r["namespace"], @@ -399,7 +402,8 @@ async def keys_search( ORDER BY length(matched) DESC, distance ASC LIMIT {{top_k:UInt32}} """ - rows = await ch.query_all(sql, params=params) + with vector_work(q.org, q.namespaces): + rows = await ch.query_all(sql, params=params) return [ ChunkHit( namespace=r["namespace"], diff --git a/server/knowledge/app/search_legs.py b/server/knowledge/app/search_legs.py index 02ab84a9cc..fbba03e495 100644 --- a/server/knowledge/app/search_legs.py +++ b/server/knowledge/app/search_legs.py @@ -2,11 +2,12 @@ from __future__ import annotations -import asyncio +from collections.abc import Awaitable from dataclasses import dataclass, replace -from typing import TYPE_CHECKING, Literal +from typing import TYPE_CHECKING, Any, Literal from server.core import ch, pg +from server.core.vectors.private_first import each from server.knowledge.app import bm25 as knowledge_bm25 from server.knowledge.app import pg_serving, search @@ -61,13 +62,17 @@ async def collect( lexical_weight: float, ) -> LegPools: """Preserve ordinary leg widths, cold-statistics behavior and fusion scales.""" + + async def _read(call: Awaitable[Any]) -> Any: + return await call + top_k = query.top_k if mode == "semantic" or not query.tokens: if not query.keys: semantic = await search.hybrid_search(query) return LegPools(query, {"semantic": semantic}, top_k, lexical_weight) wide = replace(query, top_k=max(top_k * 3, 20)) - semantic, keyed = await asyncio.gather(search.hybrid_search(wide), search.keys_search(wide)) + semantic, keyed = await each(_read)([search.hybrid_search(wide), search.keys_search(wide)]) return LegPools(wide, {"semantic": semantic, "keys": keyed}, top_k, lexical_weight) avgdl, idfs = await knowledge_bm25.lexical_stats( ch.get_client(), @@ -81,8 +86,8 @@ async def collect( lexical = await search.lexical_search(lexical_query) return LegPools(lexical_query, {"lexical": lexical}, top_k, lexical_weight) wide = replace(lexical_query, top_k=max(top_k * 3, 20)) - semantic, lexical, keyed = await asyncio.gather( - search.hybrid_search(wide), search.lexical_search(wide), search.keys_search(wide) + semantic, lexical, keyed = await each(_read)( + [search.hybrid_search(wide), search.lexical_search(wide), search.keys_search(wide)] ) return LegPools( wide, diff --git a/server/knowledge/app/source_search.py b/server/knowledge/app/source_search.py index d55bb33795..ad8e689fe2 100644 --- a/server/knowledge/app/source_search.py +++ b/server/knowledge/app/source_search.py @@ -2,13 +2,14 @@ from __future__ import annotations -import asyncio from dataclasses import dataclass, replace from typing import Any from server.core import pg, request_work from server.core.ch import sequential_reads, set_query_tag from server.core.diagnostics import private_work +from server.core.errors import VectorCapacityError +from server.core.vectors.private_first import each from server.knowledge.app.search import HybridSearchQuery, _common_where from server.knowledge.app.source_access import Admitted, ReadPlan from server.knowledge.domain.access_list import candidate_keys @@ -90,6 +91,8 @@ async def take(batch: list[dict[str, Any]]) -> None: break if batch and len(out) < limit: await take(batch) + except VectorCapacityError: + raise except Exception: # noqa: BLE001 — an unconditional discovery failure invalidates every mirror leg plan.failed = plan.unread = True request_work.mark_mirrored(unread=True) @@ -105,9 +108,11 @@ async def discover( ordinary: set[tuple[str, str]] | None = None, ) -> dict[store.Leg, list[Admitted]]: """Select all discovery queries from the request, independently of hidden matches.""" - pools = await asyncio.gather( - *(collect_leg(plan, query, leg, limit=limit, ordinary=ordinary) for leg in legs) - ) + + async def _leg(leg: store.Leg) -> list[Admitted]: + return await collect_leg(plan, query, leg, limit=limit, ordinary=ordinary) + + pools = await each(_leg)(list(legs)) if plan.failed or plan.unread: return {} return await recheck(plan, dict(zip(legs, pools, strict=True))) diff --git a/server/knowledge/store/source_candidates.py b/server/knowledge/store/source_candidates.py index 995ac6e08c..f371221587 100644 --- a/server/knowledge/store/source_candidates.py +++ b/server/knowledge/store/source_candidates.py @@ -9,6 +9,7 @@ from server.core import bm25 from server.core.ch import CHClient, get_client, set_query_tag +from server.core.vectors.admission import vector_work from server.knowledge.domain.live_chunks import live_chunk_sql Leg = Literal["semantic", "lexical", "keys"] @@ -176,6 +177,7 @@ async def candidates( sql: str, params: dict[str, Any], *, ch: CHClient | None = None ) -> AsyncIterator[AsyncIterator[dict[str, Any]]]: set_query_tag("knowledge.mirror.candidates") - async with (ch or get_client()).raw_stream(sql, params) as body: - async with aclosing(_rows(body)) as rows: - yield rows + with vector_work(str(params["org"]), list(params["nss"])): + async with (ch or get_client()).raw_stream(sql, params) as body: + async with aclosing(_rows(body)) as rows: + yield rows diff --git a/server/memory/app/ask/pool.py b/server/memory/app/ask/pool.py index a55ef1fda2..6ff0b85af8 100644 --- a/server/memory/app/ask/pool.py +++ b/server/memory/app/ask/pool.py @@ -10,7 +10,7 @@ from dataclasses import field as dc_field from typing import Any -from server.core.errors import VectorInputTooLargeError +from server.core.errors import VectorCapacityError from server.memory.app.ask.completion import ( _bridge_complete_memories, _facet_complete_memories, @@ -290,7 +290,7 @@ async def _widen_by_question( for _part in _parts[1:]: try: _sub = await handle_memory_search(_with(search_input, query=_part)) - except VectorInputTooLargeError: + except VectorCapacityError: raise except Exception: # noqa: BLE001 — additive continue @@ -348,7 +348,7 @@ async def _widen_by_question( ) ) _sub_hits = list(_sub.results or []) - except VectorInputTooLargeError: + except VectorCapacityError: raise except Exception: # noqa: BLE001 — additive continue diff --git a/server/memory/app/ask/retrieval_legs.py b/server/memory/app/ask/retrieval_legs.py index e36cb71545..58126a18b5 100644 --- a/server/memory/app/ask/retrieval_legs.py +++ b/server/memory/app/ask/retrieval_legs.py @@ -8,7 +8,7 @@ import re from typing import Any -from server.core.errors import VectorInputTooLargeError +from server.core.errors import VectorCapacityError from server.memory.app.ask.question_shapes import ( looks_multihop, ) @@ -235,8 +235,8 @@ async def decomposed_retrieval( hits into `ranked` (whole-question hits first, then per part in order, merged by row through `PooledRows`). Returns (sub_questions, merged). A part whose retrieval has an ordinary failure is skipped and counted in - `stats["failed"]`; typed input refusals propagate — the count rides the response and the - ledger, so a lever that stops working is a number, not a log line.""" + `stats["failed"]`; typed capacity refusals propagate. The count rides the + response and ledger, so a failed leg is counted rather than only logged.""" from server.memory.app.search import handle_memory_search if mode == "auto" and not looks_multihop(question): @@ -251,7 +251,7 @@ async def decomposed_retrieval( sub = await handle_memory_search( _with(search_input, query=part, top_k=max(1, int(part_top_k))) ) - except VectorInputTooLargeError: + except VectorCapacityError: raise except Exception as e: # noqa: BLE001 — additive logger.warning("memory.ask decompose: part retrieval failed (fail-open): %s", e) diff --git a/server/memory/app/bulk_set.py b/server/memory/app/bulk_set.py index c444dda88a..51787b70d7 100644 --- a/server/memory/app/bulk_set.py +++ b/server/memory/app/bulk_set.py @@ -16,9 +16,11 @@ from server.core.ch import get_client, sequential_reads from server.core.datetimes import naive_utc_now from server.core.error_summary import error_summary +from server.core.errors import VectorCapacityError from server.core.hashing import hash_text from server.core.hashing import sha256_value as _content_sha256 from server.core.vectors import codes_state +from server.core.vectors.private_first import each from server.embedding import get_embedder from server.identity.app.request_scope import request_scope from server.identity.domain.read_scope import scope_and @@ -80,6 +82,7 @@ PeerErased, heads_unchanged, ) +from server.observability import degradations from server.observability.budget_shape import note_budget_shape if TYPE_CHECKING: @@ -818,6 +821,19 @@ async def _bulk_post_insert_links( ) _n_value_links = len(_report.links) _value_links = list(_report.follow_links) + except VectorCapacityError as error: + reason = str(error.extra.get("reason", "input")) + log.warning("bulk_set: optional value-update linking refused: %s", reason) + degradations.note( + "value_updates", "versions_committed_linking_refused", reason, health=False + ) + affected = {(str(row["memory_key"]), int(row["version"])) for row in _live_rows} + for result in state.results: + if (result.memory_key, result.version) in affected: + result.next = ( + result.next + " Optional value-update linking was refused by vector " + "capacity; this version committed." + ).strip() except Exception as e: # noqa: BLE001 — never block an ingest log.warning("bulk_set: value-update linking failed (non-fatal): %s", e) last_value_updates_linked.set(_n_value_links) @@ -1475,6 +1491,8 @@ async def _batched_peers( batch=batch, user_id=input.user_id, ) + except VectorCapacityError: + raise except Exception as e: # noqa: BLE001 — each fact reads its own peers log.warning("bulk_set: batched candidate lookup failed: %s", e) return {} @@ -1561,11 +1579,17 @@ async def _one(idx: int, vec: list[float], candidate_rows: list[dict[str, Any]] ) except customer.HOLDS: raise # held, as the batch's own embeds would be: the write answers it + except VectorCapacityError: + raise except Exception as e: # noqa: BLE001 — the fact writes unchecked log.warning("bulk_set: conflict check failed for %r: %s", entry.memory_key, e) - jobs = [_one(idx, vec, prefetched.get(j)) for j, ((idx, *_), vec) in enumerate(pairs)] - await asyncio.gather(*jobs) + async def _check(item: tuple[int, int, list[float]]) -> None: + j, idx, vec = item + await _one(idx, vec, prefetched.get(j)) + + jobs = [(j, idx, vec) for j, ((idx, *_), vec) in enumerate(pairs)] + await each(_check)(jobs) return out diff --git a/server/memory/app/conflicts.py b/server/memory/app/conflicts.py index a02a8fcf38..c047021310 100644 --- a/server/memory/app/conflicts.py +++ b/server/memory/app/conflicts.py @@ -25,8 +25,10 @@ from typing import TYPE_CHECKING, Any from server.core.ch import get_client -from server.core.errors import AuthInsufficientError, IdentityIncompleteError +from server.core.errors import AuthInsufficientError, IdentityIncompleteError, VectorCapacityError from server.core.naming import is_anonymous_user +from server.core.vectors.admission import batch_size, vector_work +from server.core.vectors.private_first import each from server.identity.app.request_scope import request_narrowed, request_scope from server.identity.domain.read_scope import ( WHOLE_NAMESPACE, @@ -207,14 +209,13 @@ async def candidate_rows_many( chunk: a failed chunk's facts fall back to the per-fact statement in the caller by getting no entry here — callers treat a missing key as "fetch yourself", not as "no peers".""" - import asyncio ch = ch or get_client() scope = _writer_scope(user_id) if scope is None: return {} # every fact falls back to the per-fact path, which fails closed size = int(batch if batch is not None else settings.memory_write_candidate_batch) - size = max(size, 1) + size = batch_size(size) by_dim: dict[int, list[int]] = {} for i, (_mt, _k, emb, _md) in enumerate(items): if emb: @@ -231,18 +232,18 @@ async def _chunk(dim: int, idxs: list[int]) -> None: "keys": [items[i][1] for i in idxs], "max_dists": [float(items[i][3]) for i in idxs], } - candidates = ch.query_all( - candidate_sql_many(len(idxs), scope_and(scope, params, "m")), params=params - ) - # Pinned rows outside the writer's scope (another user's or a shared one) still - # block; the scope hides them. + + async def _read(sql: str) -> list[dict[str, Any]]: + with vector_work(org, namespace, len(idxs)): + return await ch.query_all(sql, params=params) + + sql = candidate_sql_many(len(idxs), scope_and(scope, params, "m")) + # Pinned peers retain their independent namespace-wide statement. if scope.narrowed: - rows, pinned = await asyncio.gather( - candidates, ch.query_all(pinned_sql_many(len(idxs)), params=params) - ) + rows, pinned = await each(_read)([sql, pinned_sql_many(len(idxs))]) rows = rows + [r for r in pinned if _foreign_pinned(r, scope)] else: - rows = await candidates + rows = await _read(sql) for i in idxs: out[i] = [] seen: set[tuple[int, str]] = set() @@ -252,15 +253,18 @@ async def _chunk(dim: int, idxs: list[int]) -> None: seen.add((q, str(r.get("memory_id")))) out[idxs[q]].append(dict(r)) + async def _one_chunk(item: tuple[int, list[int]]) -> None: + try: + await _chunk(*item) + except VectorCapacityError: + raise + except Exception as error: # noqa: BLE001 — ordinary failures retain per-fact fallback + log.warning("batched conflict candidates failed (per-fact fallback): %s", error) + jobs = [ - _chunk(dim, idxs[k : k + size]) - for dim, idxs in by_dim.items() - for k in range(0, len(idxs), size) + (dim, idxs[k : k + size]) for dim, idxs in by_dim.items() for k in range(0, len(idxs), size) ] - results = await asyncio.gather(*jobs, return_exceptions=True) - for res in results: - if isinstance(res, Exception): - log.warning("batched conflict candidates failed (per-fact fallback): %s", res) + await each(_one_chunk)(jobs) return out @@ -317,7 +321,8 @@ async def nearest_peer_rows( in_scope = scope_and(scope, params) k = _PEER_FIRST_K while True: - cand = await ch.query_all(_peer_candidates_sql(in_scope), params={**params, "k": k}) + with vector_work(str(params["org"]), str(params["ns"])): + cand = await ch.query_all(_peer_candidates_sql(in_scope), params={**params, "k": k}) inside = [c for c in cand if float(c["dist"]) <= float(params["max_dist"])] live: dict[tuple[str, str, str, str], dict[str, Any]] = {} if inside: @@ -353,14 +358,15 @@ async def _peer_rows_joined( has read `_PEER_MAX_K` candidates without enough live peers. Filtered to equal-length vectors so cosineDistance never sees mixed embedders (SIZES_OF_ARRAYS).""" - rows: list[dict[str, Any]] = await ch.query_all( - """ + with vector_work(str(params["org"]), str(params["ns"])): + rows: list[dict[str, Any]] = await ch.query_all( + """ SELECT m.memory_id, m.memory_key, m.value, m.pinned, m.agent, m.user_id, m.session_id, m.run_id, m.valid_at, """ - + "cosineDistance(e.embedding, {emb:Array(Float32)})" - + """ AS dist + + "cosineDistance(e.embedding, {emb:Array(Float32)})" + + """ AS dist FROM loom.memory m FINAL JOIN loom.memory_embeddings e FINAL ON e.org = m.org AND e.namespace = m.namespace @@ -376,14 +382,14 @@ async def _peer_rows_joined( AND e.tombstoned_at IS NULL AND e.embedding_dim = {dim:UInt16} AND length(e.embedding) = {dim:UInt16}""" - + in_scope - + f""" - HAVING dist <= {{max_dist:Float32}} - ORDER BY dist ASC - LIMIT {int(limit)} - """, - params=params, - ) + + in_scope + + f""" + HAVING dist <= {{max_dist:Float32}} + ORDER BY dist ASC + LIMIT {int(limit)} + """, + params=params, + ) return rows @@ -421,7 +427,8 @@ async def _foreign_pinned_rows( `row_in_read_scope` test the same user and session), so no row appears in both.""" if scope is not None and not scope.narrowed: return [] - pinned = await ch.query_all(PINNED_PEERS_SQL, params=params) + with vector_work(str(params["org"]), str(params["ns"])): + pinned = await ch.query_all(PINNED_PEERS_SQL, params=params) return [dict(r) for r in pinned if _foreign_pinned(r, scope)] @@ -433,9 +440,13 @@ async def _scoped_and_pinned_peers( the common adapter write, needs both.""" if scope is None: return await _foreign_pinned_rows(ch, params, scope) - rows, pinned = await asyncio.gather( - nearest_peer_rows(ch, params, scope), _foreign_pinned_rows(ch, params, scope) - ) + + async def _read(pinned: bool) -> list[dict[str, Any]]: + if pinned: + return await _foreign_pinned_rows(ch, params, scope) + return await nearest_peer_rows(ch, params, scope) + + rows, pinned = await each(_read)([False, True]) return rows + pinned diff --git a/server/memory/app/loom_search/fanout.py b/server/memory/app/loom_search/fanout.py index 0f84b0aa3c..0a4238e4ff 100644 --- a/server/memory/app/loom_search/fanout.py +++ b/server/memory/app/loom_search/fanout.py @@ -6,7 +6,7 @@ sections are assembled into one payload and trimmed by the answer-first rules (answer_first.py). An ordinary leg failure returns `{"error": ..., "results": []}` under its section and the others still -answer. Typed input refusals propagate instead of producing partial results. +answer. Typed capacity refusals propagate instead of producing partial results. The MCP `loom_search` tool renders this payload as its text and lean result; the `search` operation returns it as is. """ @@ -22,7 +22,8 @@ from server.core import request_tail from server.core.ch import set_query_tag from server.core.diagnostics import private_work -from server.core.errors import LoomServerError, VectorInputTooLargeError, caller_text +from server.core.errors import LoomServerError, VectorCapacityError, caller_text +from server.core.shielded import to_its_end from server.identity.app.request_scope import request_scope from server.identity.domain.read_scope import ReadFilters from server.knowledge.app import repo_docs, source_publication @@ -319,6 +320,57 @@ async def _published_knowledge( return section +async def _section_reads( + memory: Awaitable[Any], + knowledge: Awaitable[Any], + profile: Awaitable[Any], + conversations: Awaitable[Any], + tracker: Awaitable[Any], +) -> tuple[Any, Any, Any, Any, Any]: + """Join the fixed five sections, keeping a refusal ahead of empty-section fallback.""" + + async def section(call: Awaitable[Any]) -> Any: + try: + return await call + except VectorCapacityError: + raise + except Exception as error: # noqa: BLE001 — ordinary leg failures become empty sections + return error + + tasks = [ + asyncio.create_task(section(call)) + for call in (memory, knowledge, profile, conversations, tracker) + ] + try: + mem_result, know_result, profile_rows, conv_result, t_section = await asyncio.gather(*tasks) + except BaseException: + for task in tasks: + task.cancel() + + async def stopped() -> None: + await asyncio.gather(*tasks, return_exceptions=True) + + try: + await to_its_end(stopped(), name="loom.search.sections.cleanup") + except asyncio.CancelledError: + pass # Preserve the first failure after every owned section stops. + raise + + fatal = next( + ( + result + for result in (mem_result, know_result, profile_rows, conv_result, t_section) + if isinstance(result, VectorCapacityError) + or (isinstance(result, BaseException) and not isinstance(result, Exception)) + ), + None, + ) + if fatal is not None: + raise fatal + + return mem_result, know_result, profile_rows, conv_result, t_section + + async def search_sections( *, org: str, @@ -389,30 +441,16 @@ async def search_sections( prepared, plan = await _prepare_knowledge(know_input, repo_namespace) with private_work() if plan.marked else nullcontext(): - mem_result, know_result, profile_rows, conv_result, t_section = await asyncio.gather( - # A later page continues the home search alone; the knowledge leg re-runs on - # every page and keeps the repository's documents. + mem_result, know_result, profile_rows, conv_result, t_section = await _section_reads( + # Later pages continue the home search alone; repository docs remain. memory_with_repo_decisions(mem_input, repo_namespace="" if page else repo_namespace), _run_knowledge(prepared, plan), - search_profile( # By words, then by meaning + search_profile( org, namespaces, query, top_k=top_k, lexical=profile_hit_matches, scope=scope ), _conversation_leg(conv_input, scope), tracker_leg(query, plan=tracker_plan, top_k=top_k), - return_exceptions=True, - ) - - fatal = next( - ( - result - for result in (mem_result, know_result, profile_rows, conv_result, t_section) - if isinstance(result, VectorInputTooLargeError) - or (isinstance(result, BaseException) and not isinstance(result, Exception)) - ), - None, ) - if fatal is not None: - raise fatal # The QA verdict leads when the question is a defect report. payload: dict[str, Any] = { @@ -441,6 +479,8 @@ async def search_sections( if not isinstance(know_result, Exception): try: payload["knowledge"] = await _published_knowledge(know_result, plan, agent) + except VectorCapacityError: + raise except Exception as error: # noqa: BLE001 — a failed knowledge renewal leaves other sections available know_result = error if isinstance(know_result, Exception): diff --git a/server/memory/app/loom_search/repo_decisions.py b/server/memory/app/loom_search/repo_decisions.py index 84567bcfdd..33cab5e009 100644 --- a/server/memory/app/loom_search/repo_decisions.py +++ b/server/memory/app/loom_search/repo_decisions.py @@ -15,7 +15,7 @@ import math from typing import Any -from server.core.errors import VectorInputTooLargeError +from server.core.errors import VectorCapacityError from server.core.shielded import to_its_end from server.memory.api.models_search import MemorySearchInput, MemorySearchOutput, _MemorySearchHit from server.memory.domain import code_change @@ -29,7 +29,7 @@ async def search_repo_decisions( ) -> tuple[MemorySearchOutput, MemorySearchInput] | None: """The decision memories in `repo_namespace` that match `mem_input`'s query, with the input the search settled on; None when there is nothing to - add, when the search abstains, or on ordinary backend failures. Typed input + add, when the search abstains, or on ordinary backend failures. Typed capacity refusals propagate. The token read scope is pinned as for the base search.""" from server.memory.app.search import handle_memory_search from server.memory.app.search.scoring import one_per_key, per_key_fetch @@ -75,9 +75,9 @@ async def search_repo_decisions( out = await asyncio.wait_for( handle_memory_search(decision_input), timeout=settings.memory_repo_decisions_timeout_s ) - except VectorInputTooLargeError: + except VectorCapacityError: raise - except Exception as e: # noqa: BLE001 — an additive leg never fails the caller + except Exception as e: # noqa: BLE001 — ordinary additive-leg failures leave the base available log.info( "repo decisions search on %s/%s did not answer: %s", mem_input.org, repo_namespace, e ) @@ -146,7 +146,7 @@ async def memory_with_repo_decisions( ) -> MemorySearchOutput: """`loom_search`'s memory leg: the memory search, and — when the code scope names a repository the caller may read (`repo_namespace`) — its - decisions, merged into the one section. Base failures and typed input + decisions, merged into the one section. Base failures and typed capacity refusals from either read propagate.""" from server.memory.app.search import handle_memory_search from server.memory.app.search.request import apply_explain_default diff --git a/server/memory/app/loom_search/tracker_leg.py b/server/memory/app/loom_search/tracker_leg.py index 914c4d7a05..66f887a590 100644 --- a/server/memory/app/loom_search/tracker_leg.py +++ b/server/memory/app/loom_search/tracker_leg.py @@ -13,6 +13,8 @@ from dataclasses import dataclass from typing import TYPE_CHECKING, Any +from server.core.errors import VectorCapacityError + if TYPE_CHECKING: from server.tracker.app.dupcheck import DuplicateCheckPlan @@ -222,7 +224,9 @@ async def tracker_leg( if out.tracker_unavailable: return None return lean_tracker_section(out, defect_report_shape(query) or {}) - except Exception as e: # noqa: BLE001 — fail open: the memory legs answer + except VectorCapacityError: + raise + except Exception as e: # noqa: BLE001 — ordinary tracker failures leave the memory legs available logger.info("loom_search tracker leg skipped on %s: %s", plan.org, type(e).__name__) return None @@ -254,6 +258,8 @@ async def _similar_section( results: list[dict[str, Any]] = [] served: list[str] = [] for namespace, output in zip(plan.namespaces, outputs, strict=True): + if isinstance(output, VectorCapacityError): + raise output if isinstance(output, BaseException) or output.tracker_unavailable: continue served.append(namespace) diff --git a/server/memory/app/profile_search.py b/server/memory/app/profile_search.py index 1541552fab..39718d71a2 100644 --- a/server/memory/app/profile_search.py +++ b/server/memory/app/profile_search.py @@ -34,6 +34,7 @@ from collections.abc import Callable from typing import Any +from server.core.vectors.admission import vector_work from server.identity.domain.read_scope import ReadFilters, scope_and from server.settings import settings @@ -230,16 +231,17 @@ async def _similarities( log.debug("profile leg: query embed failed: %s", e) return {} by_ident = {_ident(r): r for r in rows} - vecs = await ch.query_all( - PROFILE_VECTORS_SQL, - params={ - "org": org, - "keys": list(by_ident), - "model": embedder.model_name, - "dim": len(qv), - "qv": [float(x) for x in qv], - }, - ) + with vector_work(org, tuple({str(r["namespace"]) for r in rows})): + vecs = await ch.query_all( + PROFILE_VECTORS_SQL, + params={ + "org": org, + "keys": list(by_ident), + "model": embedder.model_name, + "dim": len(qv), + "qv": [float(x) for x in qv], + }, + ) sims: dict[Ident, float] = {} settled: set[Ident] = set() # fresh, or a re-embed already queued scope: dict[Ident, str] = {} diff --git a/server/memory/app/retrieval/chunks.py b/server/memory/app/retrieval/chunks.py index d9ba90febd..35395f590f 100644 --- a/server/memory/app/retrieval/chunks.py +++ b/server/memory/app/retrieval/chunks.py @@ -9,6 +9,8 @@ import re from typing import Any +from server.core.errors import VectorCapacityError +from server.core.vectors.admission import vector_work from server.core.vectors.search_settings import vector_search_settings from server.memory.app.retrieval.retry import _call_retrying_empty_block from server.memory.domain.passage_trim import trim_chunk_excerpt @@ -714,7 +716,10 @@ async def _search_memory_chunks( } try: stage1_sql = _chunk_candidates_sql(chunk_agent_pred) - candidates = await _chunk_query(ch, stage1_sql, stage1_params) + with vector_work(q.org, q.namespace): + candidates = await _chunk_query(ch, stage1_sql, stage1_params) + except VectorCapacityError: + raise except Exception: log.exception( "chunk plane query failed; degrading to fact-only retrieval " @@ -783,6 +788,8 @@ async def _search_memory_chunks( ) try: rows = await _chunk_query(ch, sql, chunk_params) + except VectorCapacityError: + raise except Exception: # Degrade to fact-only retrieval rather than 500'ing the query, but # log it — a silently-disabled chunk plane must leave a trace. diff --git a/server/memory/app/retrieval/derived.py b/server/memory/app/retrieval/derived.py index 15dcc492fd..9185c33c49 100644 --- a/server/memory/app/retrieval/derived.py +++ b/server/memory/app/retrieval/derived.py @@ -11,6 +11,8 @@ from datetime import datetime from typing import Any +from server.core.errors import VectorCapacityError +from server.core.vectors.admission import vector_work from server.core.vectors.search_settings import vector_search_settings from server.memory.app.search_llm import expand_verb_synonyms from server.memory.domain.search_parsing import _iso_date, expand_predicate_class @@ -276,18 +278,21 @@ async def _resolve_operand_event_date( {vector_search_settings(table="loom.memory_embeddings")} """ try: - rows = await ch.query_all( - sql, - params={ - "q": operand_vec, - "org": org, - "ns": namespace, - "agent": agent, - **({"run_id": run_id} if run_id else {}), - "uid": user_id or "-", - "model": embedding_model, - }, - ) + with vector_work(org, namespace): + rows = await ch.query_all( + sql, + params={ + "q": operand_vec, + "org": org, + "ns": namespace, + "agent": agent, + **({"run_id": run_id} if run_id else {}), + "uid": user_id or "-", + "model": embedding_model, + }, + ) + except VectorCapacityError: + raise except Exception as e: # noqa: BLE001 — a failed operand read means no date answer log.warning("_resolve_operand_event_date failed (fail-open): %s", e) return None @@ -411,7 +416,10 @@ async def fetch_current_state_observation( {vector_search_settings(table="loom.memory_embeddings")} """ try: - rows = await ch.query_all(sql, params=params) + with vector_work(org, namespace): + rows = await ch.query_all(sql, params=params) + except VectorCapacityError: + raise except Exception as e: # noqa: BLE001 — the current-state leg is additive; fail open log.warning("fetch_current_state_observation failed (fail-open): %s", e) return [] diff --git a/server/memory/app/retrieval/fact_sql.py b/server/memory/app/retrieval/fact_sql.py index d0d5cd4bdd..17e33b6047 100644 --- a/server/memory/app/retrieval/fact_sql.py +++ b/server/memory/app/retrieval/fact_sql.py @@ -20,7 +20,7 @@ def _main_leg_threads_sql(settings_obj: Any) -> str: return f"max_threads = {pin},\n " if pin > 0 else "" -async def _fact_distance_expr(q: SearchQuery, ch: _CHClient, settings_obj: Any) -> str: +async def _fact_distance_expr(q: SearchQuery, ch: _CHClient, settings_obj: Any) -> tuple[str, int]: """Phase 6's vector fragment: the `distance` expression. Resolves the key axis for this read.""" # Key-embedding axis. Referencing `memory_key_embedding` @@ -30,7 +30,7 @@ async def _fact_distance_expr(q: SearchQuery, ch: _CHClient, settings_obj: Any) # is the bare value distance and the column is never named in the query. key_axis_on = await resolve_key_axis(q, ch=ch, settings_obj=settings_obj) if not key_axis_on: - return "cosineDistance(e.embedding, {q:Array(Float32)})" + return "cosineDistance(e.embedding, {q:Array(Float32)})", 1 return ( "least(" "cosineDistance(e.embedding, {q:Array(Float32)}), " @@ -39,12 +39,12 @@ async def _fact_distance_expr(q: SearchQuery, ch: _CHClient, settings_obj: Any) " AND length(e.memory_key_embedding) = length({q:Array(Float32)})," " e.memory_key_embedding, e.embedding), {q:Array(Float32)})" ")" - ) + ), 2 async def _fact_leg_sql( q: SearchQuery, ch: _CHClient, where: list[str], params: dict[str, Any], t: _FactRankTerms -) -> str: +) -> tuple[str, int]: """Phase 6 of `vector_search`: the fact leg's statement. Binds `internal_limit` into `params`.""" # Over-fetch 4× top_k (floor 60) so weak semantic candidates can rerank up @@ -89,7 +89,7 @@ async def _fact_leg_sql( where_sql = " WHERE " + " AND ".join(rest_clauses) if rest_clauses else "" # use_query_cache is deliberately NOT set — it breaks write-then-read. - distance_expr = await _fact_distance_expr(q, ch, _vs_settings) + distance_expr, weight = await _fact_distance_expr(q, ch, _vs_settings) select_from = f""" SELECT e.memory_key AS memory_key, @@ -167,4 +167,4 @@ async def _fact_leg_sql( query_plan_join_swap_table = 'auto', optimize_read_in_order = 1 """ - return sql + return sql, weight diff --git a/server/memory/app/retrieval/pool.py b/server/memory/app/retrieval/pool.py index f1799fd817..869060fe98 100644 --- a/server/memory/app/retrieval/pool.py +++ b/server/memory/app/retrieval/pool.py @@ -11,7 +11,9 @@ from typing import Any, NamedTuple from server.core.ch_params import fitting +from server.core.errors import VectorCapacityError from server.core.repo_base import query_all_sliced +from server.core.vectors.admission import vector_work from server.memory.app.retrieval.side_legs import exclude_pooled from server.memory.app.set_chunks import set_chunk_identities from server.memory.domain.search_parsing import _iso_date @@ -83,7 +85,10 @@ async def _recency_backfill( LIMIT {{newest_limit:UInt32}} """ try: - rows = await ch.query_all(sql, params=params) + with vector_work(q.org, q.namespace): + rows = await ch.query_all(sql, params=params) + except VectorCapacityError: + raise except Exception as e: # noqa: BLE001 — an additive leg fails open log.warning("recency backfill failed (fail-open): %s", e) return hits diff --git a/server/memory/app/retrieval/retry.py b/server/memory/app/retrieval/retry.py index 32a9a59c7e..02f6fb32b9 100644 --- a/server/memory/app/retrieval/retry.py +++ b/server/memory/app/retrieval/retry.py @@ -10,6 +10,8 @@ from collections.abc import Awaitable, Callable from typing import Any +from server.core.errors import VectorCapacityError +from server.core.vectors.admission import vector_work from server.memory.domain.search_types import _CHClient from server.settings import settings @@ -41,10 +43,13 @@ async def _spanned(name: str, coro: Any) -> Any: async def _query_all_retrying_empty_block( - ch: _CHClient, sql: str, params: dict[str, Any] + ch: _CHClient, sql: str, params: dict[str, Any], *, weight: int ) -> list[dict[str, Any]]: """The vector leg's statement under the fresh-part retry.""" - return await _call_retrying_empty_block(lambda: ch.query_all(sql, params=params), leg="vector") + with vector_work(str(params["org"]), str(params["ns"]), weight): + return await _call_retrying_empty_block( + lambda: ch.query_all(sql, params=params), leg="vector" + ) async def _call_retrying_empty_block[T]( @@ -62,6 +67,8 @@ async def _call_retrying_empty_block[T]( while True: try: return await factory() + except VectorCapacityError: + raise except Exception as e: if ( not _is_empty_block_column_error(e) diff --git a/server/memory/app/retrieval/search.py b/server/memory/app/retrieval/search.py index fb8e6876f6..87a77ae2ac 100644 --- a/server/memory/app/retrieval/search.py +++ b/server/memory/app/retrieval/search.py @@ -5,12 +5,13 @@ from __future__ import annotations -import asyncio import logging from collections.abc import Awaitable, Sequence from typing import Any from server.core.repo_base import _default_client +from server.core.vectors.admission import vector_work +from server.core.vectors.private_first import each from server.memory.app.retrieval.chunks import _search_memory_chunks from server.memory.app.retrieval.fact_rank import ( _fact_boost_terms, @@ -137,13 +138,13 @@ def _apply_plane_mask(q: SearchQuery) -> SearchQuery: def _fact_rows_task( - ch: _CHClient, q: SearchQuery, params: dict[str, Any], fact_sql: str + ch: _CHClient, q: SearchQuery, params: dict[str, Any], fact_sql: str, weight: int ) -> Awaitable[list[dict[str, Any]]]: """Phase 7 of `vector_search`: the fact leg's read as one awaitable, with the reinforcement fold riding inside.""" fact_rows_task = _spanned( "ch.query.memory_embeddings.ann", - _query_all_retrying_empty_block(ch, fact_sql, params), + _query_all_retrying_empty_block(ch, fact_sql, params, weight=weight), ) # The usage fold's reinforcement lives in a side table; @@ -212,7 +213,11 @@ async def _gather_legs( else None ) optional_tasks = [t for t in (chunks_task, lexical_task, records_task) if t is not None] - gathered = await asyncio.gather(rows_task, *optional_tasks) + + async def _leg(call: Awaitable[Any]) -> Any: + return await call + + gathered = await each(_leg)([rows_task, *optional_tasks]) rows = gathered[0] chunk_hits, lexical_pool, record_hits = _split_gathered( gathered, chunks_task is not None, lexical_task is not None, records_task is not None @@ -412,8 +417,8 @@ async def vector_search( _fact_rank_tail(q, params, t) - fact_sql = await _fact_leg_sql(q, ch, where, params, t) - rows_task = _fact_rows_task(ch, q, params, fact_sql) + fact_sql, weight = await _fact_leg_sql(q, ch, where, params, t) + rows_task = _fact_rows_task(ch, q, params, fact_sql, weight) rows, chunk_hits, lexical_pool, record_hits = await _gather_legs(ch, q, params, rows_task) hits = _fact_rows_to_hits(rows, q, t.lexical_score_expr) @@ -465,7 +470,8 @@ async def vector_search_trace( LIMIT {{top_k:UInt32}} SETTINGS query_plan_direct_read_from_text_index = 0 """ - rows = await ch.query_all(sql, params=params) + with vector_work(q.org, q.namespace): + rows = await ch.query_all(sql, params=params) return [ SearchTraceHit( memory_key=r["memory_key"], diff --git a/server/memory/app/retrieval/side_legs.py b/server/memory/app/retrieval/side_legs.py index 9fa05ce57e..17d43866a6 100644 --- a/server/memory/app/retrieval/side_legs.py +++ b/server/memory/app/retrieval/side_legs.py @@ -10,6 +10,8 @@ from typing import Any from server.core.ch_params import fitting +from server.core.errors import VectorCapacityError +from server.core.vectors.admission import vector_work from server.memory.app.retrieval.retry import _call_retrying_empty_block from server.memory.domain.search_parsing import _iso_date, _record_excerpt, _record_query_tokens from server.memory.domain.search_sql import _build_where @@ -134,9 +136,12 @@ async def _lexical_recall_fallback( SETTINGS query_plan_direct_read_from_text_index = 0 """ try: - rows = await _call_retrying_empty_block( - lambda: ch.query_all(sql, params=params), leg="lexical" - ) + with vector_work(q.org, q.namespace): + rows = await _call_retrying_empty_block( + lambda: ch.query_all(sql, params=params), leg="lexical" + ) + except VectorCapacityError: + raise except Exception as e: # noqa: BLE001 — the lexical fallback leg is additive; fail open log.warning("_lexical_recall_fallback failed (fail-open): %s", e) return [] @@ -465,7 +470,10 @@ async def _fetch_unmatched_candidates( LIMIT {{unmatched_limit:UInt32}} """ try: - rows = await ch.query_all(sql, params=params) + with vector_work(q.org, q.namespace): + rows = await ch.query_all(sql, params=params) + except VectorCapacityError: + raise except Exception as e: # noqa: BLE001 — the unmatched-candidate leg is additive; fail open log.warning("_fetch_unmatched_candidates failed (fail-open): %s", e) return [] diff --git a/server/memory/app/retrieval/summaries.py b/server/memory/app/retrieval/summaries.py index 3dfd5b15c0..8be0112386 100644 --- a/server/memory/app/retrieval/summaries.py +++ b/server/memory/app/retrieval/summaries.py @@ -8,6 +8,8 @@ import logging from typing import Any +from server.core.errors import VectorCapacityError +from server.core.vectors.admission import vector_work from server.core.vectors.search_settings import vector_search_settings from server.memory.domain.search_parsing import _iso_date from server.memory.domain.search_types import SearchHit @@ -197,7 +199,10 @@ async def fetch_session_summaries( wildcard = not agent or agent in ("*", "-") params["lim"] = int(n) * (_SESSION_SUMMARY_OVERFETCH if wildcard else 1) try: - rows = await ch.query_all(sql, params=params) + with vector_work(org, namespace): + rows = await ch.query_all(sql, params=params) + except VectorCapacityError: + raise except Exception as e: # noqa: BLE001 — the session-summary leg is additive; fail open log.warning("fetch_session_summaries failed (fail-open): %s", e) return [] @@ -353,7 +358,10 @@ async def fetch_entity_summaries( {vector_search_settings(table="loom.entity_summaries", column="summary_embedding")} """ try: - rows = await ch.query_all(sql, params=params) + with vector_work(org, namespace): + rows = await ch.query_all(sql, params=params) + except VectorCapacityError: + raise except Exception as e: # noqa: BLE001 — the entity-summary leg is additive; fail open log.warning("fetch_entity_summaries failed (fail-open): %s", e) return [] diff --git a/server/memory/app/search/anchor_pool.py b/server/memory/app/search/anchor_pool.py index 6bf20316cf..4520caad19 100644 --- a/server/memory/app/search/anchor_pool.py +++ b/server/memory/app/search/anchor_pool.py @@ -6,6 +6,8 @@ import logging from typing import Any +from server.core.errors import VectorCapacityError +from server.core.vectors.admission import vector_work from server.core.vectors.private_first import each from server.memory.api.models_search import _MemorySearchHit from server.memory.app.search.context import SearchContext @@ -118,7 +120,10 @@ async def _anchor_rows_for_ns(ns: str) -> tuple[str, list[dict[str, Any]]]: anchor_sql = anchor_pool_sql(where_sql) params["qv"] = ctx.query_vec try: - return ns, await _ch_a.query_all(anchor_sql, params=params) + with vector_work(input.org, ns): + return ns, await _ch_a.query_all(anchor_sql, params=params) + except VectorCapacityError: + raise except Exception as e: # noqa: BLE001 — one namespace's anchor pool failure must not fail search log.warning("anchor pool fetch failed for ns=%s (non-fatal): %s", ns, e) return ns, [] @@ -159,5 +164,7 @@ async def _anchor_rows_for_ns(ns: str) -> tuple[str, list[dict[str, Any]]]: ) hit._row_identity = row_identity(row) ctx.all_hits.append(_stamp_row_owner(hit, row.get("agent"), row.get("scope"))) + except VectorCapacityError: + raise except Exception as e: # noqa: BLE001 — the anchor pool is an optional recall stage log.warning("anchor-pool stage failed (non-fatal): %s", e) diff --git a/server/memory/app/search/ann_entity_boosts.py b/server/memory/app/search/ann_entity_boosts.py index 731631fe39..91331db21c 100644 --- a/server/memory/app/search/ann_entity_boosts.py +++ b/server/memory/app/search/ann_entity_boosts.py @@ -6,6 +6,8 @@ from typing import Any from server.core.ch import get_client +from server.core.errors import VectorCapacityError +from server.core.vectors.admission import vector_work from server.core.vectors.private_first import each from server.core.vectors.search_settings import vector_search_settings from server.observability.otel import traced @@ -98,16 +100,19 @@ async def _one_entity(entity: str) -> list[tuple[str, str, float]]: entities=len(query_entities), namespaces=len(namespaces), ): - rows = await ch.query_all( - sql, - params={ - "q": emb, - "org": org, - "namespaces": list(namespaces), - "agent": agent, - "uid": user_id or "-", - }, - ) + with vector_work(org, namespaces, 2): + rows = await ch.query_all( + sql, + params={ + "q": emb, + "org": org, + "namespaces": list(namespaces), + "agent": agent, + "uid": user_id or "-", + }, + ) + except VectorCapacityError: + raise except Exception as e: # noqa: BLE001 — entity boost is optional; search runs without it log.warning("entity_vectors lookup failed for %r: %s", entity, e) return [] diff --git a/server/memory/app/search/assemble.py b/server/memory/app/search/assemble.py index b6ae748bf9..4f89740d6d 100644 --- a/server/memory/app/search/assemble.py +++ b/server/memory/app/search/assemble.py @@ -7,6 +7,7 @@ import logging from typing import Any +from server.core.errors import VectorCapacityError from server.memory.api.models_search import _MemorySearchHit from server.memory.app.search.anchor_pool import _enforce_agent_scope from server.memory.app.search.context import SearchContext @@ -164,6 +165,8 @@ def _summary_hit_to_wire(ns: str, h: Any) -> _MemorySearchHit: try: for ns, sh in await summary_task: ctx.all_hits.append(_summary_hit_to_wire(ns, sh)) + except VectorCapacityError: + raise except Exception as e: # noqa: BLE001 log.warning("session-summary leg failed (non-fatal): %s", e) # Fold the PRE-BUILT entity summaries in as additive dense rows (source="entity_summary"; @@ -173,12 +176,16 @@ def _summary_hit_to_wire(ns: str, h: Any) -> _MemorySearchHit: try: for ns, eh in await entity_summary_task: ctx.all_hits.append(_summary_hit_to_wire(ns, eh)) + except VectorCapacityError: + raise except Exception as e: # noqa: BLE001 log.warning("entity-summary leg failed (non-fatal): %s", e) if observation_task is not None: try: for ns, oh in await observation_task: ctx.all_hits.append(_summary_hit_to_wire(ns, oh)) + except VectorCapacityError: + raise except Exception as e: # noqa: BLE001 log.warning("observation leg failed (non-fatal): %s", e) # Digest-anchored expansion: the sessions behind the top-m digests contribute their own diff --git a/server/memory/app/search/computed.py b/server/memory/app/search/computed.py index ae888cc8b2..a2a0d07bf4 100644 --- a/server/memory/app/search/computed.py +++ b/server/memory/app/search/computed.py @@ -10,7 +10,10 @@ from typing import TYPE_CHECKING, Any from server.core.ch import get_client -from server.core.errors import VectorInputTooLargeError +from server.core.errors import ( + VectorCapacityError, + VectorInputTooLargeError, +) from server.memory.api.models_search import MemorySearchInput, _MemorySearchHit from server.memory.app.search.budget import SearchBudget from server.memory.app.search.context import SearchContext @@ -180,6 +183,8 @@ async def _maybe_temporal_diff(ctx: SearchContext) -> dict[str, Any] | None: to_observation=to_observation, run_id=str(getattr(input, "run_id", "") or ""), ) + except VectorCapacityError: + raise except Exception as e: # noqa: BLE001 log.warning("temporal diff leg failed (non-fatal): %s", e) cache["v"] = result diff --git a/server/memory/app/search/consistency.py b/server/memory/app/search/consistency.py index 29dd119c7a..5937d6decb 100644 --- a/server/memory/app/search/consistency.py +++ b/server/memory/app/search/consistency.py @@ -13,7 +13,7 @@ from server.core.ch import sequential_reads, sequential_reads_active from server.core.datetimes import naive_utc_now -from server.core.errors import VectorInputTooLargeError +from server.core.errors import VectorCapacityError from server.memory.api.models_search import MemorySearchInput, MemorySearchOutput, _MemorySearchHit from server.memory.app.recent_writes import written_recently from server.memory.app.search.context import SearchContext @@ -183,7 +183,7 @@ async def _consistency_reread( ) except TimeoutError: log.warning("memory.search: fresh-hit sequential re-read exceeded the wall budget") - except VectorInputTooLargeError: + except VectorCapacityError: raise except Exception as e: # noqa: BLE001 — an optional re-read; the first pass stands log.warning("memory.search: fresh-hit sequential re-read failed: %s", e) @@ -214,7 +214,7 @@ async def _consistency_reread( ) except TimeoutError: log.warning("memory.search: sequential-consistency retry exceeded the wall budget") - except VectorInputTooLargeError: + except VectorCapacityError: raise except Exception as e: # noqa: BLE001 — an optional retry; the first pass stands log.warning("memory.search: sequential-consistency retry failed: %s", e) diff --git a/server/memory/app/search/digest_descent.py b/server/memory/app/search/digest_descent.py index f5b1dd11da..6bdd472e76 100644 --- a/server/memory/app/search/digest_descent.py +++ b/server/memory/app/search/digest_descent.py @@ -6,6 +6,8 @@ import logging from typing import Any +from server.core.errors import VectorCapacityError +from server.core.vectors.admission import vector_work from server.core.vectors.private_first import each from server.memory.api.models_search import _MemorySearchHit from server.memory.app.search.context import SearchContext @@ -95,7 +97,10 @@ async def _rows(ns: str, sid: str) -> tuple[str, str, list[dict[str, Any]]]: } sql = digest_descent_sql(" AND ".join(_digest_descent_where(input, params))) try: - return ns, sid, await _ch.query_all(sql, params=params) + with vector_work(input.org, ns): + return ns, sid, await _ch.query_all(sql, params=params) + except VectorCapacityError: + raise except Exception as e: # noqa: BLE001 — additive arm log.warning("digest descent failed for %s/%s (non-fatal): %s", ns, sid, e) return ns, sid, [] @@ -146,5 +151,7 @@ async def _rows_for_target( ctx.all_hits.append(hit) added += 1 ctx.actual_work["digest_descent_added"] = added + except VectorCapacityError: + raise except Exception as e: # noqa: BLE001 log.warning("digest descent stage failed (non-fatal): %s", e) diff --git a/server/memory/app/search/fallbacks.py b/server/memory/app/search/fallbacks.py index 6ee6dede64..b921f06ea0 100644 --- a/server/memory/app/search/fallbacks.py +++ b/server/memory/app/search/fallbacks.py @@ -9,7 +9,7 @@ import asyncio import logging -from server.core.errors import VectorInputTooLargeError +from server.core.errors import VectorCapacityError from server.memory.app.search.context import SearchContext from server.memory.app.search.scoring import drop_rerank_trail from server.memory.app.search.verdict import gate_order @@ -105,7 +105,7 @@ async def _maybe_iterative_retry(ctx: SearchContext) -> None: final_hits.append(rh) final_hits.sort(key=hit_sort_key) ctx.final_hits = final_hits[: input.top_k] - except VectorInputTooLargeError: + except VectorCapacityError: raise except Exception as e: # noqa: BLE001 log.warning("iterative retrieve retry failed: %s", e) diff --git a/server/memory/app/search/key_match.py b/server/memory/app/search/key_match.py index 837c157802..ca4a0d0a59 100644 --- a/server/memory/app/search/key_match.py +++ b/server/memory/app/search/key_match.py @@ -8,6 +8,8 @@ import re from typing import Any +from server.core.errors import VectorCapacityError +from server.core.vectors.admission import vector_work from server.core.vectors.private_first import each from server.memory.api.models_search import MemorySearchInput, _MemorySearchHit from server.memory.app.search.context import SearchContext @@ -335,7 +337,10 @@ async def _rows_for_ns(ns: str) -> tuple[str, list[dict[str, Any]], bool]: return ns, [], True sql = key_match_sql(" AND ".join(where)) try: - return ns, await _ch.query_all(sql, params=params), False + with vector_work(input.org, ns): + return ns, await _ch.query_all(sql, params=params), False + except VectorCapacityError: + raise except Exception as e: # noqa: BLE001 — additive arm log.warning("key-match arm failed for ns=%s (non-fatal): %s", ns, e) return ns, [], False @@ -390,5 +395,7 @@ async def _rows_for_ns(ns: str) -> tuple[str, list[dict[str, Any]], bool]: ctx.actual_work["key_match_added"] = added if component_only: ctx.actual_work["key_match_component_only"] = True + except VectorCapacityError: + raise except Exception as e: # noqa: BLE001 log.warning("key-match stage failed (non-fatal): %s", e) diff --git a/server/memory/app/search/supplemental.py b/server/memory/app/search/supplemental.py index 8f0c99c621..abf9c96d07 100644 --- a/server/memory/app/search/supplemental.py +++ b/server/memory/app/search/supplemental.py @@ -8,6 +8,7 @@ from dataclasses import replace from typing import Any +from server.core.errors import VectorCapacityError from server.memory.api.models_search import _MemorySearchHit from server.memory.app.search.context import SearchContext from server.memory.app.search.vector_legs import _search_query_for, _wire_hit @@ -95,6 +96,8 @@ async def _run_set_expansion(ctx: SearchContext) -> None: per_phrase = await asyncio.gather( *[_supplemental_leg(ctx, ph, leg_top_k, seen_keys) for ph in phrases] ) + except VectorCapacityError: + raise except Exception as e: # noqa: BLE001 — additive legs log.warning("set expansion legs failed (non-fatal): %s", e) return diff --git a/server/memory/app/search/vector_legs.py b/server/memory/app/search/vector_legs.py index 0b47a31ad7..fb2595e835 100644 --- a/server/memory/app/search/vector_legs.py +++ b/server/memory/app/search/vector_legs.py @@ -9,6 +9,7 @@ import logging from typing import Any +from server.core.errors import VectorCapacityError from server.core.vectors.private_first import best_similarity, each, search_private_first from server.identity.app import namespace_visibility from server.memory.api.models_search import _MemorySearchHit @@ -205,6 +206,8 @@ async def _run_vector_legs( if entity_boost_task is not None: try: ctx.entity_boost_by_key = await entity_boost_task + except VectorCapacityError: + raise except Exception as e: # noqa: BLE001 — entity boost is optional; search runs without it log.warning("entity_vectors ANN read failed (non-fatal): %s", e) diff --git a/server/memory/app/session_outcomes.py b/server/memory/app/session_outcomes.py index 5068ebc361..2bfbbc0e26 100644 --- a/server/memory/app/session_outcomes.py +++ b/server/memory/app/session_outcomes.py @@ -31,6 +31,7 @@ from datetime import UTC, datetime from typing import Any +from server.core.vectors.admission import vector_work from server.llm import customer from server.llm.identity import reads_writer from server.memory.api.models_crud import MemorySetInput @@ -469,20 +470,21 @@ async def _contrast_pair(ch: Any, llm: Any, cfg: DistillConfig, neg: Any, writer was written, False when there is no close pair or no lesson.""" sid = str(neg.get("session_id") or "") vec = [float(x) for x in (neg.get("summary_embedding") or [])] - pos = await ch.query_one( - _NEAREST_POSITIVE_SQL, - params={ - "q": vec, - "minc": float(cfg.min_confidence), - "org": str(neg["org"]), - "ns": str(neg["namespace"]), - "model": str(neg.get("embedding_model") or ""), - "sid": sid, - # the positive's summary is handed to the LLM and its lessons - # land in this user's memory: never another user's session - "uid": str(neg.get("user_id") or "-"), - }, - ) + with vector_work(str(neg["org"]), str(neg["namespace"])): + pos = await ch.query_one( + _NEAREST_POSITIVE_SQL, + params={ + "q": vec, + "minc": float(cfg.min_confidence), + "org": str(neg["org"]), + "ns": str(neg["namespace"]), + "model": str(neg.get("embedding_model") or ""), + "sid": sid, + # the positive's summary is handed to the LLM and its lessons + # land in this user's memory: never another user's session + "uid": str(neg.get("user_id") or "-"), + }, + ) sim = float((pos or {}).get("similarity") or 0.0) if not pos or sim < float(cfg.min_pair_similarity): return False diff --git a/server/memory/app/value_updates.py b/server/memory/app/value_updates.py index 486c729a8e..843a5569ce 100644 --- a/server/memory/app/value_updates.py +++ b/server/memory/app/value_updates.py @@ -35,6 +35,9 @@ from datetime import datetime from typing import Any +from server.core.errors import VectorCapacityError +from server.core.vectors.admission import batch_size, vector_work +from server.core.vectors.private_first import each from server.identity.app.request_scope import request_scope from server.identity.domain.read_scope import scope_and from server.memory.app.auto_resolution import ( @@ -230,7 +233,8 @@ async def _peers_many( } # A write retires only rows inside the writer's read scope. in_scope = scope_and(request_scope(), params, "m") - rows = await ch.query_all(_peers_sql_many(len(items), in_scope), params=params) + with vector_work(org, namespace, len(items)): + rows = await ch.query_all(_peers_sql_many(len(items), in_scope), params=params) out: dict[int, list[dict[str, Any]]] = {i: [] for i in range(len(items))} for r in rows: q = int(r.get("qidx") or 0) @@ -259,32 +263,33 @@ async def _peers_joined( "min_cos": float(min_cosine), } in_scope = scope_and(request_scope(), params, "m") - rows = await ch.query_all( - f""" - SELECT toString(m.memory_id) AS memory_id, m.memory_key AS memory_key, - m.value AS value, m.valid_at AS valid_at, m.created_at AS created_at, - 1 - cosineDistance(e.embedding, {{emb:Array(Float32)}}) AS cos - FROM loom.memory m FINAL - JOIN loom.memory_embeddings e FINAL - ON e.org = m.org AND e.namespace = m.namespace - AND e.memory_key = m.memory_key - AND e.user_id = m.user_id AND e.session_id = m.session_id - AND e.agent = m.agent - WHERE m.org = {{org:String}} - AND m.namespace = {{ns:String}} - AND m.memory_type = {{mt:String}} - AND m.memory_key != {{self_key:String}} - AND m.tombstoned_at IS NULL - AND m.status IN ('active', 'conflicted') - AND e.tombstoned_at IS NULL - AND e.embedding_dim = {{dim:UInt16}} - AND length(e.embedding) = {{dim:UInt16}}{in_scope} - HAVING cos >= {{min_cos:Float32}} - ORDER BY cos DESC - LIMIT 8 - """, - params=params, - ) + with vector_work(org, namespace): + rows = await ch.query_all( + f""" + SELECT toString(m.memory_id) AS memory_id, m.memory_key AS memory_key, + m.value AS value, m.valid_at AS valid_at, m.created_at AS created_at, + 1 - cosineDistance(e.embedding, {{emb:Array(Float32)}}) AS cos + FROM loom.memory m FINAL + JOIN loom.memory_embeddings e FINAL + ON e.org = m.org AND e.namespace = m.namespace + AND e.memory_key = m.memory_key + AND e.user_id = m.user_id AND e.session_id = m.session_id + AND e.agent = m.agent + WHERE m.org = {{org:String}} + AND m.namespace = {{ns:String}} + AND m.memory_type = {{mt:String}} + AND m.memory_key != {{self_key:String}} + AND m.tombstoned_at IS NULL + AND m.status IN ('active', 'conflicted') + AND e.tombstoned_at IS NULL + AND e.embedding_dim = {{dim:UInt16}} + AND length(e.embedding) = {{dim:UInt16}}{in_scope} + HAVING cos >= {{min_cos:Float32}} + ORDER BY cos DESC + LIMIT 8 + """, + params=params, + ) return [dict(r) for r in rows] @@ -371,6 +376,8 @@ def _eligible(row: dict[str, Any]) -> list[float] | None: from server.settings import settings as _s batch = int(_s.memory_write_candidate_batch) + if batch > 0: + batch = batch_size(batch) per_fact = _s.memory_bulk_peer_read == "per_fact" read_one = _peers_codes_first if per_fact else _peers_joined prefetched: dict[int, list[dict[str, Any]]] = {} @@ -397,14 +404,16 @@ async def _batch(idxs: list[int]) -> None: for j, i in enumerate(idxs): prefetched[i] = got.get(j, []) - jobs = [ - _batch(idxs[k : k + batch]) - for idxs in by_dim.values() - for k in range(0, len(idxs), batch) - ] - for res in await asyncio.gather(*jobs, return_exceptions=True): - if isinstance(res, Exception): - log.warning("value_updates: batched peer lookup failed (per-row): %s", res) + async def _read_batch(idxs: list[int]) -> None: + try: + await _batch(idxs) + except VectorCapacityError: + raise + except Exception as error: # noqa: BLE001 — ordinary failures retain per-row fallback + log.warning("value_updates: batched peer lookup failed (per-row): %s", error) + + jobs = [idxs[k : k + batch] for idxs in by_dim.values() for k in range(0, len(idxs), batch)] + await each(_read_batch)(jobs) async def _one(i: int, row: dict[str, Any]) -> list[UpdateLink]: key = str(row.get("memory_key") or "") @@ -424,6 +433,8 @@ async def _one(i: int, row: dict[str, Any]) -> list[UpdateLink]: embedding=emb, min_cosine=min_cosine, ) + except VectorCapacityError: + raise except Exception as e: # noqa: BLE001 — never block an ingest log.warning("value_updates: peer lookup failed (open): %s", e) report.errors += 1 @@ -437,7 +448,10 @@ async def _one(i: int, row: dict[str, Any]) -> list[UpdateLink]: peers, ) - results = await asyncio.gather(*(_one(i, r) for i, r in enumerate(rows))) + async def _read_row(item: tuple[int, dict[str, Any]]) -> list[UpdateLink]: + return await _one(*item) + + results = await each(_read_row)(list(enumerate(rows))) report.checked = len(rows) links = [ln for sub in results for ln in sub] # One write per older row; if several newer rows supersede it, the latest diff --git a/server/memory/store/conflict_resolver.py b/server/memory/store/conflict_resolver.py index f550fd6100..8d0795c0cf 100644 --- a/server/memory/store/conflict_resolver.py +++ b/server/memory/store/conflict_resolver.py @@ -26,6 +26,7 @@ from server.core.naming import SENTINEL, is_anonymous_user, session_spellings from server.core.repo_base import _CHClient, _default_client +from server.core.vectors.admission import vector_work from server.memory.store.embeddings import live_embedding_rows_for_key, reemit_rows # Aware: a naive datetime encodes as host time, which east of UTC is before 1970. @@ -383,7 +384,10 @@ async def cluster_peers( "lim": 4 * max(int(limit), 1), } half = max(int(limit) // 2, 1) - near = [c for c in await ch.query_all(_nearest_sql(), params=params) if 1 - c["dist"] >= floor] + with vector_work(str(row["org"]), str(row["namespace"])): + near = [ + c for c in await ch.query_all(_nearest_sql(), params=params) if 1 - c["dist"] >= floor + ] anchored = await ch.query_all(_ANCHORED_SQL, params={**params, "same_lim": half}) above_floor = {key_identity(c) for c in near} found = {key_identity(c): c for c in [*anchored, *near]} diff --git a/server/memory/store/lookups.py b/server/memory/store/lookups.py index 7c9426b0e9..343bca397e 100644 --- a/server/memory/store/lookups.py +++ b/server/memory/store/lookups.py @@ -8,6 +8,7 @@ from typing import Any from server.core.repo_base import _CHClient, _default_client +from server.core.vectors.admission import vector_work # --- The cosine-ordered fact read reaches the vector through a keyed re-read --- # @@ -153,7 +154,10 @@ async def find_by_identifier( # Only a cosine-ordered read touches the vector; the length and recency # orders never materialise `embedding` and keep one statement. if qv and _order_mode != "recency": - return await ch.query_all(_fact_identifier_one_trip_sql(where_sql, order), params=params) + with vector_work(org, namespace): + return await ch.query_all( + _fact_identifier_one_trip_sql(where_sql, order), params=params + ) return await ch.query_all( f""" SELECT agent, user_id, session_id, memory_key, memory_type, content_excerpt, @@ -254,23 +258,24 @@ async def find_chunks_by_identifier( # wide columns), and the outer read re-applies the predicate to those keys # only, where the tombstone of a retired chunk wins. where_sql = " AND ".join(where) - return await ch.query_all( - f""" - SELECT concat('chunk:', session_id, ':', toString(chunk_index)) AS memory_key, - 'episodic' AS memory_type, - content AS content_excerpt, - toString(created_at) AS created_at, - toString(created_at) AS valid_at - FROM loom.memory_chunks FINAL - WHERE {where_sql} - AND ({_CHUNK_KEY}) IN ( - SELECT DISTINCT {_CHUNK_KEY} FROM loom.memory_chunks WHERE {where_sql} - ) - ORDER BY {order_by} - LIMIT {{lim:UInt32}} - """, - params=params, - ) + with vector_work(org, namespace, 1 if qv and order != "recency" else 0): + return await ch.query_all( + f""" + SELECT concat('chunk:', session_id, ':', toString(chunk_index)) AS memory_key, + 'episodic' AS memory_type, + content AS content_excerpt, + toString(created_at) AS created_at, + toString(created_at) AS valid_at + FROM loom.memory_chunks FINAL + WHERE {where_sql} + AND ({_CHUNK_KEY}) IN ( + SELECT DISTINCT {_CHUNK_KEY} FROM loom.memory_chunks WHERE {where_sql} + ) + ORDER BY {order_by} + LIMIT {{lim:UInt32}} + """, + params=params, + ) async def find_chunks_by_session_date( @@ -329,38 +334,40 @@ async def find_chunks_by_session_date( # (see find_chunks_by_identifier); the subquery keeps to the sessions with # a digest dated in range, a superset of the join's max(valid_at) rule. chunk_where = [w for w in where if not w.startswith("toDate(ss.")] - return await ch.query_all( - f""" - SELECT concat('chunk:', c.session_id, ':', toString(c.chunk_index)) AS memory_key, - 'episodic' AS memory_type, - c.content AS content_excerpt, - toString(ss.session_date) AS created_at, - toString(ss.session_date) AS valid_at - FROM loom.memory_chunks AS c FINAL - INNER JOIN ( - SELECT session_id AS ss_session_id, max(valid_at) AS session_date - FROM loom.session_summaries FINAL - WHERE org = {{org:String}} - AND namespace = {{ns:String}} - AND tombstoned_at IS NULL - AND valid_at IS NOT NULL - GROUP BY session_id - ) AS ss ON ss.ss_session_id = c.session_id - WHERE {" AND ".join(where)} - AND (c.org, c.namespace, c.session_id, c.chunk_index, c.content_sha256) IN ( - SELECT DISTINCT {_CHUNK_KEY} FROM loom.memory_chunks AS c - WHERE {" AND ".join(chunk_where)} - AND c.session_id IN ( - SELECT session_id FROM loom.session_summaries - WHERE org = {{org:String}} AND namespace = {{ns:String}} - AND toDate(valid_at) BETWEEN toDate({{d0:String}}) AND toDate({{d1:String}}) - ) - ) - ORDER BY {order} - LIMIT {{lim:UInt32}} - """, - params=params, - ) + with vector_work(org, namespace, 1 if qv else 0): + return await ch.query_all( + f""" + SELECT concat('chunk:', c.session_id, ':', toString(c.chunk_index)) AS memory_key, + 'episodic' AS memory_type, + c.content AS content_excerpt, + toString(ss.session_date) AS created_at, + toString(ss.session_date) AS valid_at + FROM loom.memory_chunks AS c FINAL + INNER JOIN ( + SELECT session_id AS ss_session_id, max(valid_at) AS session_date + FROM loom.session_summaries FINAL + WHERE org = {{org:String}} + AND namespace = {{ns:String}} + AND tombstoned_at IS NULL + AND valid_at IS NOT NULL + GROUP BY session_id + ) AS ss ON ss.ss_session_id = c.session_id + WHERE {" AND ".join(where)} + AND (c.org, c.namespace, c.session_id, c.chunk_index, c.content_sha256) IN ( + SELECT DISTINCT {_CHUNK_KEY} FROM loom.memory_chunks AS c + WHERE {" AND ".join(chunk_where)} + AND c.session_id IN ( + SELECT session_id FROM loom.session_summaries + WHERE org = {{org:String}} AND namespace = {{ns:String}} + AND toDate(valid_at) + BETWEEN toDate({{d0:String}}) AND toDate({{d1:String}}) + ) + ) + ORDER BY {order} + LIMIT {{lim:UInt32}} + """, + params=params, + ) # Suffixes a row may carry for an attribute facet the extractor did not diff --git a/server/settings/catalog.py b/server/settings/catalog.py index 5d47772c69..a65266cbf6 100644 --- a/server/settings/catalog.py +++ b/server/settings/catalog.py @@ -694,6 +694,12 @@ class ClickHouseSettings(BaseModel): # HTTP connection pool size: request concurrency plus 3 x the batched-insert bound # (inserts a second x wait / 1000; docs/architecture/clickhouse.md#batched-inserts). ch_pool_maxsize: int = 64 + # Weighted vector work admitted per org in this process; two axes need two units. + vector_scan_max_concurrent_per_org: Annotated[int, LIVE] = Field(default=16, ge=2) + # Weighted org wait units; twice this also bounds the whole process population. + vector_scan_max_waiters_per_org: Annotated[int, LIVE] = Field(default=256, gt=0) + # Native throw ceiling per vector statement, including joins and row versions. + vector_scan_max_rows_to_read: int = Field(default=1_000_000, gt=0) # Seconds an idle pooled connection is kept. Must stay BELOW the Cloud load # balancer's idle cutoff (8 s) so loom closes first: a peer-closed pooled # TLS connection is never reaped under uvloop (`LOOM_UVICORN_LOOP=uvloop`). diff --git a/server/tracker/app/dupcheck.py b/server/tracker/app/dupcheck.py index 85d44c5fac..372a02a793 100644 --- a/server/tracker/app/dupcheck.py +++ b/server/tracker/app/dupcheck.py @@ -26,7 +26,10 @@ from typing import Any from server.code.domain.index.repos import registry_unread +from server.core.errors import VectorCapacityError from server.core.llm_free import reads_llm_free +from server.core.shielded import to_its_end +from server.core.vectors.private_first import each from server.llm import customer from server.settings import settings from server.tracker.api.models import ( @@ -304,6 +307,43 @@ async def one(ns: str, group: list[dict[str, Any]]) -> None: logger.info("tracker.duplicate_check: fix links skipped: %s", res) +async def _namespace_checks( + inputs: list[TrackerDuplicateCheckInput], + wall: datetime | None, + pools: list[Any], + pools_ready: asyncio.Future[None], +) -> list[Any]: + """Only an admitted namespace allocates its hand-over future.""" + loop = asyncio.get_running_loop() + ready_count = 0 + + async def one(indexed: tuple[int, TrackerDuplicateCheckInput]) -> Any: + index, inp = indexed + ready: asyncio.Future[Any] = loop.create_future() + + def received(future: asyncio.Future[Any]) -> None: + nonlocal ready_count + pools[index] = future.result() + ready_count += 1 + if ready_count == len(inputs) and not pools_ready.done(): + pools_ready.set_result(None) + + ready.add_done_callback(received) + try: + return await _check_one(inp, wall, pool_ready=ready) + except VectorCapacityError as error: + if not pools_ready.done(): + pools_ready.set_exception(error) + raise + except BaseException as error: # noqa: BLE001 — preserve gathered noncapacity outcomes + return error + finally: + if not ready.done(): + ready.set_result([]) + + return await each(one)(list(enumerate(inputs))) + + async def _checked( inputs: list[TrackerDuplicateCheckInput], wall: datetime | None, top_k: int ) -> TrackerDuplicateCheckOutput: @@ -315,26 +355,25 @@ async def _checked( from server.tracker.app.dupcheck_fanout import merge_outputs loop = asyncio.get_running_loop() - ready = [loop.create_future() for _ in inputs] - tasks = [ - asyncio.ensure_future(_check_one(inp, wall, pool_ready=f)) - for inp, f in zip(inputs, ready, strict=True) - ] - for t, f in zip(tasks, ready, strict=True): - t.add_done_callback(_empty_pool_on_exit(f)) - owned: list[asyncio.Future[Any]] = list(tasks) + pools: list[Any] = [None] * len(inputs) + pools_ready: asyncio.Future[None] = loop.create_future() + checking = asyncio.create_task(_namespace_checks(inputs, wall, pools, pools_ready)) + owned: list[asyncio.Future[Any]] = [checking, pools_ready] + failure: BaseException | None = None try: - # Cancelling a gather over the bare pool futures does not reach the checks - # (the loom_search tracker leg runs this under wait_for): every task this - # call started is cancelled on the way out unless it finished. - pools = await asyncio.gather(*ready) + if not inputs: + pools_ready.set_result(None) + # The final window can finish judging while the shared rerank runs. + await pools_ready rerank = asyncio.ensure_future( _rerank_over(inputs[0], list(pools)) if inputs[0].rerank else _none() ) owned.append(rerank) - outs = await asyncio.gather(*tasks, return_exceptions=True) + outs = await checking kept: list[Any] = [] for inp, o in zip(inputs, outs, strict=True): + if isinstance(o, VectorCapacityError): + raise o if isinstance(o, BaseException): if inp is inputs[0] or isinstance(o, customer.HOLDS): raise o # a held model call is the answer, from any tracker @@ -350,10 +389,22 @@ async def _checked( else: out = merge_outputs([i.namespace for i in inputs], kept, top_k) picked = await rerank + except BaseException as error: + failure = error + raise finally: for started in owned: if not started.done(): started.cancel() + + async def stopped() -> None: + await asyncio.gather(*owned, return_exceptions=True) + + try: + await to_its_end(stopped(), name="tracker.duplicate.cleanup") + except asyncio.CancelledError: + if failure is None: + raise if picked is None: return out rr, picks = picked @@ -364,17 +415,6 @@ async def _none() -> None: return None -def _empty_pool_on_exit(fut: Any) -> Any: - """A check that ends before handing its pool over (it failed early, or - found the record unavailable) counts as an empty pool.""" - - def done(_task: Any) -> None: - if not fut.done(): - fut.set_result([]) - - return done - - async def _rerank_over(input: Any, pools: list[list[dict[str, Any]]]) -> Any: """One rerank call over every namespace's pool (the namespaces taking turns) -> (Rerank, pick rows), or None. A pick names its namespace: two repositories' issue @@ -481,16 +521,29 @@ async def hand_over() -> None: pool_ready.set_result(rows) handing = asyncio.ensure_future(hand_over()) + failure: BaseException | None = None try: fit = await cached_fit(input.org, input.namespace) out = answer(run, fit, status=status, wall=wall, top_k=int(input.top_k)) out = await judge_uncertain(src, input, run, out, fit) if handing is not None: await handing + except BaseException as error: + failure = error + raise finally: # A failed or cancelled check does not leave its pool read running. - if handing is not None and not handing.done(): + if handing is not None: handing.cancel() + + async def stopped() -> None: + await asyncio.gather(handing, return_exceptions=True) + + try: + await to_its_end(stopped(), name="tracker.duplicate.pool.cleanup") + except asyncio.CancelledError: + if failure is None: + raise out = with_closest(out, run, input, fit) out = await with_canonical(src, input, out) return out diff --git a/server/tracker/app/handlers.py b/server/tracker/app/handlers.py index 9dc7ebf7df..c70d1afb36 100644 --- a/server/tracker/app/handlers.py +++ b/server/tracker/app/handlers.py @@ -9,11 +9,11 @@ from __future__ import annotations -import asyncio import logging from typing import Any -from server.core.errors import ValidationError +from server.core.errors import ValidationError, VectorCapacityError +from server.core.vectors.private_first import each from server.tracker.api.models import ( TrackerAnswerInput, TrackerAnswerOutput, @@ -300,13 +300,20 @@ async def _answer_elsewhere( `:` head.""" from server.tracker.app.router import TrackerAnswer, answer, render_lines - outs = await asyncio.gather( - *[answer(question, tracker_namespace=ns, **ask) for ns in namespaces], - return_exceptions=True, - ) + async def one(namespace: str) -> Any: + try: + return await answer(question, tracker_namespace=namespace, **ask) + except VectorCapacityError: + raise + except BaseException as error: # noqa: BLE001 — preserve gathered noncapacity outcomes + return error + + outs = await each(one)(namespaces) kept: list[tuple[str, Any]] = [] lines: list[str] = [] for ns, a in zip(namespaces, outs, strict=True): + if isinstance(a, VectorCapacityError): + raise a if isinstance(a, BaseException): logger.info("tracker.answer on %s failed inside a fan-out: %s", ns, a) continue diff --git a/server/tracker/app/router.py b/server/tracker/app/router.py index efa9ed7f45..cf43701c4d 100644 --- a/server/tracker/app/router.py +++ b/server/tracker/app/router.py @@ -14,12 +14,13 @@ from __future__ import annotations -import asyncio import logging import time from dataclasses import dataclass, field from typing import Any +from server.core.errors import VectorCapacityError +from server.core.vectors.private_first import each from server.tracker.app import source as tsource from server.tracker.domain import catalogue as cat from server.tracker.domain.errors import TrackerUnavailable @@ -179,6 +180,8 @@ async def _run_step( started = time.monotonic() try: rows = await _call_reader(source, org, ns, t, step) + except VectorCapacityError: + raise except TrackerUnavailable as e: logger.info("tracker.answer %s unavailable: %s", t.name, e) sec.status = "tracker_unavailable" @@ -329,11 +332,11 @@ async def answer( ) ) return out - out.sections = list( - await asyncio.gather( - *(_run_step(source, org, tracker_namespace, st, out.restricted) for st in p.steps) - ) - ) + + async def one(step: cat.Step) -> Section: + return await _run_step(source, org, tracker_namespace, step, out.restricted) + + out.sections = await each(one)(list(p.steps)) return out diff --git a/server/tracker/app/similar.py b/server/tracker/app/similar.py index 9c74357a9d..6848479a5b 100644 --- a/server/tracker/app/similar.py +++ b/server/tracker/app/similar.py @@ -9,7 +9,9 @@ import time from typing import Any -from server.core.errors import ValidationError, caller_text +from server.core.errors import ValidationError, VectorCapacityError, caller_text +from server.core.shielded import to_its_end +from server.core.vectors.private_first import each from server.llm import customer from server.settings import settings from server.tracker.api.models import ( @@ -48,12 +50,16 @@ async def handle_tracker_similar(input: TrackerSimilarInput) -> TrackerSimilarOu head = f"{item.get('title') or ''}\n{item.get('body') or ''}" text = head + (f"\n{text}" if text.strip() else "") rest = input.model_copy(update={"issue_number": 0, "text": text}) - others = await asyncio.gather( - *[_similar_one(rest.model_copy(update={"namespace": ns})) for ns in linked[1:]] - if text.strip() or rest.frames - else [], - return_exceptions=True, - ) + + async def one(namespace: str) -> Any: + try: + return await _similar_one(rest.model_copy(update={"namespace": namespace})) + except VectorCapacityError: + raise + except BaseException as error: # noqa: BLE001 — preserve gathered noncapacity outcomes + return error + + others = await each(one)(linked[1:] if text.strip() or rest.frames else []) return _merge_similar(linked, [first, *others], int(input.top_k)) @@ -65,6 +71,8 @@ def _merge_similar(namespaces: list[str], outs: list[Any], top_k: int) -> Tracke held model call (`customer.HOLDS`), which is the answer.""" served: list[tuple[str, Any]] = [] for ns, o in zip(namespaces, outs, strict=False): + if isinstance(o, VectorCapacityError): + raise o if isinstance(o, customer.HOLDS): raise o # the org's model calls are held: the request answers that if isinstance(o, BaseException): @@ -113,10 +121,26 @@ async def similar_in_namespace(input: TrackerSimilarInput) -> TrackerSimilarOutp status: dict[str, str] = {} meta: dict[str, Any] = {} record: dict[str, Any] = {"status": "skipped", "relations": {}} - ranked, record = await asyncio.gather( - _similar_legs(src, input, q, stats, item, kinds, legs_on, status, meta), - _similar_record(src, input, q, item), - ) + tasks = [ + asyncio.create_task( + _similar_legs(src, input, q, stats, item, kinds, legs_on, status, meta) + ), + asyncio.create_task(_similar_record(src, input, q, item)), + ] + try: + ranked, record = await asyncio.gather(*tasks) + except BaseException: + for task in tasks: + task.cancel() + + async def stopped() -> None: + await asyncio.gather(*tasks, return_exceptions=True) + + try: + await to_its_end(stopped(), name="tracker.similar.namespace.cleanup") + except asyncio.CancelledError: + pass # Preserve the first failure after both reads stop. + raise # The vector leg adds its own candidates after the ranked ones and never re-ranks. fused = sim.fuse( ranked, @@ -289,6 +313,8 @@ async def vector() -> list[dict[str, Any]]: return await _similar_vector_leg( src, input, q, item, kinds, before, status, meta if meta is not None else {} ) + except VectorCapacityError: + raise except TrackerUnavailable: raise except customer.HOLDS: @@ -309,22 +335,51 @@ def leg(name: str, coro: Any) -> Any: name, coro, ns=input.namespace, status=status, errors=errors, timings=timings ) - # Every leg runs to its end; then a held model call (`customer.HOLDS`) is the - # request's answer, rather than a degraded one without the vector leg. - results = await asyncio.gather( + return await _collect_similar_legs( leg("lexical", lexical()), leg("frames", frames()), leg("mentions", mentions()), leg("vector", vector()), - return_exceptions=True, ) - legs: dict[str, list[dict[str, Any]]] = {} - for res in results: - if isinstance(res, BaseException): - raise res - if res[1]: - legs[res[0]] = res[1] - return legs + + +async def _collect_similar_legs( + lexical: Any, + frames: Any, + mentions: Any, + vector: Any, +) -> dict[str, list[dict[str, Any]]]: + """Settle the fixed retrieval legs before selecting their complete answer.""" + # Ordinary leg errors settle with the other legs before selecting the answer. + tasks = [ + asyncio.ensure_future(lexical), + asyncio.ensure_future(frames), + asyncio.ensure_future(mentions), + asyncio.ensure_future(vector), + ] + try: + results = await asyncio.gather(*tasks, return_exceptions=True) + legs: dict[str, list[dict[str, Any]]] = {} + for res in results: + if isinstance(res, VectorCapacityError): + raise res + if isinstance(res, BaseException): + raise res + if res[1]: + legs[res[0]] = res[1] + return legs + except BaseException: + for task in tasks: + task.cancel() + + async def stopped() -> None: + await asyncio.gather(*tasks, return_exceptions=True) + + try: + await to_its_end(stopped(), name="tracker.similar.legs.cleanup") + except asyncio.CancelledError: + pass # Preserve the first failure after every owned leg stops. + raise async def _guarded( @@ -340,6 +395,8 @@ async def _guarded( t0 = time.monotonic() try: return name, await coro + except VectorCapacityError: + raise except TrackerUnavailable as e: common._unavailable(f"tracker.similar/{name}", ns, e) status[name] = "tracker_unavailable" diff --git a/server/tracker/store/embed.py b/server/tracker/store/embed.py index f5c700dbfd..07cb35742a 100644 --- a/server/tracker/store/embed.py +++ b/server/tracker/store/embed.py @@ -23,6 +23,7 @@ from server.core.ch import CHClient, get_client from server.core.datetimes import naive_utc_now_seconds as now_naive +from server.core.vectors.admission import vector_work TABLE = "loom.qa_item_embeddings" ZERO_SHA = "0" * 64 @@ -306,17 +307,18 @@ async def nearest( table=TABLE, fetch_multiplier=float(settings.tracker_embed_fetch_multiplier), ) - rows = await self._ch.query_all( - f""" - SELECT {_HIT_COLS}, cosineDistance(embedding, {{q:Array(Float32)}}) AS distance - FROM {TABLE} FINAL - WHERE {" AND ".join(conds)} - ORDER BY distance ASC - LIMIT {{limit:UInt32}} - {vec_settings} - """, - params, - ) + with vector_work(org, namespace): + rows = await self._ch.query_all( + f""" + SELECT {_HIT_COLS}, cosineDistance(embedding, {{q:Array(Float32)}}) AS distance + FROM {TABLE} FINAL + WHERE {" AND ".join(conds)} + ORDER BY distance ASC + LIMIT {{limit:UInt32}} + {vec_settings} + """, + params, + ) rows.sort( key=lambda r: ( float(r.get("distance") or 0.0), diff --git a/server/tracker/store/source/base.py b/server/tracker/store/source/base.py index e253560ed5..d49b2cc852 100644 --- a/server/tracker/store/source/base.py +++ b/server/tracker/store/source/base.py @@ -16,7 +16,9 @@ import time from typing import Any, Protocol -from server.core.errors import caller_text +from server.core.ch import get_client +from server.core.errors import VectorAdmissionError, VectorCapacityError, caller_text +from server.core.vectors.admission import admit, native_settings, vector_work from server.settings import settings from server.tracker.domain.contract import ContractFailed from server.tracker.domain.errors import TrackerUnavailable @@ -212,6 +214,7 @@ async def _rows( budget_ms: int | None = None, ttl_s: float | None = None, query_settings: dict[str, Any] | None = None, + vector: bool = False, ) -> list[dict[str, Any]]: await self._admit() assert self.cfg is not None @@ -230,19 +233,30 @@ async def _rows( ms = budget_ms if budget_ms else int(settings.tracker_live_budget_ms) budget_s = max(0.05, ms / 1000.0) try: - rows = await asyncio.wait_for( - gate.query_rows( - sql, - params, - credential=self.cfg.credential, - max_execution_s=budget_s if budget_ms else None, - query_settings=query_settings, - ), - timeout=budget_s, - ) - except TrackerUnavailable: + ready = get_client().vector_pool_ready() if vector else False + with vector_work(self.org, self.namespace, 1 if vector else 0): + async with admit(pool_ready=ready, wait_s=budget_s): + rows = await asyncio.wait_for( + gate.query_rows( + sql, + params, + credential=self.cfg.credential, + max_execution_s=budget_s if budget_ms else None, + query_settings=native_settings( + dict(query_settings or {}), timeout=budget_s + ), + ), + timeout=budget_s, + ) + except (TrackerUnavailable, VectorCapacityError): raise except TimeoutError as e: + if vector: + raise VectorAdmissionError( + "the complete vector read exceeded its deadline", + reason="deadline", + retry_after_s=1, + ) from e # the caller's deadline, not the upstream's failure: the read is # unavailable, the breaker stays closed raise TrackerUnavailable(f"{tag} exceeded {int(budget_s * 1000)} ms") from e diff --git a/server/tracker/store/source/qa_intel.py b/server/tracker/store/source/qa_intel.py index 693f599866..05e8c0b7d4 100644 --- a/server/tracker/store/source/qa_intel.py +++ b/server/tracker/store/source/qa_intel.py @@ -24,8 +24,10 @@ from typing import Any from server.core.ch_errors import error_code +from server.core.errors import VectorCapacityError from server.core.hosts import vet_host from server.core.remote_ch import connect +from server.core.vectors.admission import native_settings, raise_native from server.settings import settings from server.tracker.domain import sources as _sources from server.tracker.domain.source_circuit import SourceCircuit @@ -211,11 +213,15 @@ async def query_rows( settings_for_query: dict[str, Any] = dict(query_settings or {}) if max_execution_s: settings_for_query["max_execution_time"] = max(0.1, float(max_execution_s)) + native_settings( + settings_for_query, timeout=max_execution_s or float(settings.tracker_live_timeout_s) + ) res = await client.query(sql, parameters=params or {}, settings=settings_for_query) rows = [dict(zip(res.column_names, r, strict=True)) for r in res.result_rows] - except QaIntelUnavailable: + except (QaIntelUnavailable, VectorCapacityError): raise except Exception as e: + raise_native(e) # an upstream quota refusal (code 201) is the source rationing this # shape, not the service failing: the read is unavailable, the # breaker stays closed for every other shape (the diff --git a/server/tracker/store/source/similarity.py b/server/tracker/store/source/similarity.py index ec5c1ba182..8c6acb7d48 100644 --- a/server/tracker/store/source/similarity.py +++ b/server/tracker/store/source/similarity.py @@ -421,6 +421,7 @@ async def similar_vector( params, cache_key=["vec", refs, before, repos, lim], budget_ms=int(settings.tracker_live_bisect_budget_ms), + vector=True, ) return [{**similar_row(r), "distance": float(r.get("distance") or 0.0)} for r in rows] diff --git a/tests/unit/snapshots/live_settings.json b/tests/unit/snapshots/live_settings.json index bacfac9b67..cd246e991b 100644 --- a/tests/unit/snapshots/live_settings.json +++ b/tests/unit/snapshots/live_settings.json @@ -391,6 +391,8 @@ "tracker_snapshot_ttl_s", "tracker_source_check_timeout_s", "tracker_test_candidates_timeout_s", + "vector_scan_max_concurrent_per_org", + "vector_scan_max_waiters_per_org", "worker_backoff_base_s", "worker_backoff_cap_s", "worker_partial_retry_s" diff --git a/tests/unit/test_vector_admission_boundaries.py b/tests/unit/test_vector_admission_boundaries.py new file mode 100644 index 0000000000..20c4ce9a0a --- /dev/null +++ b/tests/unit/test_vector_admission_boundaries.py @@ -0,0 +1,124 @@ +"""Scalar bookkeeping and consumers retain their own query attribution.""" + +from __future__ import annotations + +import asyncio +from typing import Any + +import pytest + +from server.core import ch as ch_module +from server.core import ch_schema_guard +from server.core.schema import status +from server.knowledge.store import source_candidates +from server.memory.app.search import ann_entity_boosts +from server.settings import settings +from tests.unit.test_vector_read_admission import Driver, Embedder, client + + +@pytest.mark.asyncio +async def test_nonvector_read_keeps_disabled_caller_shaped_deadline( + monkeypatch: pytest.MonkeyPatch, +) -> None: + ch, driver = client(monkeypatch) + monkeypatch.setattr(settings, "ch_caller_shaped_max_execution_s", 0.0) + driver.release.set() + assert await ch.query_all("SELECT 1") == [] + assert driver.started == 1 + + +class Stream: + def __init__(self) -> None: + self.opened = False + self.closed = False + self.sent = False + + async def __aenter__(self) -> Stream: + self.opened = True + return self + + async def __aexit__(self, *_args: Any) -> None: + self.closed = True + + def __aiter__(self) -> Stream: + return self + + async def __anext__(self) -> bytes: + if self.sent: + raise StopAsyncIteration + self.sent = True + return b'{"namespace":"private","chunk_idx":0}\n' + + +class StreamingDriver(Driver): + def __init__(self) -> None: + super().__init__() + self.stream = Stream() + self.stream_settings: dict[str, Any] = {} + + async def raw_stream(self, _sql: str, **kwargs: Any) -> Stream: + self.stream_settings = kwargs["settings"] + return self.stream + + +@pytest.mark.asyncio +async def test_mirror_stream_clears_only_consumer_attribution_and_closes_in_its_owner( + monkeypatch: pytest.MonkeyPatch, +) -> None: + ch, _ = client(monkeypatch) + driver = StreamingDriver() + driver.release.set() + monkeypatch.setattr(ch, "_client", driver) + async with source_candidates.candidates( + "SELECT namespace, cosineDistance(embedding, {q:Array(Float32)}) AS distance FROM loom.chunks", + {"org": "one-org", "nss": ["private"], "q": [1.0, 0.0]}, + ch=ch, + ) as rows: + assert await anext(rows) == {"namespace": "private", "chunk_idx": 0} + assert driver.stream.opened and not driver.stream.closed + assert await ch.query_all("SELECT 1") == [] + assert "max_rows_to_read" not in driver.settings[-1] + assert driver.stream.closed + assert driver.stream_settings["max_rows_to_read"] == 1_000_000 + assert driver.stream_settings["read_overflow_mode"] == "throw" + assert await ch.query_all("SELECT 1") == [] + assert "max_rows_to_read" not in driver.settings[-1] + + +@pytest.mark.asyncio +async def test_entity_boost_cold_schema_guard_keeps_ledger_scalar( + monkeypatch: pytest.MonkeyPatch, +) -> None: + ch, driver = client(monkeypatch) + driver.release.set() + monkeypatch.setattr(settings, "vector_scan_max_concurrent_per_org", 2) + monkeypatch.setattr(ch_module, "guard_pending_schema", ch_schema_guard.guard_pending_schema) + monkeypatch.setattr( + ch_schema_guard, "_schema_guard", {"at": None, "tables": {}, "refreshing": False} + ) + monkeypatch.setattr(ann_entity_boosts, "get_client", lambda: ch) + + async def cold_status() -> dict[str, Any]: + await ch.query_all("SELECT 1") + return {"pending": [], "applied": 1} + + monkeypatch.setattr(status, "schema_status", cold_status) + result = await asyncio.wait_for( + ann_entity_boosts._ann_entity_boosts( + org="one-org", + namespaces=["private"], + agent="a", + user_id="u", + query_entities=["entity"], + embedder=Embedder(), + ), + timeout=1, + ) + assert result == {} + assert driver.started == 2 + assert "max_rows_to_read" not in driver.settings[0] + assert driver.settings[1]["max_rows_to_read"] == 1_000_000 + assert driver.settings[1]["read_overflow_mode"] == "throw" + assert type(driver.settings[1]["max_rows_to_read"]) is int + assert type(driver.settings[1]["max_memory_usage"]) is int + assert driver.active == 0 diff --git a/tests/unit/test_vector_capacity_composition.py b/tests/unit/test_vector_capacity_composition.py new file mode 100644 index 0000000000..6c1e7078af --- /dev/null +++ b/tests/unit/test_vector_capacity_composition.py @@ -0,0 +1,176 @@ +"""Capacity refusals survive composed reads and deadline cancellation.""" + +from __future__ import annotations + +import asyncio +from types import SimpleNamespace +from typing import Any + +import pytest + +from server.code.app import search_legs +from server.core.errors import VectorInputTooLargeError +from server.memory.app.loom_search import fanout +from server.tracker.api.models import TrackerSimilarInput, TrackerSimilarOutput +from server.tracker.app import similar + + +@pytest.mark.asyncio +@pytest.mark.parametrize("capacity", [True, False], ids=["capacity-refusal", "ordinary-deadline"]) +async def test_code_model_deadline_keeps_a_capacity_failure_during_cancellation( + monkeypatch: pytest.MonkeyPatch, capacity: bool +) -> None: + monkeypatch.setattr(search_legs.settings, "code_search_llm_leg_grace_ms", 1) + started, stopped = asyncio.Event(), asyncio.Event() + failure = ( + VectorInputTooLargeError("deadline capacity refusal") + if capacity + else RuntimeError("ordinary optional model failure") + ) + + async def lexical() -> list[str]: + await started.wait() + return ["lexical-row"] + + async def model() -> list[str]: + started.set() + try: + await asyncio.Event().wait() + finally: + stopped.set() + raise failure + + result = SimpleNamespace(legs_timed_out=[], epistemic="", budget=None) + work: dict[str, Any] = {} + call = search_legs.run_legs( + {"lexical": (lexical(), []), "hyde": (model(), [])}, + result=result, + work=work, + deadline_ms=20, + ) + if capacity: + with pytest.raises(VectorInputTooLargeError) as caught: + await asyncio.wait_for(call, timeout=1) + assert caught.value is failure + assert result.legs_timed_out == [] + else: + assert await asyncio.wait_for(call, timeout=1) == [["lexical-row"], []] + assert result.legs_timed_out == ["hyde"] + assert result.epistemic == "partial" + assert work["legs_timed_out"] == 1 + assert started.is_set() and stopped.is_set() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("site", ["returned-memory", "knowledge-renewal"]) +@pytest.mark.parametrize("capacity", [True, False], ids=["capacity-refusal", "ordinary-fallback"]) +async def test_section_assembly_preserves_refusal_values_and_renewal_failures( + monkeypatch: pytest.MonkeyPatch, site: str, capacity: bool +) -> None: + failure = ( + VectorInputTooLargeError("section capacity refusal") + if capacity + else RuntimeError("ordinary unavailable section") + ) + renewed: list[bool] = [] + + async def namespaces(_org: str, ns: str, _scope: str) -> tuple[list[str], list[str]]: + return [ns], [] + + async def prepare(*_args: Any) -> Any: + return None, SimpleNamespace(marked=False) + + async def memory(*_args: Any, **_kwargs: Any) -> Any: + # A returned Exception is a possible gather result and must be inspected. + return failure if site == "returned-memory" else RuntimeError("ordinary memory failure") + + async def knowledge(*_args: Any) -> Any: + if site == "knowledge-renewal": + return object() + raise RuntimeError("ordinary knowledge failure") + + async def publish(*_args: Any) -> Any: + renewed.append(True) + raise failure + + async def empty(*_args: Any, **_kwargs: Any) -> list[Any]: + return [] + + async def conversation(*_args: Any) -> Any: + raise RuntimeError("ordinary conversation failure") + + monkeypatch.setattr("server.auth.fanout.namespaces_to_search", namespaces) + monkeypatch.setattr(fanout, "request_scope", lambda: None) + monkeypatch.setattr(fanout, "_prepare_knowledge", prepare) + monkeypatch.setattr(fanout, "memory_with_repo_decisions", memory) + monkeypatch.setattr(fanout, "_run_knowledge", knowledge) + monkeypatch.setattr(fanout, "_published_knowledge", publish) + monkeypatch.setattr(fanout, "search_profile", empty) + monkeypatch.setattr(fanout, "_conversation_leg", conversation) + monkeypatch.setattr(fanout, "tracker_leg", empty) + monkeypatch.setattr(fanout, "answer_first", lambda payload, *_a, **_k: payload) + call = fanout.search_sections( + org="org", + namespace="home", + agent="a", + query="question", + top_k=3, + tags=[], + namespace_scope="this", + recipe="", + bodies="excerpt", + page="", + explain=False, + repo_namespace="", + tracker_plan=None, + ) + if capacity: + with pytest.raises(VectorInputTooLargeError) as caught: + await asyncio.wait_for(call, timeout=1) + assert caught.value is failure + else: + payload = await asyncio.wait_for(call, timeout=1) + assert payload["memory"] == {"error": "RuntimeError", "results": []} + assert payload["knowledge"] == {"error": "RuntimeError", "results": []} + assert payload["profile"] == {"results": []} + assert payload["conversations"] == {"error": "RuntimeError", "results": []} + assert renewed == ([True] if site == "knowledge-renewal" else []) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("capacity", [True, False], ids=["capacity-refusal", "ordinary-fallback"]) +async def test_tracker_similar_secondary_namespace_keeps_capacity_refusal( + monkeypatch: pytest.MonkeyPatch, capacity: bool +) -> None: + failure = ( + VectorInputTooLargeError("secondary tracker capacity refusal") + if capacity + else RuntimeError("ordinary secondary tracker failure") + ) + addressed: list[str] = [] + + async def linked(_org: str, _namespace: str) -> list[str]: + return ["primary", "secondary"] + + async def one(inp: TrackerSimilarInput) -> TrackerSimilarOutput: + addressed.append(inp.namespace) + if inp.namespace == "secondary": + raise failure + return TrackerSimilarOutput(items=[], query={"known": True}, legs={"vector": "empty"}) + + monkeypatch.setattr(similar.common, "linked_trackers", linked) + monkeypatch.setattr(similar, "_similar_one", one) + call = similar.handle_tracker_similar( + TrackerSimilarInput(org="org", namespace="repository", text="question", top_k=3) + ) + if capacity: + with pytest.raises(VectorInputTooLargeError) as caught: + await asyncio.wait_for(call, timeout=1) + assert caught.value is failure + else: + out = await asyncio.wait_for(call, timeout=1) + assert out.namespaces == ["primary"] + assert out.query == {"known": True} + assert out.legs == {"vector": "empty"} + assert not out.tracker_unavailable + assert addressed == ["primary", "secondary"] diff --git a/tests/unit/test_vector_read_admission.py b/tests/unit/test_vector_read_admission.py new file mode 100644 index 0000000000..6c219fac6b --- /dev/null +++ b/tests/unit/test_vector_read_admission.py @@ -0,0 +1,306 @@ +"""Owning vector APIs retain bounded, complete read work.""" + +from __future__ import annotations + +import asyncio +from types import SimpleNamespace +from typing import Any + +import pytest + +from server.core import ch as ch_module +from server.core.errors import VectorInputTooLargeError +from server.knowledge.app.search import HybridSearchQuery, hybrid_search +from server.memory.app.search import ann_entity_boosts +from server.settings import settings + + +class Driver: + def __init__(self) -> None: + self.active = 0 + self.peak = 0 + self.started = 0 + self.release = asyncio.Event() + self.settings: list[dict[str, Any]] = [] + self.failure: Exception | None = None + + async def query(self, sql: str, **kwargs: Any) -> Any: + self.active += 1 + self.peak = max(self.peak, self.active) + self.started += 1 + self.settings.append(kwargs["settings"]) + try: + await self.release.wait() + if self.failure is not None: + raise self.failure + return SimpleNamespace(result_rows=[], column_names=[], summary={}) + finally: + self.active -= 1 + + +class Embedder: + async def embed(self, text: str) -> list[float]: + return [1.0, 0.0] + + +def client(monkeypatch: pytest.MonkeyPatch, *, pool: int = 64) -> tuple[ch_module.CHClient, Driver]: + monkeypatch.setattr(settings, "ch_pool_maxsize", pool) + monkeypatch.setattr(settings, "ch_pool_wait_ms", 5000) + + async def schema_ready(*args: Any, **kwargs: Any) -> None: + return None + + monkeypatch.setattr(ch_module, "guard_pending_schema", schema_ready) + ch = ch_module.CHClient() + driver = Driver() + monkeypatch.setattr(ch, "_client", driver) + return ch, driver + + +def query(org: str) -> HybridSearchQuery: + return HybridSearchQuery( + org=org, namespaces=("private",), embedding_model="test", query_vec=[1.0, 0.0], top_k=1 + ) + + +async def settled() -> None: + # Every fake driver suspends on an Event; no wall-clock race is required. + for _ in range(80): + await asyncio.sleep(0) + + +async def join(tasks: list[asyncio.Task[Any]], driver: Driver) -> list[Any]: + driver.release.set() + return await asyncio.gather(*tasks, return_exceptions=True) + + +@pytest.mark.asyncio +async def test_knowledge_read_limits_one_org_without_dropping_inputs( + monkeypatch: pytest.MonkeyPatch, +) -> None: + ch, driver = client(monkeypatch) + tasks = [asyncio.create_task(hybrid_search(query("one-org"), ch=ch)) for _ in range(24)] + try: + await settled() + assert driver.active == 16 + assert driver.peak == 16 + finally: + results = await join(tasks, driver) + assert results == [[]] * 24 + assert driver.started == 24 + assert driver.active == 0 + + +@pytest.mark.asyncio +async def test_two_axis_entity_boost_costs_two_units(monkeypatch: pytest.MonkeyPatch) -> None: + ch, driver = client(monkeypatch) + monkeypatch.setattr(ann_entity_boosts, "get_client", lambda: ch) + tasks = [ + asyncio.create_task( + ann_entity_boosts._ann_entity_boosts( + org="one-org", + namespaces=["private"], + agent="a", + user_id="u", + query_entities=[str(i)], + embedder=Embedder(), + ) + ) + for i in range(12) + ] + try: + await settled() + assert driver.active == 8 + assert driver.peak == 8 + finally: + results = await join(tasks, driver) + assert results == [{}] * 12 + assert driver.started == 12 + assert driver.active == 0 + + +@pytest.mark.asyncio +async def test_distinct_org_population_includes_existing_pool_waiters( + monkeypatch: pytest.MonkeyPatch, +) -> None: + ch, driver = client(monkeypatch) + tasks = [asyncio.create_task(hybrid_search(query(f"org-{i}"), ch=ch)) for i in range(577)] + try: + await settled() + assert driver.active == 64 + assert sum(not task.done() for task in tasks) == 512 + finally: + results = await join(tasks, driver) + refused = [result for result in results if isinstance(result, Exception)] + assert len(refused) == 65 + assert all(getattr(result, "code", "") == "OVERLOADED" for result in refused) + assert results.count([]) == 512 + assert driver.started == 512 + assert driver.active == 0 + + +@pytest.mark.asyncio +async def test_vector_read_refuses_disabled_pool_before_dispatch( + monkeypatch: pytest.MonkeyPatch, +) -> None: + ch, driver = client(monkeypatch, pool=0) + driver.release.set() + with pytest.raises(Exception) as caught: + await hybrid_search(query("one-org"), ch=ch) + assert getattr(caught.value, "code", "") == "OVERLOADED" + assert driver.started == 0 + + +@pytest.mark.asyncio +async def test_vector_native_rows_throw_cap_applies_without_traffic_class( + monkeypatch: pytest.MonkeyPatch, +) -> None: + ch, driver = client(monkeypatch) + driver.release.set() + assert await hybrid_search(query("one-org"), ch=ch) == [] + assert driver.settings[0].get("max_rows_to_read") == 1_000_000 + assert driver.settings[0].get("read_overflow_mode") == "throw" + assert 0 < driver.settings[0]["max_execution_time"] <= 30 + assert 0 < driver.settings[0]["max_memory_usage"] <= 4_000_000_000 + + +@pytest.mark.asyncio +async def test_entity_boost_does_not_turn_typed_refusal_into_empty_ranking( + monkeypatch: pytest.MonkeyPatch, +) -> None: + ch, driver = client(monkeypatch) + driver.release.set() + driver.failure = VectorInputTooLargeError("bounded test refusal") + monkeypatch.setattr(ann_entity_boosts, "get_client", lambda: ch) + with pytest.raises(VectorInputTooLargeError, match="bounded test refusal"): + await ann_entity_boosts._ann_entity_boosts( + org="one-org", + namespaces=["private"], + agent="a", + user_id="u", + query_entities=["entity"], + embedder=Embedder(), + ) + assert driver.active == 0 + + +@pytest.mark.asyncio +async def test_entity_boost_keeps_ordinary_optional_failure( + monkeypatch: pytest.MonkeyPatch, +) -> None: + ch, driver = client(monkeypatch) + driver.release.set() + driver.failure = RuntimeError("ordinary optional failure") + monkeypatch.setattr(ann_entity_boosts, "get_client", lambda: ch) + assert ( + await ann_entity_boosts._ann_entity_boosts( + org="one-org", + namespaces=["private"], + agent="a", + user_id="u", + query_entities=["entity"], + embedder=Embedder(), + ) + == {} + ) + assert driver.active == 0 + + +@pytest.mark.asyncio +async def test_nonvector_read_retains_existing_pool_behavior( + monkeypatch: pytest.MonkeyPatch, +) -> None: + ch, driver = client(monkeypatch, pool=0) + driver.release.set() + assert await ch.query_all("SELECT 1") == [] + assert driver.started == 1 + + +class NativeBudgetError(RuntimeError): + def __init__(self, code: int) -> None: + super().__init__(f"Code: {code}. Native budget test refusal") + self.code = code + + +@pytest.mark.asyncio +@pytest.mark.parametrize("code,reason", [(158, "rows"), (241, "memory")]) +async def test_entity_boost_native_budget_is_a_complete_failure( + monkeypatch: pytest.MonkeyPatch, code: int, reason: str +) -> None: + ch, driver = client(monkeypatch) + driver.release.set() + driver.failure = NativeBudgetError(code) + monkeypatch.setattr(ann_entity_boosts, "get_client", lambda: ch) + with pytest.raises(Exception) as caught: + await ann_entity_boosts._ann_entity_boosts( + org="one-org", + namespaces=["private"], + agent="a", + user_id="u", + query_entities=["entity"], + embedder=Embedder(), + ) + assert getattr(caught.value, "code", "") == "VECTOR_CAPACITY_REFUSED" + assert getattr(caught.value, "extra", {}).get("reason") == reason + assert driver.active == 0 + + +@pytest.mark.asyncio +async def test_selected_bulk_check_refusal_is_not_clear(monkeypatch: pytest.MonkeyPatch) -> None: + from server.memory.app import bulk_set, conflicts + + async def policy(*args: Any, **kwargs: Any) -> str: + return "surface" + + async def refused(*args: Any, **kwargs: Any) -> Any: + raise VectorInputTooLargeError("selected conflict read refused") + + monkeypatch.setattr(settings, "memory_bulk_peer_read", "per_fact") + monkeypatch.setattr(bulk_set, "_namespace_policy", policy) + monkeypatch.setattr(conflicts, "check_on_commit", refused) + input = SimpleNamespace(org="one-org", namespace="private", agent="a", user_id="u") + row = {"memory_type": "semantic", "memory_key": "fact"} + entry = SimpleNamespace(memory_type="semantic", memory_key="fact") + with pytest.raises(VectorInputTooLargeError, match="selected conflict read refused"): + await bulk_set._bulk_conflict_verdicts( + input, + [row], + [(entry, "value", None, None, None)], + [(0, None, None, None)], + [[1.0, 0.0]], + floors={"semantic": (0.8, 0.8)}, + ) + + +@pytest.mark.asyncio +async def test_postcommit_refusal_preserves_versions_and_reports_optional_work( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from server.memory.app import bulk_set, key_supersession, value_updates + + async def no_retirement(**kwargs: Any) -> None: + return None + + async def refused(**kwargs: Any) -> Any: + raise VectorInputTooLargeError("optional linking read refused") + + monkeypatch.setattr(key_supersession, "supersede_same_key_rows", no_retirement) + monkeypatch.setattr(value_updates, "link_value_updates", refused) + input = SimpleNamespace(org="one-org", namespace="private", agent="a", detect_updates=True) + row = { + "org": "one-org", + "namespace": "private", + "agent": "a", + "user_id": "u", + "session_id": "s", + "memory_key": "fact", + "version": 7, + "status": "active", + } + result = SimpleNamespace(memory_key="fact", version=7, next="Existing next.") + state = SimpleNamespace(memory_rows=[row], embedding_rows=[], results=[result]) + assert await bulk_set._bulk_post_insert_links(None, input, state, []) == [] + assert result.version == 7 + assert result.next.startswith("Existing next.") + assert "vector capacity" in result.next + assert "committed" in result.next From 318efe894c3c8f3dacdd53683600b0f57b76a16a Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Wed, 18 Nov 2026 04:18:28 +0000 Subject: [PATCH 02/28] Supplemental reads join siblings before returning failures Keep the existing concurrent launch and result order while canceling and shield-joining owned namespace, phrase and subquery tasks. Preserve the first refusal through repeated cancellation. Signed-off-by: zlareb1 --- server/memory/app/search/supplemental.py | 28 +++- tests/unit/test_supplemental_owned_reads.py | 160 ++++++++++++++++++++ 2 files changed, 185 insertions(+), 3 deletions(-) create mode 100644 tests/unit/test_supplemental_owned_reads.py diff --git a/server/memory/app/search/supplemental.py b/server/memory/app/search/supplemental.py index abf9c96d07..900f002b1d 100644 --- a/server/memory/app/search/supplemental.py +++ b/server/memory/app/search/supplemental.py @@ -5,10 +5,12 @@ import asyncio import logging +from collections.abc import Awaitable from dataclasses import replace from typing import Any from server.core.errors import VectorCapacityError +from server.core.shielded import to_its_end from server.memory.api.models_search import _MemorySearchHit from server.memory.app.search.context import SearchContext from server.memory.app.search.vector_legs import _search_query_for, _wire_hit @@ -17,6 +19,26 @@ log = logging.getLogger(__package__) +async def _supplemental_reads[T](*reads: Awaitable[T]) -> list[T]: + """Join supplemental siblings on failure without changing successful scheduling.""" + tasks = [asyncio.ensure_future(read) for read in reads] + try: + return await asyncio.gather(*tasks) + except BaseException: + for task in tasks: + task.cancel() + + async def stopped() -> None: + await asyncio.gather(*tasks, return_exceptions=True) + + try: + await to_its_end(stopped(), name="supplemental-read-cleanup") + except asyncio.CancelledError: + # The first failed read stays the answer after every sibling retires. + pass + raise + + async def _supplemental_leg( ctx: SearchContext, sub_q: str, @@ -36,7 +58,7 @@ async def _supplemental_leg( except Exception: # noqa: BLE001 — a failed sub-query embed drops only that sub-query return [] ctx.actual_work["vector_queries"] = int(ctx.actual_work["vector_queries"]) + len(ctx.namespaces) - sub_per_ns = await asyncio.gather( + sub_per_ns = await _supplemental_reads( *[ vector_search( replace( @@ -93,7 +115,7 @@ async def _run_set_expansion(ctx: SearchContext) -> None: default=1.0, ) try: - per_phrase = await asyncio.gather( + per_phrase = await _supplemental_reads( *[_supplemental_leg(ctx, ph, leg_top_k, seen_keys) for ph in phrases] ) except VectorCapacityError: @@ -152,7 +174,7 @@ async def _run_sub_queries(ctx: SearchContext) -> None: else: sub_top_k = max(5, input.top_k // max(1, len(ctx.sub_queries))) - sub_results_by_query = await asyncio.gather( + sub_results_by_query = await _supplemental_reads( *[_supplemental_leg(ctx, sq, sub_top_k, seen_keys) for sq in ctx.sub_queries] ) for sub_hits in sub_results_by_query: diff --git a/tests/unit/test_supplemental_owned_reads.py b/tests/unit/test_supplemental_owned_reads.py new file mode 100644 index 0000000000..7cc4d47ed4 --- /dev/null +++ b/tests/unit/test_supplemental_owned_reads.py @@ -0,0 +1,160 @@ +"""Supplemental read scopes retire every sibling before returning a refusal.""" + +from __future__ import annotations + +import asyncio +from collections.abc import Awaitable +from typing import Any + +import pytest + +from server.core.errors import VectorAdmissionError +from server.memory.app import retrieval +from server.memory.app.search import supplemental +from tests.unit.test_supplemental_leg_shares_the_main_leg import _ctx + + +class _ReadGroup: + def __init__(self) -> None: + self.started = asyncio.Event() + self.release = asyncio.Event() + self.never = asyncio.Event() + self.children: list[asyncio.Task[Any]] = [] + self.cleaning: set[str] = set() + self.retired: set[str] = set() + self.failure = VectorAdmissionError("first supplemental refusal", reason="org_queue") + + async def read(self, name: str) -> list[Any]: + child = asyncio.current_task() + assert child is not None + self.children.append(child) + if len(self.children) == 3: + self.started.set() + if name == "first": + await self.started.wait() + raise self.failure + try: + await self.never.wait() + finally: + self.cleaning.add(name) + await self.release.wait() + self.retired.add(name) + return [] + + +async def _observe_cleanup(caller: asyncio.Task[Any], ready: Any) -> None: + for _ in range(50): + if caller.done() or ready(): + return + await asyncio.sleep(0) + + +def _owning_read(monkeypatch: pytest.MonkeyPatch, group: _ReadGroup, owner: str) -> Awaitable[Any]: + ctx = _ctx() + ctx.all_hits = [] + if owner == "namespace": + ctx.namespaces = ["first", "second", "third"] + + async def search(query: Any) -> list[Any]: + return await group.read(query.namespace) + + monkeypatch.setattr(retrieval, "vector_search", search) + return supplemental._supplemental_leg(ctx, "sub q", 7, set()) + + async def phrase(_ctx: Any, text: str, _top_k: int, _seen: Any) -> list[Any]: + return await group.read(text) + + monkeypatch.setattr(supplemental, "_supplemental_leg", phrase) + if owner == "phrase": + ctx.set_expansion_phrases = ["first", "second", "third"] + return supplemental._run_set_expansion(ctx) + ctx.sub_queries = ["first", "second", "third"] + ctx.query_shape = "simple" + return supplemental._run_sub_queries(ctx) + + +@pytest.mark.parametrize("owner", ["namespace", "phrase", "subquery"]) +@pytest.mark.parametrize("repeat_cancel", [False, True], ids=["refusal", "cancel-during-cleanup"]) +async def test_supplemental_refusal_retires_owned_siblings( + monkeypatch: pytest.MonkeyPatch, owner: str, repeat_cancel: bool +) -> None: + group = _ReadGroup() + caller = asyncio.ensure_future(_owning_read(monkeypatch, group, owner)) + try: + await group.started.wait() + await _observe_cleanup(caller, lambda: len(group.cleaning) == 2) + if repeat_cancel: + caller.cancel() + await asyncio.sleep(0) + caller.cancel() + group.release.set() + with pytest.raises(VectorAdmissionError) as raised: + await caller + assert raised.value is group.failure + assert len(group.children) == 3 + assert all(child.done() for child in group.children) + assert group.retired == {"second", "third"} + finally: + group.release.set() + for child in group.children: + if not child.done(): + child.cancel() + await asyncio.gather(caller, *group.children, return_exceptions=True) + + +async def test_nested_subquery_refusal_survives_repeated_outer_cleanup_cancellation( + monkeypatch: pytest.MonkeyPatch, +) -> None: + ctx = _ctx() + ctx.all_hits = [] + ctx.namespaces = ["ns0", "ns1"] + ctx.sub_queries = ["failed", "sibling"] + ctx.query_shape = "simple" + started, release_inner, release_outer, never = (asyncio.Event() for _ in range(4)) + children: list[asyncio.Task[Any]] = [] + cleaning: set[tuple[str, str]] = set() + retired: set[tuple[str, str]] = set() + failure = VectorAdmissionError("first nested refusal", reason="org_queue") + + async def search(query: Any) -> list[Any]: + child = asyncio.current_task() + assert child is not None + children.append(child) + if len(children) == 4: + started.set() + key = (query.query_text, query.namespace) + if key == ("failed", "ns0"): + await started.wait() + raise failure + try: + await never.wait() + finally: + cleaning.add(key) + await (release_inner if query.query_text == "failed" else release_outer).wait() + retired.add(key) + return [] + + monkeypatch.setattr(retrieval, "vector_search", search) + caller = asyncio.create_task(supplemental._run_sub_queries(ctx)) + try: + await started.wait() + await _observe_cleanup(caller, lambda: ("failed", "ns1") in cleaning) + release_inner.set() + await _observe_cleanup(caller, lambda: len(cleaning) == 3) + caller.cancel() + await asyncio.sleep(0) + caller.cancel() + release_outer.set() + with pytest.raises(VectorAdmissionError) as raised: + await caller + assert raised.value is failure + assert len(children) == 4 and all(child.done() for child in children) + assert retired == {("failed", "ns1"), ("sibling", "ns0"), ("sibling", "ns1")} + assert ctx.all_hits == [] + finally: + release_inner.set() + release_outer.set() + for child in children: + if not child.done(): + child.cancel() + await asyncio.gather(caller, *children, return_exceptions=True) From 86e6fd4550acb40aa3fbb9bd532479a2f135a957 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Wed, 18 Nov 2026 04:43:02 +0000 Subject: [PATCH 03/28] Attribute composed vector reads and retire their children Keep native capacity refusals visible through neighbor and promotion scans. Retire card and duplicate-candidate readers before a failed owner returns. Signed-off-by: zlareb1 --- server/code/app/search/cards.py | 33 +- server/memory/app/conflicts.py | 32 +- server/memory/app/retrieval/chunks.py | 8 +- server/tracker/app/dupcheck_candidates.py | 32 +- .../test_vector_admission_review_edges.py | 289 ++++++++++++++++++ 5 files changed, 370 insertions(+), 24 deletions(-) create mode 100644 tests/unit/test_vector_admission_review_edges.py diff --git a/server/code/app/search/cards.py b/server/code/app/search/cards.py index 2324b50efb..86ef05c9fa 100644 --- a/server/code/app/search/cards.py +++ b/server/code/app/search/cards.py @@ -9,6 +9,7 @@ import logging import re import time +from collections.abc import Awaitable from typing import Any from server.code.app.search.lexical import _lexical_terms @@ -18,12 +19,36 @@ from server.code.domain.search_scope import SearchScope from server.code.store.index import CodeRepo from server.core.errors import VectorCapacityError +from server.core.shielded import to_its_end from server.settings import settings from server.settings.budgets import llm_max_tokens logger = logging.getLogger(__name__) +async def _card_reads[T]( + *reads: Awaitable[T], source: asyncio.Future[Any] | None = None +) -> list[T]: + """Retire both card legs and their source before a failed read escapes.""" + tasks = [asyncio.ensure_future(read) for read in reads] + owned = [*tasks, *([source] if source is not None else [])] + try: + return await asyncio.gather(*tasks) + except BaseException: + for task in owned: + task.cancel() + + async def stopped() -> None: + await asyncio.gather(*owned, return_exceptions=True) + + try: + await to_its_end(stopped(), name="card-read-cleanup") + except asyncio.CancelledError: + # Cleanup cancellation does not replace the first failed read. + pass + raise + + async def _card_leg_rows( repo: CodeRepo, org: str, @@ -287,13 +312,9 @@ async def _cards() -> list[dict[str, Any]]: cards=(await asyncio.shield(source)) if shared else None, ) - try: - cards, concepts = await asyncio.gather(_cards(), _concepts()) - finally: - if not source.done(): - source.cancel() + cards, concepts = await _card_reads(_cards(), _concepts(), source=source) return cards, concepts - cards, concepts = await asyncio.gather( + cards, concepts = await _card_reads( _card_leg_rows( repo, org, diff --git a/server/memory/app/conflicts.py b/server/memory/app/conflicts.py index c047021310..1449cf8e65 100644 --- a/server/memory/app/conflicts.py +++ b/server/memory/app/conflicts.py @@ -235,7 +235,8 @@ async def _chunk(dim: int, idxs: list[int]) -> None: async def _read(sql: str) -> list[dict[str, Any]]: with vector_work(org, namespace, len(idxs)): - return await ch.query_all(sql, params=params) + rows: list[dict[str, Any]] = await ch.query_all(sql, params=params) + return rows sql = candidate_sql_many(len(idxs), scope_and(scope, params, "m")) # Pinned peers retain their independent namespace-wide statement. @@ -936,14 +937,15 @@ async def check_against_shared( dim = len(embedding) # equal-length filter; see check_on_commit ch = get_client() - rows = await ch.query_all( - """ + with vector_work(org, namespace): + rows = await ch.query_all( + """ SELECT s.memory_key, s.value, """ - + "cosineDistance(e.embedding, {emb:Array(Float32)})" - + """ AS dist + + "cosineDistance(e.embedding, {emb:Array(Float32)})" + + """ AS dist FROM loom.shared s FINAL JOIN loom.memory_embeddings e FINAL ON e.org = s.org AND e.namespace = s.namespace @@ -961,16 +963,16 @@ async def check_against_shared( ORDER BY dist ASC LIMIT 5 """, - params={ - "org": org, - "ns": namespace, - "mt": memory_type, - "self_key": target_key, - "dim": dim, - "emb": embedding, - "max_dist": max_distance, - }, - ) + params={ + "org": org, + "ns": namespace, + "mt": memory_type, + "self_key": target_key, + "dim": dim, + "emb": embedding, + "max_dist": max_distance, + }, + ) return rows diff --git a/server/memory/app/retrieval/chunks.py b/server/memory/app/retrieval/chunks.py index 35395f590f..1f3dab71fb 100644 --- a/server/memory/app/retrieval/chunks.py +++ b/server/memory/app/retrieval/chunks.py @@ -631,7 +631,10 @@ async def _chunk_anchor_windows( fallback_anchors, chunk_agent_pred=chunk_agent_pred, base_params=chunk_params ) try: - neighbor_rows.extend(await _chunk_query(ch, neighbor_sql, neighbor_params)) + with vector_work(chunk_params["org"], chunk_params["ns"]): + neighbor_rows.extend(await _chunk_query(ch, neighbor_sql, neighbor_params)) + except VectorCapacityError: + raise except Exception as e: # noqa: BLE001 — neighbour chunks are optional context around the hits log.warning("chunk leg: neighbour window read failed (served without): %s", e) return neighbor_rows @@ -787,7 +790,8 @@ async def _search_memory_chunks( neighbors=neighbors, ) try: - rows = await _chunk_query(ch, sql, chunk_params) + with vector_work(q.org, q.namespace, int(neighbors)): + rows = await _chunk_query(ch, sql, chunk_params) except VectorCapacityError: raise except Exception: diff --git a/server/tracker/app/dupcheck_candidates.py b/server/tracker/app/dupcheck_candidates.py index 1ea6a6578d..101f306d9c 100644 --- a/server/tracker/app/dupcheck_candidates.py +++ b/server/tracker/app/dupcheck_candidates.py @@ -8,9 +8,11 @@ import asyncio import logging import time +from collections.abc import Awaitable from datetime import datetime from typing import Any +from server.core.shielded import to_its_end from server.settings import settings from server.tracker.api.models import ( TrackerSimilarInput, @@ -37,6 +39,34 @@ def _max_length(field: Any) -> int: return next(int(m.max_length) for m in field.metadata if getattr(m, "max_length", None)) +async def _candidate_reads[A, B, C, D]( + legs: Awaitable[A], record: Awaitable[B], flight: Awaitable[C], fingerprint: Awaitable[D] +) -> tuple[A, B, C, D]: + """Retire all four candidate readers before the first failed read escapes.""" + tasks = ( + asyncio.ensure_future(legs), + asyncio.ensure_future(record), + asyncio.ensure_future(flight), + asyncio.ensure_future(fingerprint), + ) + try: + a, b, c, d = await asyncio.gather(*tasks) + return a, b, c, d + except BaseException: + for task in tasks: + task.cancel() + + async def stopped() -> None: + await asyncio.gather(*tasks, return_exceptions=True) + + try: + await to_its_end(stopped(), name="duplicate-candidate-cleanup") + except asyncio.CancelledError: + # Cleanup cancellation does not replace the first failed read. + pass + raise + + async def candidates( src: Any, input: Any, @@ -72,7 +102,7 @@ async def candidates( await _resolve_error_name(src, input, q, meta) draft_paths = _draft_paths(q, input) errors: dict[str, BaseException] = {} - lists, record, in_flight, fingerprint = await asyncio.gather( + lists, record, in_flight, fingerprint = await _candidate_reads( tsim._similar_legs( src, sim_input, diff --git a/tests/unit/test_vector_admission_review_edges.py b/tests/unit/test_vector_admission_review_edges.py new file mode 100644 index 0000000000..f3a719856b --- /dev/null +++ b/tests/unit/test_vector_admission_review_edges.py @@ -0,0 +1,289 @@ +"""Vector dispatches and composed reads keep capacity attribution and child ownership.""" + +from __future__ import annotations + +import asyncio +from types import SimpleNamespace +from typing import Any + +import pytest + +from server.code.app.search import cards +from server.core.errors import VectorAdmissionError, VectorCapacityError +from server.core.vectors.admission import current_work +from server.memory.app import conflicts +from server.memory.app.retrieval import chunks +from server.settings import settings +from server.tracker.api.models import TrackerDuplicateCheckInput +from server.tracker.app import dupcheck_candidates as candidates +from tests.unit.test_memory_search_service import _q +from tests.unit.test_vector_read_admission import Driver, NativeBudgetError, client + + +class _DispatchDriver(Driver): + def __init__(self, rows: list[list[dict[str, Any]]]) -> None: + super().__init__() + self.rows = rows + self.work: list[Any] = [] + self.failure_at = 0 + + async def query(self, sql: str, **kwargs: Any) -> Any: + self.started += 1 + self.settings.append(kwargs["settings"]) + self.work.append(current_work()) + if self.started == self.failure_at: + raise self.failure or RuntimeError("configured test failure") + rows = self.rows.pop(0) if self.rows else [] + names = list(rows[0]) if rows else [] + return SimpleNamespace( + result_rows=[tuple(row[name] for name in names) for row in rows], + column_names=names, + summary={}, + ) + + +def _dispatch(monkeypatch: pytest.MonkeyPatch, rows: list[list[dict[str, Any]]]): + ch, _ = client(monkeypatch) + driver = _DispatchDriver(rows) + monkeypatch.setattr(ch, "_client", driver) + return ch, driver + + +def _assert_vector(driver: _DispatchDriver, index: int, org: str, namespace: str) -> None: + work = driver.work[index] + assert work is not None + assert (work.org, work.namespaces, work.weight) == (org, (namespace,), 1) + assert driver.settings[index]["max_rows_to_read"] == 1_000_000 + assert driver.settings[index]["read_overflow_mode"] == "throw" + + +@pytest.mark.parametrize("text", ["neighbor words", ""], ids=["neighbor-vector", "keyed-only"]) +async def test_chunk_stage_two_attributes_only_its_actual_neighbor_cosine( + monkeypatch: pytest.MonkeyPatch, text: str +) -> None: + ch, driver = _dispatch( + monkeypatch, [[{"session_id": "session", "chunk_index": 3, "distance": 0.1}], []] + ) + q = _q(entity_filter=[], query_text=text) + assert await chunks._search_memory_chunks(ch, q, {"q": q.query_vec}) == [] + assert driver.started == 2 + _assert_vector(driver, 0, q.org, q.namespace) + if text: + _assert_vector(driver, 1, q.org, q.namespace) + else: + assert driver.work[1] is None + assert "max_rows_to_read" not in driver.settings[1] + + +async def test_chunk_stage_two_native_refusal_does_not_become_an_empty_leg(monkeypatch) -> None: + ch, driver = _dispatch( + monkeypatch, [[{"session_id": "session", "chunk_index": 3, "distance": 0.1}]] + ) + driver.failure_at = 2 + driver.failure = NativeBudgetError(158) + q = _q(entity_filter=[], query_text="neighbor words") + with pytest.raises(VectorCapacityError) as raised: + await chunks._search_memory_chunks(ch, q, {"q": q.query_vec}) + assert raised.value.extra["reason"] == "rows" + assert driver.started == 2 + + +@pytest.mark.parametrize("outcome", ["success", "native", "ordinary"]) +async def test_chunk_fallback_window_keeps_vector_budget_and_ordinary_fallback( + monkeypatch: pytest.MonkeyPatch, outcome: str +) -> None: + ch, driver = _dispatch(monkeypatch, [[]]) + if outcome != "success": + driver.failure_at = 1 + driver.failure = NativeBudgetError(158) if outcome == "native" else RuntimeError("optional") + + async def run(): + return await chunks._chunk_anchor_windows( + ch, + [("session", 3)], + prefetched=[], + window_rows=[], + chunk_agent_pred="", + chunk_params={"q": [0.1], "org": "one-org", "ns": "private", "model": "fake"}, + ) + + if outcome == "native": + with pytest.raises(VectorCapacityError) as raised: + await run() + assert raised.value.extra["reason"] == "rows" + else: + assert await run() == [] + if outcome == "success": + _assert_vector(driver, 0, "one-org", "private") + assert driver.started == 1 + + +@pytest.mark.parametrize("outcome", ["success", "native", "empty"]) +async def test_promote_shared_peer_scan_is_one_vector_probe(monkeypatch, outcome: str) -> None: + ch, driver = _dispatch(monkeypatch, [[]]) + monkeypatch.setattr(conflicts, "get_client", lambda: ch) + + async def threshold(*_args, **_kwargs): + return 0.8 + + monkeypatch.setattr(conflicts, "_resolve_threshold", threshold) + if outcome == "native": + driver.failure_at = 1 + driver.failure = NativeBudgetError(158) + + async def run(): + return await conflicts.check_against_shared( + org="one-org", + namespace="private", + target_key="new-key", + memory_type="semantic", + embedding=[] if outcome == "empty" else [0.1], + ) + + if outcome == "native": + with pytest.raises(Exception) as raised: + await run() + assert isinstance(raised.value, VectorCapacityError) + assert raised.value.extra["reason"] == "rows" + else: + assert await run() == [] + if outcome == "success": + _assert_vector(driver, 0, "one-org", "private") + assert driver.started == (0 if outcome == "empty" else 1) + + +class _OwnedReads: + def __init__(self, count: int) -> None: + self.count = count + self.started = asyncio.Event() + self.release = asyncio.Event() + self.never = asyncio.Event() + self.children: list[asyncio.Task[Any]] = [] + self.cleaning: set[str] = set() + self.retired: set[str] = set() + self.failure = VectorAdmissionError("first composed refusal", reason="org_queue") + + async def read(self, name: str, *, fail: bool) -> Any: + task = asyncio.current_task() + assert task is not None + self.children.append(task) + if len(self.children) == self.count: + self.started.set() + if fail: + await self.started.wait() + raise self.failure + try: + await self.never.wait() + finally: + self.cleaning.add(name) + await self.release.wait() + self.retired.add(name) + return [] + + +async def _refusal_joins(coro: Any, group: _OwnedReads, repeat_cancel: bool) -> None: + caller = asyncio.ensure_future(coro) + try: + await group.started.wait() + for _ in range(60): + if caller.done() or len(group.cleaning) == group.count - 1: + break + await asyncio.sleep(0) + if repeat_cancel: + caller.cancel() + await asyncio.sleep(0) + caller.cancel() + group.release.set() + with pytest.raises(VectorCapacityError) as raised: + await caller + assert raised.value is group.failure + assert len(group.children) == group.count + assert all(child.done() for child in group.children) + assert len(group.retired) == group.count - 1 + finally: + group.release.set() + for child in group.children: + if not child.done(): + child.cancel() + await asyncio.gather(caller, *group.children, return_exceptions=True) + + +@pytest.mark.parametrize("branch", ["source-refuses", "source-owned", "direct-concept"]) +@pytest.mark.parametrize("repeat_cancel", [False, True], ids=["refusal", "cleanup-cancel"]) +async def test_code_card_owner_joins_both_branches_and_explicit_source( + monkeypatch: pytest.MonkeyPatch, branch: str, repeat_cancel: bool +) -> None: + group = _OwnedReads(2) + monkeypatch.setattr(settings, "code_search_card_leg_mode", "skeleton") + + class Repo: + async def vector_search_cards(self, *_args, skip_tests=False, **_kwargs): + fail = skip_tests if branch == "source-refuses" else not skip_tests + return await group.read("source" if skip_tests else "cards", fail=fail) + + if branch == "direct-concept": + + async def concept(*_args, **_kwargs): + return await group.read("concept", fail=False) + + monkeypatch.setattr(cards, "_concept_leg_rows", concept) + + async def vector(): + return [0.1] + + await _refusal_joins( + cards._card_and_concept_rows( + Repo(), + "one-org", + "private", + "fake", + query="query", + query_vec=vector, + use_cards=True, + card_gate=0.0, + concept_model="" if branch == "direct-concept" else "cards:5", + path_prefix="", + scope=None, + work={}, + card_stats={"files": 0}, + concept_stats={"terms": []}, + ), + group, + repeat_cancel, + ) + + +@pytest.mark.parametrize("repeat_cancel", [False, True], ids=["refusal", "cleanup-cancel"]) +async def test_duplicate_candidates_join_all_four_owned_reads(monkeypatch, repeat_cancel) -> None: + group = _OwnedReads(4) + + async def query(*_args, **_kwargs): + return SimpleNamespace(), {}, None + + async def resolve(*_args, **_kwargs): + return None + + async def legs(*_args, **_kwargs): + return await group.read("legs", fail=True) + + async def record(*_args, **_kwargs): + return await group.read("record", fail=False) + + async def flight(*_args, **_kwargs): + return await group.read("flight", fail=False) + + async def fingerprint(*_args, **_kwargs): + return await group.read("fingerprint", fail=False) + + monkeypatch.setattr(candidates.tsim, "_similar_query", query) + monkeypatch.setattr(candidates, "_shape_query", lambda *_args: None) + monkeypatch.setattr(candidates, "_resolve_error_name", resolve) + monkeypatch.setattr(candidates, "_draft_paths", lambda *_args: []) + monkeypatch.setattr(candidates.tsim, "_similar_legs", legs) + monkeypatch.setattr(candidates.tsim, "_similar_record", record) + monkeypatch.setattr(candidates, "_fix_in_flight_leg", flight) + monkeypatch.setattr(candidates, "_fingerprint_leg", fingerprint) + draft = TrackerDuplicateCheckInput(org="one-org", namespace="qa-example", title="draft") + await _refusal_joins( + candidates.candidates(None, draft, wall=None, status={}), group, repeat_cancel + ) From 69e7c3e0961e36afda80e83c754b34c37acdbe60 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Wed, 18 Nov 2026 04:50:07 +0000 Subject: [PATCH 04/28] Pin vector admission defaults in the settings snapshot Record the three new defaults through the existing snapshot updater without changing any pinned value. Signed-off-by: zlareb1 --- tests/unit/snapshots/settings_defaults.json | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/unit/snapshots/settings_defaults.json b/tests/unit/snapshots/settings_defaults.json index 2ad349f8d4..cd17c49ddf 100644 --- a/tests/unit/snapshots/settings_defaults.json +++ b/tests/unit/snapshots/settings_defaults.json @@ -1275,6 +1275,9 @@ "usage_fold_enabled": true, "usage_fold_interval_seconds": 300, "vector_codes_state_probe_seconds": 60, + "vector_scan_max_concurrent_per_org": 16, + "vector_scan_max_rows_to_read": 1000000, + "vector_scan_max_waiters_per_org": 256, "vector_search_quantized_codes": true, "vector_search_quantized_fetch_multiplier": 3.0, "web_workers": 1, From c55c526720a6fec27bdfbe33a08ab3c55dc95552 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Wed, 18 Nov 2026 05:38:56 +0000 Subject: [PATCH 05/28] Account for resolver scans and classify vector dispatches Signed-off-by: zlareb1 --- server/memory/store/conflict_resolver.py | 3 +- tests/unit/test_vector_admission_resolver.py | 62 ++ tests/unit/test_vector_dispatch_contract.py | 1009 ++++++++++++++++++ 3 files changed, 1073 insertions(+), 1 deletion(-) create mode 100644 tests/unit/test_vector_admission_resolver.py create mode 100644 tests/unit/test_vector_dispatch_contract.py diff --git a/server/memory/store/conflict_resolver.py b/server/memory/store/conflict_resolver.py index 8d0795c0cf..3b359e0182 100644 --- a/server/memory/store/conflict_resolver.py +++ b/server/memory/store/conflict_resolver.py @@ -388,7 +388,8 @@ async def cluster_peers( near = [ c for c in await ch.query_all(_nearest_sql(), params=params) if 1 - c["dist"] >= floor ] - anchored = await ch.query_all(_ANCHORED_SQL, params={**params, "same_lim": half}) + with vector_work(str(row["org"]), str(row["namespace"]), 2): + anchored = await ch.query_all(_ANCHORED_SQL, params={**params, "same_lim": half}) above_floor = {key_identity(c) for c in near} found = {key_identity(c): c for c in [*anchored, *near]} if not found: diff --git a/tests/unit/test_vector_admission_resolver.py b/tests/unit/test_vector_admission_resolver.py new file mode 100644 index 0000000000..e743e1e932 --- /dev/null +++ b/tests/unit/test_vector_admission_resolver.py @@ -0,0 +1,62 @@ +"""Resolver cluster reads account for each actual vector scan branch.""" + +from __future__ import annotations + +import pytest + +from server.core.errors import VectorCapacityError +from server.memory.store import conflict_resolver as queue +from tests.unit.test_conflict_resolver import _near, _peer_row, _q +from tests.unit.test_vector_admission_review_edges import _dispatch +from tests.unit.test_vector_read_admission import NativeBudgetError + + +async def test_resolver_cluster_accounts_for_nearest_and_both_anchored_branches( + monkeypatch, +) -> None: + near = _near("s2", "near-key", 0.1) + anchored = _near("s3", "office_city", 0.2) + rows = [ + _peer_row("near", "s2", "near-key", "near value"), + _peer_row("anchor", "s3", "office_city", "anchored value"), + ] + ch, driver = _dispatch(monkeypatch, [[near], [anchored], rows]) + + result = await queue.cluster_peers(_q(), floor=0.6, limit=8, ch=ch) + + assert result == queue.Cluster( + [{**rows[0], "similarity": 0.9}, {**rows[1], "similarity": 0.8}], frozenset() + ) + assert driver.started == 3 + assert [work.weight if work is not None else None for work in driver.work] == [1, 2, None] + for index in (0, 1): + work = driver.work[index] + assert work is not None + assert (work.org, work.namespaces) == ("o", ("ns",)) + assert driver.settings[index]["max_rows_to_read"] == 1_000_000 + assert driver.settings[index]["read_overflow_mode"] == "throw" + assert "max_rows_to_read" not in driver.settings[2] + + +@pytest.mark.parametrize("failure_at", [1, 2], ids=["nearest", "anchored"]) +async def test_resolver_cluster_native_rows_refusal_is_typed(monkeypatch, failure_at: int) -> None: + ch, driver = _dispatch(monkeypatch, [[]]) + driver.failure_at = failure_at + driver.failure = NativeBudgetError(158) + + with pytest.raises(Exception) as raised: + await queue.cluster_peers(_q(), floor=0.6, limit=8, ch=ch) + + assert isinstance(raised.value, VectorCapacityError) + assert raised.value.extra["reason"] == "rows" + assert driver.started == failure_at + + +async def test_resolver_cluster_without_embedding_dispatches_nothing(monkeypatch) -> None: + ch, driver = _dispatch(monkeypatch, []) + + assert await queue.cluster_peers(_q(embedding=[]), floor=0.6, limit=8, ch=ch) == queue.Cluster( + [], frozenset() + ) + assert driver.started == 0 + assert driver.work == [] diff --git a/tests/unit/test_vector_dispatch_contract.py b/tests/unit/test_vector_dispatch_contract.py new file mode 100644 index 0000000000..772123ef63 --- /dev/null +++ b/tests/unit/test_vector_dispatch_contract.py @@ -0,0 +1,1009 @@ +"""First-party vector SQL builders and dispatches have an explicit classification. + +The repository inventory runs only in the dedicated CI test body. Collection and the +six recognizer controls never read repository source or import server modules. +This source contract cannot prove arbitrary runtime SQL, optimizer work, native +execution, or ranking equality. Caller-supplied analyst SQL keeps its separate gate. +""" + +from __future__ import annotations + +import ast +import hashlib +import re +from collections import Counter +from pathlib import Path + +# Match function spellings only in executable AST string literals, never comments +# or docstrings. This is test-only source discovery, never runtime SQL admission. +_DISTANCE = re.compile( + r"\b(?:cosineDistance|L1Distance|L2Distance|L2SquaredDistance|LinfDistance|dotProduct)\b", re.I +) +_SINKS = { + "query", + "query_all", + "query_one", + "query_rows", + "query_columns", + "raw_bytes", + "raw_stream", + "_rows", + "_chunk_query", + "_call_retrying_empty_block", +} + +# A row is (distance-literal count, vector_work arguments, query contexts, +# references to classified SQL fragments). Repeated SQL distance expressions do +# not infer a weight: the independently reviewed marker expressions below do. +# The final scalar peer-row fetch, lookup metadata fetch and key-match fetch are +# deliberately recorded with None. Builder-only rows have no dispatch context. +_CLASSIFIED = { + "server/code/store/index/answers.py:AnswersMixin.nearest_answers": ( + 0, + (("org", "ns", "1"),), + (("query_all", ("org", "ns", "1")),), + (("server/code/store/index/answers.py:_candidate_sql", 1),), + ), + "server/code/store/index/answers.py:_candidate_sql": (1, (), (), ()), + "server/code/store/index/files.py:FilesMixin.best_chunks_for_paths": ( + 1, + (("org", "ns", "1"),), + (("query_all", ("org", "ns", "1")),), + (), + ), + "server/code/store/index/files.py:FilesMixin.track_path_chunks": ( + 1, + (("org", "ns", "1 if query_vec else 0"),), + (("query_all", ("org", "ns", "1 if query_vec else 0")),), + (), + ), + "server/code/store/index/search.py:SearchMixin.exact_rerank_by_ids._ranked": ( + 1, + (("org", "ns", "1"),), + (("query_all", ("org", "ns", "1")),), + (), + ), + "server/code/store/index/search.py:SearchMixin.vector_search_cards": ( + 1, + (("org", "ns", "1"),), + (("query_all", ("org", "ns", "1")),), + (), + ), + "server/code/store/index/search.py:SearchMixin.vector_search_chunks": ( + 1, + (("org", "ns", "1"),), + (("query_all", ("org", "ns", "1")),), + (), + ), + "server/code/store/index/search.py:SearchMixin.vector_shortlist_ids": ( + 1, + (("org", "ns", "1"),), + (("query_all", ("org", "ns", "1")),), + (), + ), + "server/core/ch.py:CHClient._query_attempts": (0, (("''", "()", "0"),), (), ()), + "server/core/ch.py:CHClient._raw_stream_query": (0, (("''", "()", "0"),), (), ()), + "server/core/ch.py:CHClient.raw_stream": (0, (("''", "()", "0"),), (), ()), + "server/identity/app/namespace_catalog.py:handle_namespace_search_categories": ( + 1, + (("input.org", "input.namespace", "1"),), + (("query_all", ("input.org", "input.namespace", "1")),), + (), + ), + "server/knowledge/app/conversation_search.py:vector_search": ( + 2, + (("q.org", "q.namespace", "1"),), + (("query_all", ("q.org", "q.namespace", "1")),), + (), + ), + "server/knowledge/app/search.py:hybrid_search": ( + 1, + (("q.org", "q.namespaces", "1"),), + (("query_all", ("q.org", "q.namespaces", "1")),), + (), + ), + "server/knowledge/app/search.py:keys_search": ( + 1, + (("q.org", "q.namespaces", "1"),), + (("query_all", ("q.org", "q.namespaces", "1")),), + (), + ), + "server/knowledge/app/search.py:lexical_search": ( + 1, + (("q.org", "q.namespaces", "1"),), + (("query_all", ("q.org", "q.namespaces", "1")),), + (), + ), + "server/knowledge/app/source_search.py:collect_leg": ( + 0, + (), + (), + (("server/knowledge/store/source_candidates.py:statement", 1),), + ), + "server/knowledge/store/source_candidates.py:candidates": ( + 0, + (("str(params['org'])", "list(params['nss'])", "1"),), + (("_rows", ("str(params['org'])", "list(params['nss'])", "1")),), + (), + ), + "server/knowledge/store/source_candidates.py:statement": (1, (), (), ()), + "server/memory/app/conflicts.py:_foreign_pinned_rows": ( + 0, + (("str(params['org'])", "str(params['ns'])", "1"),), + (("query_all", ("str(params['org'])", "str(params['ns'])", "1")),), + (("server/memory/store/conflict_candidates.py:PINNED_PEERS_SQL", 1),), + ), + "server/memory/app/conflicts.py:_peer_rows_joined": ( + 1, + (("str(params['org'])", "str(params['ns'])", "1"),), + (("query_all", ("str(params['org'])", "str(params['ns'])", "1")),), + (), + ), + "server/memory/app/conflicts.py:candidate_rows_many._chunk": ( + 0, + (), + (), + ( + ("server/memory/store/conflict_candidates.py:candidate_sql_many", 1), + ("server/memory/store/conflict_candidates.py:pinned_sql_many", 1), + ), + ), + "server/memory/app/conflicts.py:candidate_rows_many._chunk._read": ( + 0, + (("org", "namespace", "len(idxs)"),), + (("query_all", ("org", "namespace", "len(idxs)")),), + (), + ), + "server/memory/app/conflicts.py:check_against_shared": ( + 1, + (("org", "namespace", "1"),), + (("query_all", ("org", "namespace", "1")),), + (), + ), + "server/memory/app/conflicts.py:nearest_peer_rows": ( + 0, + (("str(params['org'])", "str(params['ns'])", "1"),), + (("query_all", ("str(params['org'])", "str(params['ns'])", "1")), ("query_all", None)), + (("server/memory/store/conflict_candidates.py:_peer_candidates_sql", 1),), + ), + "server/memory/app/profile_search.py:PROFILE_VECTORS_SQL": (1, (), (), ()), + "server/memory/app/profile_search.py:_similarities": ( + 0, + (("org", "tuple({str(r['namespace']) for r in rows})", "1"),), + (("query_all", ("org", "tuple({str(r['namespace']) for r in rows})", "1")),), + (("server/memory/app/profile_search.py:PROFILE_VECTORS_SQL", 1),), + ), + "server/memory/app/retrieval/chunks.py:_chunk_anchor_windows": ( + 0, + (("chunk_params['org']", "chunk_params['ns']", "1"),), + (("_chunk_query", ("chunk_params['org']", "chunk_params['ns']", "1")),), + (("server/memory/app/retrieval/chunks.py:_chunk_neighbor_sql", 1),), + ), + "server/memory/app/retrieval/chunks.py:_chunk_candidates_sql": (1, (), (), ()), + "server/memory/app/retrieval/chunks.py:_chunk_neighbor_sql": (1, (), (), ()), + "server/memory/app/retrieval/chunks.py:_chunk_stage2_one_trip_sql": (1, (), (), ()), + "server/memory/app/retrieval/chunks.py:_search_memory_chunks": ( + 0, + (("q.org", "q.namespace", "1"), ("q.org", "q.namespace", "int(neighbors)")), + ( + ("_chunk_query", ("q.org", "q.namespace", "1")), + ("_chunk_query", ("q.org", "q.namespace", "int(neighbors)")), + ), + ( + ("server/memory/app/retrieval/chunks.py:_chunk_candidates_sql", 1), + ("server/memory/app/retrieval/chunks.py:_chunk_stage2_one_trip_sql", 1), + ), + ), + "server/memory/app/retrieval/derived.py:_resolve_operand_event_date": ( + 1, + (("org", "namespace", "1"),), + (("query_all", ("org", "namespace", "1")),), + (), + ), + "server/memory/app/retrieval/derived.py:fetch_current_state_observation": ( + 1, + (("org", "namespace", "1"),), + (("query_all", ("org", "namespace", "1")),), + (), + ), + "server/memory/app/retrieval/fact_rank.py:_bm25_hybrid_rank_score": (1, (), (), ()), + "server/memory/app/retrieval/fact_rank.py:_fact_mode_rank_score": ( + 0, + (), + (), + (("server/memory/app/retrieval/fact_rank.py:_bm25_hybrid_rank_score", 1),), + ), + "server/memory/app/retrieval/fact_sql.py:_fact_distance_expr": (3, (), (), ()), + "server/memory/app/retrieval/fact_sql.py:_fact_leg_sql": ( + 2, + (), + (), + (("server/memory/app/retrieval/fact_sql.py:_fact_distance_expr", 1),), + ), + "server/memory/app/retrieval/pool.py:_recency_backfill": ( + 1, + (("q.org", "q.namespace", "1"),), + (("query_all", ("q.org", "q.namespace", "1")),), + (), + ), + "server/memory/app/retrieval/retry.py:_query_all_retrying_empty_block": ( + 0, + (("str(params['org'])", "str(params['ns'])", "weight"),), + ( + ("_call_retrying_empty_block", ("str(params['org'])", "str(params['ns'])", "weight")), + ("query_all", ("str(params['org'])", "str(params['ns'])", "weight")), + ), + (), + ), + "server/memory/app/retrieval/search.py:vector_search": ( + 0, + (), + (), + (("server/memory/app/retrieval/fact_sql.py:_fact_leg_sql", 1),), + ), + "server/memory/app/retrieval/search.py:vector_search_trace": ( + 1, + (("q.org", "q.namespace", "1"),), + (("query_all", ("q.org", "q.namespace", "1")),), + (), + ), + "server/memory/app/retrieval/side_legs.py:_fetch_unmatched_candidates": ( + 1, + (("q.org", "q.namespace", "1"),), + (("query_all", ("q.org", "q.namespace", "1")),), + (), + ), + "server/memory/app/retrieval/side_legs.py:_lexical_recall_fallback": ( + 1, + (("q.org", "q.namespace", "1"),), + ( + ("_call_retrying_empty_block", ("q.org", "q.namespace", "1")), + ("query_all", ("q.org", "q.namespace", "1")), + ), + (), + ), + "server/memory/app/retrieval/summaries.py:fetch_entity_summaries": ( + 1, + (("org", "namespace", "1"),), + (("query_all", ("org", "namespace", "1")),), + (), + ), + "server/memory/app/retrieval/summaries.py:fetch_session_summaries": ( + 1, + (("org", "namespace", "1"),), + (("query_all", ("org", "namespace", "1")),), + (), + ), + "server/memory/app/search/anchor_pool.py:_run_anchor_pool._anchor_rows_for_ns": ( + 0, + (("input.org", "ns", "1"),), + (("query_all", ("input.org", "ns", "1")),), + (("server/memory/app/search/sql.py:anchor_pool_sql", 1),), + ), + "server/memory/app/search/ann_entity_boosts.py:_ann_entity_boosts._one_entity": ( + 2, + (("org", "namespaces", "2"),), + (("query_all", ("org", "namespaces", "2")),), + (), + ), + "server/memory/app/search/digest_descent.py:_run_digest_descent._rows": ( + 0, + (("input.org", "ns", "1"),), + (("query_all", ("input.org", "ns", "1")),), + (("server/memory/app/search/sql.py:digest_descent_sql", 1),), + ), + "server/memory/app/search/key_match.py:_run_key_match_pool._rows_for_ns": ( + 0, + (("input.org", "ns", "1"),), + (("query_all", None), ("query_all", ("input.org", "ns", "1"))), + (("server/memory/app/search/sql.py:key_match_sql", 1),), + ), + "server/memory/app/search/sql.py:anchor_pool_sql": (2, (), (), ()), + "server/memory/app/search/sql.py:digest_descent_sql": (2, (), (), ()), + "server/memory/app/search/sql.py:key_match_sql": (2, (), (), ()), + "server/memory/app/session_outcomes.py:_NEAREST_POSITIVE_SQL": (1, (), (), ()), + "server/memory/app/session_outcomes.py:_contrast_pair": ( + 0, + (("str(neg['org'])", "str(neg['namespace'])", "1"),), + (("query_one", ("str(neg['org'])", "str(neg['namespace'])", "1")),), + (("server/memory/app/session_outcomes.py:_NEAREST_POSITIVE_SQL", 1),), + ), + "server/memory/app/value_updates.py:_peers_joined": ( + 1, + (("org", "namespace", "1"),), + (("query_all", ("org", "namespace", "1")),), + (), + ), + "server/memory/app/value_updates.py:_peers_many": ( + 0, + (("org", "namespace", "len(items)"),), + (("query_all", ("org", "namespace", "len(items)")),), + (("server/memory/app/value_updates.py:_peers_sql_many", 1),), + ), + "server/memory/app/value_updates.py:_peers_sql_many": (1, (), (), ()), + "server/memory/store/conflict_candidates.py:PINNED_PEERS_SQL": (1, (), (), ()), + "server/memory/store/conflict_candidates.py:_peer_candidates_sql": (1, (), (), ()), + "server/memory/store/conflict_candidates.py:candidate_sql_many": (1, (), (), ()), + "server/memory/store/conflict_candidates.py:pinned_sql_many": (1, (), (), ()), + "server/memory/store/conflict_resolver.py:_ANCHORED_SQL": ( + 0, + (), + (), + (("server/memory/store/conflict_resolver.py:_LIVE_VECTORS", 2),), + ), + "server/memory/store/conflict_resolver.py:_LIVE_VECTORS": (1, (), (), ()), + "server/memory/store/conflict_resolver.py:_nearest_sql": ( + 0, + (), + (), + (("server/memory/store/conflict_resolver.py:_LIVE_VECTORS", 1),), + ), + "server/memory/store/conflict_resolver.py:cluster_peers": ( + 0, + ( + ("str(row['org'])", "str(row['namespace'])", "1"), + ("str(row['org'])", "str(row['namespace'])", "2"), + ), + ( + ("query_all", ("str(row['org'])", "str(row['namespace'])", "1")), + ("query_all", ("str(row['org'])", "str(row['namespace'])", "2")), + ("query_all", None), + ), + ( + ("server/memory/store/conflict_resolver.py:_ANCHORED_SQL", 1), + ("server/memory/store/conflict_resolver.py:_nearest_sql", 1), + ), + ), + "server/memory/store/lookups.py:find_by_identifier": ( + 1, + (("org", "namespace", "1"),), + (("query_all", ("org", "namespace", "1")), ("query_all", None)), + (), + ), + "server/memory/store/lookups.py:find_chunks_by_identifier": ( + 1, + (("org", "namespace", "1 if qv and order != 'recency' else 0"),), + (("query_all", ("org", "namespace", "1 if qv and order != 'recency' else 0")),), + (), + ), + "server/memory/store/lookups.py:find_chunks_by_session_date": ( + 1, + (("org", "namespace", "1 if qv else 0"),), + (("query_all", ("org", "namespace", "1 if qv else 0")),), + (), + ), + "server/tracker/store/embed.py:TrackerEmbedRepo.nearest": ( + 1, + (("org", "namespace", "1"),), + (("query_all", ("org", "namespace", "1")),), + (), + ), + "server/tracker/store/source/base.py:SourceBase._rows": ( + 0, + (("self.org", "self.namespace", "1 if vector else 0"),), + (("query_rows", ("self.org", "self.namespace", "1 if vector else 0")),), + (), + ), + "server/tracker/store/source/similarity.py:SimilarReads.similar_vector": ( + 1, + (), + (("_rows", None),), + (), + ), +} + +_BUILDERS = { + "server/code/store/index/answers.py:_candidate_sql", + "server/knowledge/store/source_candidates.py:statement", + "server/memory/app/profile_search.py:PROFILE_VECTORS_SQL", + "server/memory/app/retrieval/chunks.py:_chunk_candidates_sql", + "server/memory/app/retrieval/chunks.py:_chunk_neighbor_sql", + "server/memory/app/retrieval/chunks.py:_chunk_stage2_one_trip_sql", + "server/memory/app/retrieval/fact_rank.py:_bm25_hybrid_rank_score", + "server/memory/app/retrieval/fact_sql.py:_fact_distance_expr", + "server/memory/app/retrieval/fact_sql.py:_fact_leg_sql", + "server/memory/app/search/sql.py:anchor_pool_sql", + "server/memory/app/search/sql.py:digest_descent_sql", + "server/memory/app/search/sql.py:key_match_sql", + "server/memory/app/session_outcomes.py:_NEAREST_POSITIVE_SQL", + "server/memory/app/value_updates.py:_peers_sql_many", + "server/memory/store/conflict_candidates.py:PINNED_PEERS_SQL", + "server/memory/store/conflict_candidates.py:_peer_candidates_sql", + "server/memory/store/conflict_candidates.py:candidate_sql_many", + "server/memory/store/conflict_candidates.py:pinned_sql_many", + "server/memory/store/conflict_resolver.py:_ANCHORED_SQL", + "server/memory/store/conflict_resolver.py:_LIVE_VECTORS", + "server/memory/store/conflict_resolver.py:_nearest_sql", +} + + +# Reviewed AST shapes of SQL producers only. A changed producer is reclassified +# with its dispatch weight evidence; these are not file/head/runtime guard pins. +_PRODUCER_SHAPES = { + "server/code/store/index/answers.py:_candidate_sql": "cb7e26315031c8b75b101894e9892b1eab9a30e9eb1764de388a5b175ff7cb78", + "server/knowledge/store/source_candidates.py:statement": "ca17d3458e5510b6a84c5fed87d29cc19059427acf663cbc3b1ac3a433269074", + "server/memory/app/profile_search.py:PROFILE_VECTORS_SQL": "c7340eba552c3737748ab15fe9142b423a080d8824a2ca9808d91d87ac86a3f0", + "server/memory/app/retrieval/chunks.py:_chunk_candidates_sql": "49fc7bd583581d70b0abb749ce6ea6991ec5a02de8bc07042f573ec01b5fcc16", + "server/memory/app/retrieval/chunks.py:_chunk_neighbor_sql": "93e2052b713ee7af7db0491f2ee5f20422d5ec0fae55d070419e08c4301d8c77", + "server/memory/app/retrieval/chunks.py:_chunk_stage2_one_trip_sql": "031697e9e97e8d455c4cfdc993a1425f6a6e1536dc1aeb75f23da4e255409736", + "server/memory/app/retrieval/fact_rank.py:_bm25_hybrid_rank_score": "11cf2389dfea336dd908f3920a84d9fbd490f645a3b762417458ce81a372c207", + "server/memory/app/retrieval/fact_sql.py:_fact_distance_expr": "43ac98d7a19379f9cf3135d6b4392b2b90f51a41ef0987baa97c5fe540d9efeb", + "server/memory/app/retrieval/fact_sql.py:_fact_leg_sql": "116155d18336a59aac48e52bd18730b9c22299db1b5137e200b79074c294d535", + "server/memory/app/search/sql.py:anchor_pool_sql": "36a63191e5c0706ddd50fe7d62cae2254c211beac62489b24edb1671bb016d82", + "server/memory/app/search/sql.py:digest_descent_sql": "2ff4e7cfd2958f75cb233fd1e4c78e54374f379564a6d2fb55cdd2eb0eebf13d", + "server/memory/app/search/sql.py:key_match_sql": "c6264d8d4b9a1c7c1c5120a01b10528442e1ec8cf7dd0e4587f331a63dbdbf02", + "server/memory/app/session_outcomes.py:_NEAREST_POSITIVE_SQL": "643e1247cf3e1d2866ed8fa67e1f75d1f35a6064afbe791bbe965a1039e0c3e4", + "server/memory/app/value_updates.py:_peers_sql_many": "e5363c90fe391dc78eaf22662bfbacc0c34091f74173f447df5889eb52a11adb", + "server/memory/store/conflict_candidates.py:PINNED_PEERS_SQL": "a6bd82afd8cdc9296411f148141f474f530bdde4f77cd4cbf0b6bad4252083b0", + "server/memory/store/conflict_candidates.py:_peer_candidates_sql": "de9d6da049bc73c8863d466aa1f6319a3911b05998c39975c052653699d8e14c", + "server/memory/store/conflict_candidates.py:candidate_sql_many": "e9a0840b791ecbf3134fb856f187acd0b905d73b5ddaca790502e4169dc50b39", + "server/memory/store/conflict_candidates.py:pinned_sql_many": "c57de290301feef0053752f6a24de6a2c333da33059f8fc716a59909be17fed4", + "server/memory/store/conflict_resolver.py:_ANCHORED_SQL": "943c9482ad51a170a2bd1fab47a5bd9285bae9b66cb2bf41790bd43ff53e76be", + "server/memory/store/conflict_resolver.py:_LIVE_VECTORS": "192a524488636780e2012d69a6ba4f4431c75fb3542fd0147b040a61988dcdf5", + "server/memory/store/conflict_resolver.py:_nearest_sql": "3bba90c6c2f8ddafe6a9481852825a69ce9546b9f25987071c075cb59f898f4f", +} + + +def _name(node: ast.AST) -> str: + return ( + node.id + if isinstance(node, ast.Name) + else node.attr + if isinstance(node, ast.Attribute) + else "" + ) + + +def _marker(call: ast.Call) -> tuple[str, ...]: + parameters = ("org", "namespaces", "weight") + if len(call.args) > 3 or any(isinstance(arg, ast.Starred) for arg in call.args): + raise ValueError("vector_work needs explicit org, namespaces and optional weight") + bound = dict(zip(parameters, (ast.unparse(arg) for arg in call.args), strict=False)) + for keyword in call.keywords: + if keyword.arg not in parameters or keyword.arg in bound: + raise ValueError("vector_work has an unresolved or duplicate argument") + bound[keyword.arg] = ast.unparse(keyword.value) + if not {"org", "namespaces"} <= bound.keys(): + raise ValueError("vector_work is missing org or namespaces") + return (bound["org"], bound["namespaces"], bound.get("weight", "1")) + + +def _producer_shape(node: ast.AST) -> str: + """Reviewed SQL-producing expressions/bodies, independent of Python formatting. + + A body fingerprint also covers repeated local aliases or callee composition; + reference counts alone cannot distinguish those shapes. This applies only to + SQL producers, not whole files or row-returning consumers. SQL string contents + remain significant. Function annotations and the leading docstring do not. + """ + if isinstance(node, ast.FunctionDef | ast.AsyncFunctionDef): + body = node.body + if ( + body + and isinstance(body[0], ast.Expr) + and isinstance(body[0].value, ast.Constant) + and isinstance(body[0].value.value, str) + ): + body = body[1:] + node = ast.Module(body=body, type_ignores=[]) + return hashlib.sha256(ast.dump(node, include_attributes=False).encode()).hexdigest() + + +class _Inventory(ast.NodeVisitor): + def __init__(self, path: str, builders: set[str]) -> None: + self.path, self.builders = path, builders + self.scope: list[str] = [] + self.context: tuple[str, ...] | None = None + self.rows: dict[str, list] = {} + self.imports: dict[tuple[str, str], str] = {} + self.modules: dict[tuple[str, str], str] = {} + self.forwarding: dict[tuple[str, str, str], list[str]] = {} + self.values: dict[str, set[str]] = {} + self.producers: set[str] = set() + self.shapes: dict[str, str] = {} + + def _owner(self) -> str: + return self.path + ":" + ".".join(self.scope) + + def _row(self) -> list: + return self.rows.setdefault(self._owner(), [0, [], [], Counter()]) + + def visit_Expr(self, node: ast.Expr) -> None: + if not isinstance(node.value, ast.Constant) or not isinstance(node.value.value, str): + self.generic_visit(node) + + def _scope(self, name: str, body: list[ast.stmt]) -> None: + old_context = self.context + self.scope.append(name) + self.context = None + for child in body: + self.visit(child) + self.scope.pop() + self.context = old_context + + def visit_FunctionDef(self, node: ast.FunctionDef) -> None: + self.shapes[self._owner() + ("." if self.scope else "") + node.name] = _producer_shape(node) + self._scope(node.name, node.body) + + def visit_AsyncFunctionDef(self, node: ast.AsyncFunctionDef) -> None: + self.shapes[self._owner() + ("." if self.scope else "") + node.name] = _producer_shape(node) + self._scope(node.name, node.body) + + def visit_ClassDef(self, node: ast.ClassDef) -> None: + self._scope(node.name, node.body) + + def visit_Assign(self, node: ast.Assign) -> None: + if not self.scope and len(node.targets) == 1 and isinstance(node.targets[0], ast.Name): + self.scope.append(node.targets[0].id) + self.shapes[self._owner()] = _producer_shape(node.value) + self.visit(node.value) + if self._sql(node.value): + self.producers.add(self._owner()) + self.scope.pop() + else: + self.generic_visit(node) + if self._sql(node.value): + for target in node.targets: + self.values.setdefault(self._owner(), set()).update( + name.id for name in ast.walk(target) if isinstance(name, ast.Name) + ) + + def visit_AnnAssign(self, node: ast.AnnAssign) -> None: + if not self.scope and isinstance(node.target, ast.Name) and node.value is not None: + self.scope.append(node.target.id) + self.shapes[self._owner()] = _producer_shape(node.value) + self.visit(node.value) + if self._sql(node.value): + self.producers.add(self._owner()) + self.scope.pop() + else: + self.generic_visit(node) + if ( + node.value is not None + and self._sql(node.value) + and isinstance(node.target, ast.Name) + ): + self.values.setdefault(self._owner(), set()).add(node.target.id) + + def visit_Return(self, node: ast.Return) -> None: + if node.value is not None and self._sql(node.value): + self.producers.add(self._owner()) + self.generic_visit(node) + + def visit_ImportFrom(self, node: ast.ImportFrom) -> None: + module = node.module or "" + if node.level: + parent = self.path.removesuffix(".py").split("/")[: -node.level] + module = ".".join([*parent, *([module] if module else [])]) + for alias in node.names: + self.imports[(".".join(self.scope), alias.asname or alias.name)] = ( + module.replace(".", "/") + ".py:" + alias.name + ) + self.modules[(".".join(self.scope), alias.asname or alias.name)] = ( + module + "." + alias.name + ) + + def visit_Import(self, node: ast.Import) -> None: + for alias in node.names: + self.modules[(".".join(self.scope), alias.asname or alias.name.split(".")[0])] = ( + alias.name if alias.asname else alias.name.split(".")[0] + ) + + def _attribute(self, node: ast.Attribute) -> str | None: + parts = [] + root = node + while isinstance(root, ast.Attribute): + parts.insert(0, root.attr) + root = root.value + if isinstance(root, ast.Name): + for length in range(len(self.scope), -1, -1): + module = self.modules.get((".".join(self.scope[:length]), root.id)) + if module is None: + continue + names = [*module.split("."), *parts] + for split in range(1, len(names)): + target = "/".join(names[:split]) + ".py:" + ".".join(names[split:]) + if target in self.builders: + return target + break + return None + + def _binding(self, node: ast.Name) -> str | None: + for length in range(len(self.scope), -1, -1): + prefix = ".".join(self.scope[:length]) + target = self.imports.get((prefix, node.id)) + local = self.path + ":" + ".".join([*self.scope[:length], node.id]) + if target in self.builders or local in self.builders: + return target if target in self.builders else local + return None + + def visit_Attribute(self, node: ast.Attribute) -> None: + if (target := self._attribute(node)) is not None: + self._row()[3][target] += 1 + else: + self.generic_visit(node) + + def visit_Name(self, node: ast.Name) -> None: + if isinstance(node.ctx, ast.Load) and (target := self._binding(node)) is not None: + self._row()[3][target] += 1 + + def _sql(self, node: ast.AST) -> bool: + """Conservative named SQL outputs, not the row results of query calls. + + Local aliases only grow within their lexical owner. Arbitrary callees do + not inherit SQL from arguments; known SQL-producing wrappers do, after + fixed-point discovery. This is not general interprocedural data flow. + """ + if isinstance(node, ast.Name): + return ( + node.id in self.values.get(self._owner(), set()) or self._binding(node) is not None + ) + if isinstance(node, ast.Attribute): + return self._attribute(node) is not None + if isinstance(node, ast.Constant): + return isinstance(node.value, str) and _DISTANCE.search(node.value) is not None + if isinstance(node, ast.Call): + return self._sql(node.func) + if isinstance(node, ast.Await | ast.FormattedValue | ast.NamedExpr): + return self._sql(node.value) + if isinstance(node, ast.Subscript): + return self._sql(node.value) + if isinstance(node, ast.BinOp): + return self._sql(node.left) or self._sql(node.right) + if isinstance(node, ast.IfExp): + return self._sql(node.body) or self._sql(node.orelse) + if isinstance(node, ast.Tuple | ast.List | ast.Set): + return any(self._sql(value) for value in node.elts) + if isinstance(node, ast.JoinedStr | ast.BoolOp): + return any(self._sql(value) for value in node.values) + if isinstance(node, ast.Dict): + return any(self._sql(value) for value in node.values) + return False + + def visit_Constant(self, node: ast.Constant) -> None: + if isinstance(node.value, str) and _DISTANCE.search(node.value): + self._row()[0] += 1 + + def visit_With(self, node: ast.With) -> None: + old_context = self.context + for item in node.items: + self.visit(item.context_expr) + if ( + isinstance(item.context_expr, ast.Call) + and _name(item.context_expr.func) == "vector_work" + ): + self.context = _marker(item.context_expr) + for child in node.body: + self.visit(child) + self.context = old_context + + def visit_Call(self, node: ast.Call) -> None: + name = _name(node.func) + if name == "vector_work": + self._row()[1].append(_marker(node)) + # The core transport executes arbitrary passed SQL. Its zero-weight + # schema/consumer clears are classified, but its driver calls are not + # first-party SQL builders and must not acquire another reservation. + if name in _SINKS and self.path != "server/core/ch.py": + self._row()[2].append((name, self.context)) + for kw in node.keywords: + if kw.arg in {"vector", "weight"}: + self.forwarding.setdefault((self._owner(), name, kw.arg), []).append( + ast.unparse(kw.value) + ) + self.generic_visit(node) + + +def _unclassified( + sources: dict[str, str], + classified: dict, + builders: set[str], + producer_shapes: dict | None = None, +) -> list[str]: + trees = {path: ast.parse(source, filename=path) for path, source in sources.items()} + builders = set(builders) + while True: + visitors = [] + for path, tree in trees.items(): + visitor = _Inventory(path, builders) + visitor.visit(tree) + visitors.append(visitor) + expanded = builders | {owner for visitor in visitors for owner in visitor.producers} + if expanded == builders: + break + builders = expanded + observed = {} + for visitor in visitors: + for owner, (literals, markers, queries, refs) in visitor.rows.items(): + if literals or markers or refs: + observed[owner] = ( + literals, + tuple(markers), + tuple(queries), + tuple(sorted(refs.items())), + ) + failures = [ + f"{owner}: expected {classified.get(owner)!r}, found {observed.get(owner)!r}" + for owner in sorted(classified.keys() | observed.keys()) + if classified.get(owner) != observed.get(owner) + ] + if producer_shapes is not None: + actual_shapes = { + owner: shape + for visitor in visitors + for owner, shape in visitor.shapes.items() + if owner in builders + } + failures.extend( + f"{owner}: SQL producer shape needs classification" + for owner in sorted(producer_shapes.keys() | actual_shapes.keys()) + if producer_shapes.get(owner) != actual_shapes.get(owner) + ) + return failures + + +def test_first_party_vector_dispatches_have_a_classification() -> None: + """Whole source contract: run in CI, not the local small focused cohort.""" + root = Path(__file__).resolve().parents[2] + # No source walk at import/collection time. A new first-party Python module + # is included even when it is outside today's literal-hit directories. + sources = { + str(path.relative_to(root)): path.read_text() + for path in sorted((root / "server").rglob("*.py")) + } + assert not (failures := _unclassified(sources, _CLASSIFIED, _BUILDERS, _PRODUCER_SHAPES)), ( + "\n".join(failures) + ) + forwarding = {} + for path in ( + "server/tracker/store/source/similarity.py", + "server/memory/app/retrieval/search.py", + ): + visitor = _Inventory(path, _BUILDERS) + visitor.visit(ast.parse(sources[path])) + forwarding.update(visitor.forwarding) + assert forwarding[ + ("server/tracker/store/source/similarity.py:SimilarReads.similar_vector", "_rows", "vector") + ] == ["True"] + assert forwarding[ + ( + "server/memory/app/retrieval/search.py:_fact_rows_task", + "_query_all_retrying_empty_block", + "weight", + ) + ] == ["weight"] + + +def test_new_builder_wrapper_and_relative_dispatch_need_classification() -> None: + sources = { + "server/new.py": 'def sql():\n return "SELECT cosineDistance(v, q)"\n', + "server/wrapper.py": "from server.known import sql\ndef wrap():\n fragment = sql()\n return fragment\n", + "server/final.py": "from server.wrapper import wrap\nasync def read(ch):\n return await ch.query_all(wrap())\n", + "server/pkg/relative.py": "from . import known as sql\nasync def read(ch):\n return await ch.query_all(sql.fragment)\n", + "server/pkg/nested/relative.py": "from .. import known as sql\nasync def read(ch):\n return await ch.query_all(sql.fragment)\n", + } + failures = _unclassified(sources, {}, {"server/known.py:sql", "server/pkg/known.py:fragment"}) + assert len(failures) == 5 + for owner in ( + "server/new.py:sql", + "server/wrapper.py:wrap", + "server/final.py:read", + "server/pkg/relative.py:read", + "server/pkg/nested/relative.py:read", + ): + assert any(message.startswith(owner + ":") for message in failures), owner + # A real query's row result must not turn its callers into SQL builders. + row_sources = { + "server/rows.py": "from server.known import sql\nasync def read(ch):\n return await ch.query_all(sql())\n", + "server/caller.py": "from server.rows import read\nasync def handle(ch):\n return await read(ch)\n", + } + failures = _unclassified(row_sources, {}, {"server/known.py:sql"}) + assert len(failures) == 1 + assert failures[0].startswith("server/rows.py:read:") + + +def test_changed_dispatch_weight_is_rejected_without_adding_a_read() -> None: + source = """from server.known import sql +async def read(ch, org, ns): + with vector_work(org, ns, 1): + return await ch.query_all(sql()) +""" + expected = { + "server/caller.py:read": ( + 0, + (("org", "ns", "1"),), + (("query_all", ("org", "ns", "1")),), + (("server/known.py:sql", 1),), + ) + } + assert _unclassified({"server/caller.py": source}, expected, {"server/known.py:sql"}) == [] + failures = _unclassified( + {"server/caller.py": source.replace("ns, 1", "ns, 2")}, expected, {"server/known.py:sql"} + ) + assert len(failures) == 1 + assert "('org', 'ns', '2')" in failures[0] + + +def test_comments_scalar_analyst_sql_and_classified_conditional_read_are_distinct() -> None: + source = '''# cosineDistance(embedding, q) is a comment. +def scalar(): + """cosineDistance is mentioned in documentation only.""" + return "SELECT count()" +async def analyst(ch, supplied_sql): + return await ch.query_all(supplied_sql) +async def lookup(ch, org, ns, vector): + sql = "SELECT cosineDistance(embedding, q)" if vector else "SELECT key" + with vector_work(org, ns, 1 if vector else 0): + return await ch.query_all(sql) +''' + expected = { + "server/example.py:lookup": ( + 1, + (("org", "ns", "1 if vector else 0"),), + (("query_all", ("org", "ns", "1 if vector else 0")),), + (), + ) + } + assert _unclassified({"server/example.py": source}, expected, set()) == [] + + +def test_an_extra_fragment_branch_is_rejected_with_unchanged_dispatch_weight() -> None: + source = """_BASE = "SELECT cosineDistance(v, q) AS distance FROM vectors" +_ANCHORED = _BASE + " UNION ALL " + _BASE +async def read(ch, org, ns): + with vector_work(org, ns, 2): + return await ch.query_all(_ANCHORED) +""" + base = "server/example.py:_BASE" + anchored = "server/example.py:_ANCHORED" + expected = { + base: (1, (), (), ()), + anchored: (0, (), (), ((base, 2),)), + "server/example.py:read": ( + 0, + (("org", "ns", "2"),), + (("query_all", ("org", "ns", "2")),), + ((anchored, 1),), + ), + } + assert _unclassified({"server/example.py": source}, expected, {base}) == [] + third = source.replace( + '_ANCHORED = _BASE + " UNION ALL " + _BASE', + '_ANCHORED = _BASE + " UNION ALL " + _BASE + " UNION ALL " + _BASE', + ) + failures = _unclassified({"server/example.py": third}, expected, {base}) + assert len(failures) == 1 + assert failures[0].startswith(anchored + ":") + assert "_BASE', 3)" in failures[0] + + # Ordinary alias composition has only one direct _BASE reference. Its third + # branch must still fail while the actual dispatch marker remains weight 2. + alias_source = '_BASE = "SELECT cosineDistance(v, q) AS distance FROM vectors"\ndef wrap():\n part = _BASE\n return part + " UNION ALL " + part\nasync def read(ch, org, ns):\n with vector_work(org, ns, 2):\n return await ch.query_all(wrap())\n' + wrapped = "server/example.py:wrap" + alias_expected = { + base: (1, (), (), ()), + wrapped: (0, (), (), ((base, 1),)), + "server/example.py:read": ( + 0, + (("org", "ns", "2"),), + (("query_all", ("org", "ns", "2")),), + ((wrapped, 1),), + ), + } + alias_shapes = { + "server/example.py:_BASE": "74b97265f930b608d0ae6e1ea7cb097aed174596182bbecebb8867f6db334ff5", + "server/example.py:wrap": "a195b6fe77e1a647c15223306fcdd9c4d55f0220eb1303c0b128e7d4dc98c73c", + } + assert ( + _unclassified({"server/example.py": alias_source}, alias_expected, {base}, alias_shapes) + == [] + ) + formatted_alias = alias_source.replace( + " part = _BASE", " # Same SQL producer.\n part = (_BASE)" + ) + assert ( + _unclassified({"server/example.py": formatted_alias}, alias_expected, {base}, alias_shapes) + == [] + ) + third_alias = alias_source.replace( + 'return part + " UNION ALL " + part', + 'return part + " UNION ALL " + part + " UNION ALL " + part', + ) + failures = _unclassified( + {"server/example.py": third_alias}, alias_expected, {base}, alias_shapes + ) + assert failures == [wrapped + ": SQL producer shape needs classification"] + + +def test_marker_keywords_bind_identity_scope_weight_and_reject_unresolved_arguments() -> None: + import pytest + + for call in ( + "vector_work(org, ns, 2)", + "vector_work(org, namespaces=ns, weight=2)", + "vector_work(org=org, namespaces=ns, weight=2)", + "vector_work(weight=2, namespaces=ns, org=org)", + ): + assert _marker(ast.parse(call, mode="eval").body) == ("org", "ns", "2") + assert _marker(ast.parse("vector_work(org=org, namespaces=ns)", mode="eval").body) == ( + "org", + "ns", + "1", + ) + for call in ( + "vector_work(*args)", + "vector_work(**kwargs)", + "vector_work(org, ns, weight=2, org=other)", + "vector_work(namespaces=ns)", + "vector_work(org, ns, 2, 3)", + "vector_work(org, ns, unknown=2)", + ): + with pytest.raises(ValueError): + _marker(ast.parse(call, mode="eval").body) + source = "from server.known import sql\nasync def read(ch, org, ns):\n with vector_work(org=org, namespaces=ns, weight=2):\n return await ch.query_all(sql())\n" + expected = { + "server/caller.py:read": ( + 0, + (("org", "ns", "2"),), + (("query_all", ("org", "ns", "2")),), + (("server/known.py:sql", 1),), + ) + } + assert _unclassified({"server/caller.py": source}, expected, {"server/known.py:sql"}) == [] + failures = _unclassified( + {"server/caller.py": source.replace("namespaces=ns", "namespaces=other")}, + expected, + {"server/known.py:sql"}, + ) + assert len(failures) == 1 + assert "('org', 'other', '2')" in failures[0] + + +def test_added_unmarked_read_and_nested_context_scope_are_checked_independently() -> None: + source = """from server.known import sql +async def read(ch, org, ns): + with vector_work(org, ns): + await ch.query_all(sql()) + with vector_work(org, ns, 2): + await ch.query_all(sql()) + await ch.query_all(sql()) + async def child(): + with vector_work(org, ns, 3): + return await ch.query_all(sql()) +""" + reference = (("server/known.py:sql", 3),) + expected = { + "server/caller.py:read": ( + 0, + (("org", "ns", "1"), ("org", "ns", "2")), + ( + ("query_all", ("org", "ns", "1")), + ("query_all", ("org", "ns", "2")), + ("query_all", ("org", "ns", "1")), + ), + reference, + ), + "server/caller.py:read.child": ( + 0, + (("org", "ns", "3"),), + (("query_all", ("org", "ns", "3")),), + (("server/known.py:sql", 1),), + ), + } + assert _unclassified({"server/caller.py": source}, expected, {"server/known.py:sql"}) == [] + failures = _unclassified( + {"server/caller.py": source + " await ch.query_all(sql())\n"}, + expected, + {"server/known.py:sql"}, + ) + assert len(failures) == 1 + assert "('query_all', None)" in failures[0] + unmarked_child = source.replace( + " with vector_work(org, ns, 3):\n return", " return" + ) + failures = _unclassified( + {"server/caller.py": unmarked_child}, expected, {"server/known.py:sql"} + ) + assert len(failures) == 1 + assert failures[0].startswith("server/caller.py:read.child:") + assert "('query_all', None)" in failures[0] From f3765ef9d2e6ab070b25b18a2dcd202fa10ccd23 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Wed, 18 Nov 2026 06:05:53 +0000 Subject: [PATCH 06/28] Keep internal vector refusals outside the wire error catalog Retain concrete input, overload and capacity wire contracts while testing configured queue and row bounds and both public SDK mappings. Signed-off-by: zlareb1 --- sdk/typescript/src/errors.test.ts | 60 +++++++++ server/core/errors.py | 6 +- tests/unit/test_sdk_errors_from_response.py | 26 ++++ tests/unit/test_vector_read_admission.py | 140 ++++++++++++++++++++ 4 files changed, 230 insertions(+), 2 deletions(-) diff --git a/sdk/typescript/src/errors.test.ts b/sdk/typescript/src/errors.test.ts index 47fb36c40d..b4ae0fea5c 100644 --- a/sdk/typescript/src/errors.test.ts +++ b/sdk/typescript/src/errors.test.ts @@ -17,6 +17,12 @@ import { ValidationError, WrongCell, } from "./errors.js"; +import { + BackendUnavailable as PublicBackendUnavailable, + RateLimited as PublicRateLimited, + errorFromResponse as publicErrorFromResponse, + VectorCapacityRefused, +} from "./index.js"; describe("errorFromResponse", () => { it.each([ @@ -110,6 +116,60 @@ describe("errorFromResponse", () => { expect([wait.category, wait.retryable]).toEqual(["capacity", true]); expect(wait.detail).toEqual({}); }); + + it("preserves vector capacity refusal metadata through the public SDK exports", () => { + const detail = { + code: "VECTOR_CAPACITY_REFUSED", + message: "vector queue is full", + category: "capacity", + retryable: true, + reason: "org_queue", + }; + const error = publicErrorFromResponse("memory.search", 429, { detail }, "vector-rid", 2.5); + expect(error.constructor).toBe(VectorCapacityRefused); + expect(error).toBeInstanceOf(PublicRateLimited); + if (!(error instanceof VectorCapacityRefused)) throw new Error("Expected public vector capacity refusal"); + expect([error.status, error.code, error.requestId, error.retryAfterS]).toEqual([ + 429, + "VECTOR_CAPACITY_REFUSED", + "vector-rid", + 2.5, + ]); + expect(error.detail).toEqual(detail); + expect([error.category, error.retryable]).toEqual(["capacity", true]); + expect(error.message).toBe("vector queue is full"); + }); + + it("keeps ordinary rate limits unchanged through the public SDK exports", () => { + const error = publicErrorFromResponse( + "memory.search", + 429, + { detail: { code: "RATE_LIMITED", message: "slow down" } }, + "rate-rid", + 3, + ); + expect(error.constructor).toBe(PublicRateLimited); + expect(error).not.toBeInstanceOf(VectorCapacityRefused); + if (!(error instanceof PublicRateLimited)) throw new Error("Expected public rate limit"); + expect([error.status, error.code, error.requestId, error.retryAfterS]).toEqual([ + 429, + "RATE_LIMITED", + "rate-rid", + 3, + ]); + }); + + it("keeps overloaded errors unchanged through the public SDK exports", () => { + const error = publicErrorFromResponse( + "memory.search", + 503, + { detail: { code: "OVERLOADED", message: "busy" } }, + "overloaded-rid", + ); + expect(error.constructor).toBe(PublicBackendUnavailable); + expect(error).not.toBeInstanceOf(VectorCapacityRefused); + expect([error.status, error.code, error.requestId]).toEqual([503, "OVERLOADED", "overloaded-rid"]); + }); }); describe("connectFailed", () => { diff --git a/server/core/errors.py b/server/core/errors.py index 5416ab6964..b60cb6f80b 100644 --- a/server/core/errors.py +++ b/server/core/errors.py @@ -57,11 +57,13 @@ class ValidationError(LoomServerError): ] -class VectorCapacityError(LoomServerError): +class VectorCapacityError(Exception): """Common refusal family: a read must not return a ranking subset.""" + extra: dict[str, Any] -class VectorAdmissionError(VectorCapacityError): + +class VectorAdmissionError(LoomServerError, VectorCapacityError): """429 — complete vector work exceeds a weighted queue or native budget. `reason` names the bounded refusal; retry after `retry_after_s`, or narrow the read.""" diff --git a/tests/unit/test_sdk_errors_from_response.py b/tests/unit/test_sdk_errors_from_response.py index 615825e10f..d131f46805 100644 --- a/tests/unit/test_sdk_errors_from_response.py +++ b/tests/unit/test_sdk_errors_from_response.py @@ -12,6 +12,7 @@ import httpx import pytest +from loom import VectorCapacityRefused from loom.errors import ( AuthInsufficient, AuthInvalid, @@ -130,6 +131,7 @@ def test_a_5xx_is_backend_unavailable_and_a_429_rate_limited_whatever_the_code() "SESSION_PROTECTION_REQUIRED": PreconditionFailed, "OVERLOADED": Overloaded, "RATE_LIMITED": RateLimited, + "VECTOR_CAPACITY_REFUSED": VectorCapacityRefused, "BACKEND_UNAVAILABLE": BackendUnavailable, "BACKEND_TIMEOUT": BackendTimeout, "SCHEMA_PENDING": SchemaPending, @@ -445,3 +447,27 @@ def test_a_wrong_cell_error_names_the_org_and_its_cell() -> None: assert (err.org, err.cell) == ("acme", "eu-2") unnamed = from_response({"detail": WrongCellError("acme").to_detail()}, 409) assert isinstance(unnamed, WrongCell) and (unnamed.org, unnamed.cell) == ("acme", "") + + +def test_public_vector_capacity_refusal_preserves_rate_limit_response_metadata() -> None: + detail = { + "code": "VECTOR_CAPACITY_REFUSED", + "message": "vector queue is full", + "category": "capacity", + "retryable": True, + "reason": "org_queue", + } + error = from_response({"detail": detail}, 429, request_id="vector-rid", retry_after_s=2.5) + + assert type(error) is VectorCapacityRefused + assert isinstance(error, RateLimited) + assert isinstance(error, BackendUnavailable) + assert (error.code, error.status, error.request_id, error.retry_after_s) == ( + "VECTOR_CAPACITY_REFUSED", + 429, + "vector-rid", + 2.5, + ) + assert error.detail == detail + assert (error.category, error.retryable) == ("capacity", True) + assert "vector queue is full" in str(error) diff --git a/tests/unit/test_vector_read_admission.py b/tests/unit/test_vector_read_admission.py index 6c219fac6b..91148452f5 100644 --- a/tests/unit/test_vector_read_admission.py +++ b/tests/unit/test_vector_read_admission.py @@ -304,3 +304,143 @@ async def refused(**kwargs: Any) -> Any: assert result.next.startswith("Existing next.") assert "vector capacity" in result.next assert "committed" in result.next + + +def test_vector_scan_waiter_setting_accepts_one_and_refuses_invalid_values() -> None: + from pydantic import ValidationError + + from server.settings import Settings + + assert Settings(vector_scan_max_waiters_per_org=1).vector_scan_max_waiters_per_org == 1 + for invalid in (0, -1, float("inf"), float("-inf"), float("nan")): + with pytest.raises(ValidationError, match="vector_scan_max_waiters_per_org"): + Settings.model_validate({"vector_scan_max_waiters_per_org": invalid}) + + +def test_vector_scan_row_setting_accepts_one_and_refuses_invalid_values() -> None: + from pydantic import ValidationError + + from server.settings import Settings + + assert Settings(vector_scan_max_rows_to_read=1).vector_scan_max_rows_to_read == 1 + for invalid in (0, -1, float("inf"), float("-inf"), float("nan")): + with pytest.raises(ValidationError, match="vector_scan_max_rows_to_read"): + Settings.model_validate({"vector_scan_max_rows_to_read": invalid}) + + +@pytest.mark.asyncio +async def test_vector_waiter_setting_bounds_queued_reads_and_reclaims_cancelled_capacity( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from server.core.errors import VectorCapacityError + + monkeypatch.setattr(settings, "vector_scan_max_concurrent_per_org", 2) + monkeypatch.setattr(settings, "vector_scan_max_waiters_per_org", 4) + ch, driver = client(monkeypatch) + tasks = [asyncio.create_task(hybrid_search(query("one-org"), ch=ch)) for _ in range(6)] + try: + await settled() + assert driver.active == 2 + assert sum(not task.done() for task in tasks) == 6 + with pytest.raises(VectorCapacityError) as raised: + await hybrid_search(query("one-org"), ch=ch) + assert raised.value.extra["reason"] == "org_queue" + assert driver.started == 2 + + tasks[-1].cancel() + with pytest.raises(asyncio.CancelledError): + await tasks[-1] + replacement = asyncio.create_task(hybrid_search(query("one-org"), ch=ch)) + tasks.append(replacement) + await settled() + assert not replacement.done() + assert driver.started == 2 + finally: + results = await join(tasks, driver) + assert results.count([]) == 6 + assert sum(isinstance(result, asyncio.CancelledError) for result in results) == 1 + assert driver.started == 6 + assert driver.active == 0 + + # Both org credits are usable after refusal, queued cancellation and drain. + driver.release.clear() + probes = [asyncio.create_task(hybrid_search(query("one-org"), ch=ch)) for _ in range(2)] + try: + await settled() + assert driver.active == 2 + finally: + probe_results = await join(probes, driver) + assert probe_results == [[], []] + assert driver.started == 8 + assert driver.active == 0 + + +@pytest.mark.asyncio +async def test_vector_row_setting_reaches_the_actual_native_read_budget( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(settings, "vector_scan_max_rows_to_read", 1234) + ch, driver = client(monkeypatch) + driver.release.set() + + assert await hybrid_search(query("one-org"), ch=ch) == [] + assert driver.started == 1 + assert driver.settings[0]["max_rows_to_read"] == 1234 + assert driver.settings[0]["read_overflow_mode"] == "throw" + assert driver.active == 0 + + +@pytest.mark.parametrize("kind", ["admission", "input", "overload"]) +def test_vector_concrete_error_hierarchy_preserves_wire_contracts(kind: str) -> None: + from server.core.errors import ( + LoomServerError, + OverloadedError, + ValidationError, + VectorAdmissionError, + VectorCapacityError, + VectorOverloadedError, + classified, + ) + + if kind == "admission": + error = VectorAdmissionError("refused", reason="org_queue", retry_after_s=3) + public_family = LoomServerError + expected = ( + 429, + { + "code": "VECTOR_CAPACITY_REFUSED", + "message": "refused", + "reason": "org_queue", + "retry_after_s": 3, + }, + {"Retry-After": "3"}, + ) + category, retryable = "capacity", True + elif kind == "input": + error = VectorInputTooLargeError("too many seeds") + public_family = ValidationError + expected = (422, {"code": "VECTOR_INPUT_TOO_LARGE", "message": "too many seeds"}, None) + category, retryable = "input", False + else: + error = VectorOverloadedError("busy", reason="population", retry_after_s=2) + public_family = OverloadedError + expected = ( + 503, + {"code": "OVERLOADED", "message": "busy", "reason": "population", "retry_after_s": 2}, + {"Retry-After": "2"}, + ) + category, retryable = "capacity", True + + assert isinstance(error, public_family) + with pytest.raises(VectorCapacityError) as caught_capacity: + raise error + assert caught_capacity.value is error + with pytest.raises(LoomServerError) as caught_wire: + raise error + assert caught_wire.value is error + assert (error.status_code, error.to_detail(), error.headers()) == expected + assert classified(error.to_detail(), error.status_code) == { + **expected[1], + "category": category, + "retryable": retryable, + } From 17f59ccc1e1e336cc1b1fb6fc71315c4e4b4b17b Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Wed, 18 Nov 2026 07:10:22 +0000 Subject: [PATCH 07/28] Refuse supplemental vector reads that would queue Keep completion-sensitive duplicate selection on the eager path and return the complete capacity refusal when immediate credits are unavailable. Retire no-op client generations inline while preserving owned closure for real retired clients. Signed-off-by: zlareb1 --- docs/architecture/clickhouse.md | 23 ++ server/core/ch.py | 2 +- server/core/vectors/admission.py | 15 +- server/memory/app/search/supplemental.py | 15 +- ...t_vector_admission_supplemental_ranking.py | 297 ++++++++++++++++++ .../unit/test_vector_generation_retirement.py | 152 +++++++++ ...test_vector_supplemental_no_wait_policy.py | 212 +++++++++++++ 7 files changed, 707 insertions(+), 9 deletions(-) create mode 100644 tests/unit/test_vector_admission_supplemental_ranking.py create mode 100644 tests/unit/test_vector_generation_retirement.py create mode 100644 tests/unit/test_vector_supplemental_no_wait_policy.py diff --git a/docs/architecture/clickhouse.md b/docs/architecture/clickhouse.md index ea5642c732..104a67b02b 100644 --- a/docs/architecture/clickhouse.md +++ b/docs/architecture/clickhouse.md @@ -1039,6 +1039,29 @@ partial acquisition. Lower live limits refuse new reservations while existing charges drain; an overweight queued statement refuses instead of blocking its successors. Disabling the existing connection gate refuses new vector work. +Sub-query and set-expansion phrases share a completion-sensitive duplicate set. +Their vector statements take an immediate atomic grant or refuse the complete read +with `VECTOR_CAPACITY_REFUSED` and reason `org_queue`; they never join an org wait +queue or pass an older waiter. Owned siblings finish cancellation before the refusal +returns. Other vector owners retain bounded FIFO waits. The policy is inherited by +child tasks independently of statement attribution and resets when the phrase owner +exits. Admitted work still holds its charge through connection waits, retries and +local retirement. + +With no competing work, two phrases over one namespace can require eight weighted +units; two namespaces can require sixteen. Actual overlap determines admission, +so a nine-namespace fact-only read can succeed at the default sixteen-unit ceiling. +It is not refused from input counts alone. Wide shapes, including thirty-three +namespaces and eight phrases, can refuse even at the default ceiling. A two-unit +ceiling can refuse overlapping weighted statements. These are explicit capacity +failures, not partial successful rankings. Request frequency and representative +refusal rates require deployment qualification. + +The admitted and queued unit bounds do not bound eager task objects created by +upstream producers. Supplemental phrases retain eager scheduling and have no +four-task producer window. There is no established composed maximum fanout or +process-memory bound from these admission limits alone. + Every vector attempt has finite time and memory budgets and a native throw ceiling of one million read rows by default. Versions, joins and several namespaces can reach that ceiling before one million logical keys. Admission and native refusals diff --git a/server/core/ch.py b/server/core/ch.py index 4bb327de75..f7bc90c229 100644 --- a/server/core/ch.py +++ b/server/core/ch.py @@ -1198,7 +1198,7 @@ async def _bounded(self, op_name: str, cls: str, fn: Callable[[], Awaitable[Any] raise finally: self._inflight[gen] = self._inflight.get(gen, 1) - 1 - if current_work() is None: + if current_work() is None or self._inflight.get(gen, 0) > 0 or gen not in self._retired: await self._retire_if_done(gen) else: try: diff --git a/server/core/vectors/admission.py b/server/core/vectors/admission.py index 23a608e22a..ca75fee656 100644 --- a/server/core/vectors/admission.py +++ b/server/core/vectors/admission.py @@ -29,6 +29,17 @@ class VectorWork: _work: ContextVar[VectorWork | None] = ContextVar("vector_statement_work", default=None) +_without_wait: ContextVar[bool] = ContextVar("vector_admission_without_wait", default=False) + + +@contextmanager +def _without_admission_wait() -> Iterator[None]: + """Refuse capacity waits in completion-sensitive supplemental reads.""" + token = _without_wait.set(True) + try: + yield + finally: + _without_wait.reset(token) @contextmanager @@ -143,9 +154,9 @@ async def enter(self, work: VectorWork, wait_s: float) -> tuple[_Org, _Waiter | if not org.queue and org.held + work.weight <= c: org.held += work.weight return org, None - if org.waiting + work.weight > w: + if _without_wait.get() or org.waiting + work.weight > w: raise VectorAdmissionError( - "the org vector queue is full", reason="org_queue", retry_after_s=1 + "org vector capacity is unavailable", reason="org_queue", retry_after_s=1 ) waiter = _Waiter(work.weight, asyncio.get_running_loop().create_future()) org.queue.append(waiter) diff --git a/server/memory/app/search/supplemental.py b/server/memory/app/search/supplemental.py index 900f002b1d..0afa6cd2e4 100644 --- a/server/memory/app/search/supplemental.py +++ b/server/memory/app/search/supplemental.py @@ -11,6 +11,7 @@ from server.core.errors import VectorCapacityError from server.core.shielded import to_its_end +from server.core.vectors.admission import _without_admission_wait from server.memory.api.models_search import _MemorySearchHit from server.memory.app.search.context import SearchContext from server.memory.app.search.vector_legs import _search_query_for, _wire_hit @@ -115,9 +116,10 @@ async def _run_set_expansion(ctx: SearchContext) -> None: default=1.0, ) try: - per_phrase = await _supplemental_reads( - *[_supplemental_leg(ctx, ph, leg_top_k, seen_keys) for ph in phrases] - ) + with _without_admission_wait(): + per_phrase = await _supplemental_reads( + *[_supplemental_leg(ctx, ph, leg_top_k, seen_keys) for ph in phrases] + ) except VectorCapacityError: raise except Exception as e: # noqa: BLE001 — additive legs @@ -174,8 +176,9 @@ async def _run_sub_queries(ctx: SearchContext) -> None: else: sub_top_k = max(5, input.top_k // max(1, len(ctx.sub_queries))) - sub_results_by_query = await _supplemental_reads( - *[_supplemental_leg(ctx, sq, sub_top_k, seen_keys) for sq in ctx.sub_queries] - ) + with _without_admission_wait(): + sub_results_by_query = await _supplemental_reads( + *[_supplemental_leg(ctx, sq, sub_top_k, seen_keys) for sq in ctx.sub_queries] + ) for sub_hits in sub_results_by_query: ctx.all_hits.extend(sub_hits) diff --git a/tests/unit/test_vector_admission_supplemental_ranking.py b/tests/unit/test_vector_admission_supplemental_ranking.py new file mode 100644 index 0000000000..2aea643cc8 --- /dev/null +++ b/tests/unit/test_vector_admission_supplemental_ranking.py @@ -0,0 +1,297 @@ +"""Supplemental reads refuse org waits and preserve successful phrase winners.""" + +from __future__ import annotations + +import asyncio +from dataclasses import replace +from typing import Any + +import pytest + +from server.core.errors import VectorAdmissionError +from server.core.vectors import admission +from server.memory.api.models_search import _MemorySearchHit +from server.memory.app.search import supplemental +from server.memory.app.search.vector_legs import _search_query_for +from server.settings import settings +from tests.unit.test_memory_fanout_windows import _ctx, _hit + + +async def test_default_c16_sub_query_keeps_the_complete_original_success_body( + monkeypatch: pytest.MonkeyPatch, +) -> None: + concurrent = 16 + from server.memory.app import retrieval + + ctx = _ctx() + ctx.sub_queries = ["first", "second"] + gate = admission._Admission() + monkeypatch.setattr(admission, "_admission", gate) + monkeypatch.setattr(settings, "vector_scan_max_concurrent_per_org", concurrent) + monkeypatch.setattr(settings, "vector_scan_max_waiters_per_org", 256) + monkeypatch.setattr(settings, "ch_pool_maxsize", 64) + monkeypatch.setattr(settings, "ch_pool_wait_ms", 5000) + reads: set[asyncio.Task[Any]] = set() + queries: list[Any] = [] + admitted: list[admission.VectorWork] = [] + active = 0 + peak = 0 + + async def search(q): + nonlocal active, peak + task = asyncio.current_task() + assert task is not None + reads.add(task) + queries.append(q) + assert admission.current_work() is None + with admission.vector_work(q.org, q.namespace): + expected_work = admission.VectorWork(q.org, (q.namespace,), 1) + assert admission.current_work() == expected_work + async with admission.admit(pool_ready=True, wait_s=5.0): + assert admission.current_work() == expected_work + admitted.append(expected_work) + active += 1 + peak = max(peak, active) + try: + ticks = (100 if q.namespace == "ns0" else 0) if q.query_text == "first" else 60 + for _ in range(ticks): + await asyncio.sleep(0) + result = [_hit(q.namespace, distance=0.2 if q.query_text == "first" else 0.4)] + finally: + active -= 1 + assert admission.current_work() is None + return result + + monkeypatch.setattr(retrieval, "vector_search", search) + assert admission.current_work() is None + request = asyncio.create_task(supplemental._run_sub_queries(ctx)) + try: + await request + expected_queries = [ + replace(_search_query_for(ctx, ns), query_vec=[0.5] * 4, top_k=5, query_text=phrase) + for phrase in ctx.sub_queries + for ns in ctx.namespaces + ] + assert queries == expected_queries + assert admitted == [ + admission.VectorWork("o", (ns,), 1) + for _phrase in ctx.sub_queries + for ns in ctx.namespaces + ] + assert ctx.actual_work == { + "embedding_calls": 2, + "vector_queries": 18, + "sub_query_count": 2, + } + assert len(reads) == 18 and all(task.done() for task in reads) + assert 1 < peak <= concurrent + assert active == 0 and gate.population == 0 and gate.orgs == {} + assert admission.current_work() is None + assert [h.namespace for h in ctx.all_hits] == ctx.namespaces + assert [h.rank_score for h in ctx.all_hits] == pytest.approx([0.6] * 9) + assert [h.model_dump() for h in ctx.all_hits] == [ + _MemorySearchHit( + memory_key=ns, + memory_type="semantic", + content_excerpt=ns, + distance=0.4, + similarity=0.6, + rank_score=0.6, + namespace=ns, + ).model_dump() + for ns in ctx.namespaces + ] + finally: + for task in (request, *reads): + if not task.done(): + task.cancel() + await asyncio.gather(request, *reads, return_exceptions=True) + assert all(task.done() for task in (request, *reads)) + assert active == 0 and gate.population == 0 and gate.orgs == {} + assert admission.current_work() is None + + +@pytest.mark.parametrize("owner", ["sub_queries", "set_expansion"]) +async def test_supplemental_c2_refuses_complete_read_instead_of_waiting( + monkeypatch: pytest.MonkeyPatch, owner: str +) -> None: + from server.memory.app import retrieval + + ctx = _ctx() + ctx.namespaces = ["ns0", "ns1"] + ctx.sub_queries = ["first", "second"] + ctx.set_expansion_phrases = ["first", "second"] + gate = admission._Admission() + monkeypatch.setattr(admission, "_admission", gate) + monkeypatch.setattr(settings, "vector_scan_max_concurrent_per_org", 2) + monkeypatch.setattr(settings, "vector_scan_max_waiters_per_org", 256) + monkeypatch.setattr(settings, "ch_pool_maxsize", 64) + monkeypatch.setattr(settings, "ch_pool_wait_ms", 5000) + reads: set[asyncio.Task[Any]] = set() + active = 0 + peak = 0 + completed = 0 + + async def search(q): + nonlocal active, peak, completed + task = asyncio.current_task() + assert task is not None + reads.add(task) + assert admission.current_work() is None + with admission.vector_work(q.org, q.namespace): + async with admission.admit(pool_ready=True, wait_s=5.0): + active += 1 + peak = max(peak, active) + try: + for _ in range(20): + await asyncio.sleep(0) + completed += 1 + return [_hit(q.namespace)] + finally: + active -= 1 + + monkeypatch.setattr(retrieval, "vector_search", search) + run = ( + supplemental._run_sub_queries if owner == "sub_queries" else supplemental._run_set_expansion + ) + request = asyncio.create_task(run(ctx)) + try: + with pytest.raises(VectorAdmissionError) as caught: + await request + assert caught.value.code == "VECTOR_CAPACITY_REFUSED" + assert caught.value.extra == {"reason": "org_queue", "retry_after_s": 1} + assert caught.value.status_code == 429 + assert caught.value.headers() == {"Retry-After": "1"} + assert completed == 0 + assert peak == 2 + assert len(reads) == 4 and all(task.done() for task in reads) + assert ctx.all_hits == [] + assert active == 0 and gate.population == 0 and gate.orgs == {} + assert admission.current_work() is None + finally: + for task in (request, *reads): + if not task.done(): + task.cancel() + await asyncio.gather(request, *reads, return_exceptions=True) + assert all(task.done() for task in (request, *reads)) + assert active == 0 and gate.population == 0 and gate.orgs == {} + assert admission.current_work() is None + + +async def test_default_c16_public_client_keeps_the_complete_phrase_winner( + monkeypatch: pytest.MonkeyPatch, +) -> None: + concurrent = 16 + from types import SimpleNamespace + + from tests.unit.test_vector_read_admission import client + + ch, driver = client(monkeypatch) + native_calls: list[tuple[dict[str, Any], dict[str, Any]]] = [] + native_work: list[admission.VectorWork | None] = [] + + async def native_query(sql: str, **kwargs: Any) -> Any: + assert sql == "SELECT {ticks:UInt32} AS ticks" + native_calls.append((kwargs["parameters"], kwargs["settings"])) + native_work.append(admission.current_work()) + for _ in range(kwargs["parameters"]["ticks"]): + await asyncio.sleep(0) + return SimpleNamespace(result_rows=[], column_names=[], summary={}) + + monkeypatch.setattr(driver, "query", native_query) + from server.memory.app import retrieval + + ctx = _ctx() + ctx.sub_queries = ["first", "second"] + gate = admission._Admission() + monkeypatch.setattr(admission, "_admission", gate) + monkeypatch.setattr(settings, "vector_scan_max_concurrent_per_org", concurrent) + monkeypatch.setattr(settings, "vector_scan_max_waiters_per_org", 256) + monkeypatch.setattr(settings, "ch_pool_maxsize", 64) + monkeypatch.setattr(settings, "ch_pool_wait_ms", 5000) + reads: set[asyncio.Task[Any]] = set() + queries: list[Any] = [] + admitted: list[admission.VectorWork] = [] + active = 0 + peak = 0 + + async def search(q): + nonlocal active, peak + task = asyncio.current_task() + assert task is not None + reads.add(task) + queries.append(q) + assert admission.current_work() is None + with admission.vector_work(q.org, q.namespace): + expected_work = admission.VectorWork(q.org, (q.namespace,), 1) + assert admission.current_work() == expected_work + admitted.append(expected_work) + active += 1 + peak = max(peak, active) + try: + ticks = (100 if q.namespace == "ns0" else 0) if q.query_text == "first" else 60 + assert await ch.query_all("SELECT {ticks:UInt32} AS ticks", {"ticks": ticks}) == [] + result = [_hit(q.namespace, distance=0.2 if q.query_text == "first" else 0.4)] + finally: + active -= 1 + assert admission.current_work() is None + return result + + monkeypatch.setattr(retrieval, "vector_search", search) + assert admission.current_work() is None + request = asyncio.create_task(supplemental._run_sub_queries(ctx)) + try: + await request + expected_queries = [ + replace(_search_query_for(ctx, ns), query_vec=[0.5] * 4, top_k=5, query_text=phrase) + for phrase in ctx.sub_queries + for ns in ctx.namespaces + ] + assert queries == expected_queries + assert admitted == [ + admission.VectorWork("o", (ns,), 1) + for _phrase in ctx.sub_queries + for ns in ctx.namespaces + ] + assert ctx.actual_work == { + "embedding_calls": 2, + "vector_queries": 18, + "sub_query_count": 2, + } + assert len(reads) == 18 and all(task.done() for task in reads) + assert 1 < peak <= concurrent + assert active == 0 and gate.population == 0 and gate.orgs == {} + assert admission.current_work() is None + assert [h.namespace for h in ctx.all_hits] == ctx.namespaces + assert [h.rank_score for h in ctx.all_hits] == pytest.approx([0.6] * 9) + assert [h.model_dump() for h in ctx.all_hits] == [ + _MemorySearchHit( + memory_key=ns, + memory_type="semantic", + content_excerpt=ns, + distance=0.4, + similarity=0.6, + rank_score=0.6, + namespace=ns, + ).model_dump() + for ns in ctx.namespaces + ] + finally: + for task in (request, *reads): + if not task.done(): + task.cancel() + await asyncio.gather(request, *reads, return_exceptions=True) + assert all(task.done() for task in (request, *reads)) + assert active == 0 and gate.population == 0 and gate.orgs == {} + assert admission.current_work() is None + + assert len(native_calls) == 18 + assert native_work == [admission.VectorWork(q.org, (q.namespace,), 1) for q in queries] + for params, caps in native_calls: + assert set(params) == {"ticks"} + assert caps["max_rows_to_read"] == settings.vector_scan_max_rows_to_read + assert caps["read_overflow_mode"] == "throw" + assert 0 < caps["max_execution_time"] <= settings.ch_caller_shaped_max_execution_s + assert 0 < caps["max_memory_usage"] <= settings.ch_caller_shaped_max_memory_bytes + assert ch._slots is not None and ch._slots._value == 64 + assert ch._inflight == {} and ch._retired == {} diff --git a/tests/unit/test_vector_generation_retirement.py b/tests/unit/test_vector_generation_retirement.py new file mode 100644 index 0000000000..b6ee0ec5f5 --- /dev/null +++ b/tests/unit/test_vector_generation_retirement.py @@ -0,0 +1,152 @@ +"""Generation cleanup adds no suspension unless an old client must close.""" + +from __future__ import annotations + +import asyncio +from typing import Any + +import pytest + +from server.core.vectors import admission +from server.settings import settings +from tests.unit.test_vector_read_admission import Driver, client + + +@pytest.mark.parametrize("vector", [False, True], ids=["ordinary", "vector"]) +@pytest.mark.parametrize("retired_busy", [False, True], ids=["current", "retired-still-used"]) +async def test_completed_query_does_not_yield_for_noop_generation_retirement( + monkeypatch: pytest.MonkeyPatch, vector: bool, retired_busy: bool +) -> None: + gate = admission._Admission() + monkeypatch.setattr(admission, "_admission", gate) + ch, driver = client(monkeypatch, pool=2) + driver.release.set() + replacement = Driver() + replacement.release.set() + closed: list[bool] = [] + + async def connect() -> None: + ch._client = replacement + + async def close() -> None: + closed.append(True) + + monkeypatch.setattr(ch, "connect", connect) + monkeypatch.setattr(driver, "close", close, raising=False) + query = driver.query + + async def complete(sql: str, **kwargs: Any) -> Any: + result = await query(sql, **kwargs) + if retired_busy: + await ch._reconnect(seen=0) + return result + + monkeypatch.setattr(driver, "query", complete) + if retired_busy: + # One other owner still uses generation zero when this query reconnects. + ch._inflight[0] = 1 + turns: list[str] = [] + scheduled = asyncio.get_running_loop().call_soon(turns.append, "next-turn") + try: + with admission.vector_work("one-org", "private", int(vector)): + assert await ch.query_all("SELECT 1") == [] + assert turns == [], "a completed query must return before the next event-loop turn" + assert driver.started == 1 and driver.active == 0 + assert gate.population == 0 and gate.orgs == {} + assert ch._slots is not None and ch._slots._value == 2 + assert closed == [] + if retired_busy: + assert ch.generation == 1 + assert ch._inflight == {0: 1} + assert ch._retired == {0: driver} + else: + assert ch.generation == 0 + assert ch._inflight == {} and ch._retired == {} + if vector: + assert driver.settings[0]["read_overflow_mode"] == "throw" + assert driver.settings[0]["max_rows_to_read"] == settings.vector_scan_max_rows_to_read + finally: + scheduled.cancel() + if retired_busy: + ch._inflight[0] = 0 + await ch._retire_if_done(0) + assert ch._inflight == {} and ch._retired == {} + assert closed == ([True] if retired_busy else []) + assert admission.current_work() is None + + +@pytest.mark.parametrize( + ("failed", "cancelled"), + [(False, False), (False, True), (True, True)], + ids=["success", "repeated-cancellation", "first-error-with-repeated-cancellation"], +) +async def test_real_generation_close_keeps_credit_until_owned_cleanup_finishes( + monkeypatch: pytest.MonkeyPatch, failed: bool, cancelled: bool +) -> None: + gate = admission._Admission() + monkeypatch.setattr(admission, "_admission", gate) + ch, driver = client(monkeypatch, pool=2) + driver.release.set() + replacement = Driver() + replacement.release.set() + close_started = asyncio.Event() + release_close = asyncio.Event() + closed: list[bool] = [] + original = RuntimeError("driver failure before generation cleanup") + query = driver.query + + async def connect() -> None: + ch._client = replacement + + async def close() -> None: + close_started.set() + await release_close.wait() + closed.append(True) + + async def reconnect_during_query(sql: str, **kwargs: Any) -> Any: + result = await query(sql, **kwargs) + await ch._reconnect(seen=0) + if failed: + raise original + return result + + monkeypatch.setattr(ch, "connect", connect) + monkeypatch.setattr(driver, "close", close, raising=False) + monkeypatch.setattr(driver, "query", reconnect_during_query) + + async def read() -> Any: + with admission.vector_work("one-org", "private"): + return await ch.query_all("SELECT 1") + + caller = asyncio.create_task(read()) + try: + async with asyncio.timeout(1): + await close_started.wait() + assert ch.generation == 1 and ch._client is replacement + assert ch._inflight == {} and ch._retired == {} + assert closed == [] and not caller.done() + assert gate.population == 1 and gate.orgs["one-org"].held == 1 + assert ch._slots is not None and ch._slots._value == 1 + if cancelled: + caller.cancel() + await asyncio.sleep(0) + caller.cancel() + await asyncio.sleep(0) + assert not caller.done() and closed == [] + assert gate.population == 1 and gate.orgs["one-org"].held == 1 + assert ch._slots._value == 1 + finally: + release_close.set() + results = await asyncio.gather(caller, return_exceptions=True) + assert closed == [True] + if failed: + assert results[0] is original + elif cancelled: + assert isinstance(results[0], asyncio.CancelledError) + else: + assert results == [[]] + assert driver.started == 1 and driver.active == 0 + assert gate.population == 0 and gate.orgs == {} + assert ch._slots is not None and ch._slots._value == 2 + assert ch._inflight == {} and ch._retired == {} + assert admission.current_work() is None diff --git a/tests/unit/test_vector_supplemental_no_wait_policy.py b/tests/unit/test_vector_supplemental_no_wait_policy.py new file mode 100644 index 0000000000..5eaaa94f1b --- /dev/null +++ b/tests/unit/test_vector_supplemental_no_wait_policy.py @@ -0,0 +1,212 @@ +"""Completion-sensitive reads refuse waits without changing ordinary admission.""" + +from __future__ import annotations + +import asyncio +from typing import Any + +import pytest +from clickhouse_connect.driver.exceptions import DatabaseError + +from server.core.errors import VectorAdmissionError +from server.core.vectors import admission, quantized_setting +from server.settings import settings +from tests.unit.test_vector_read_admission import client, settled + + +def _gate(monkeypatch: pytest.MonkeyPatch) -> admission._Admission: + gate = admission._Admission() + monkeypatch.setattr(admission, "_admission", gate) + monkeypatch.setattr(settings, "vector_scan_max_concurrent_per_org", 2) + monkeypatch.setattr(settings, "vector_scan_max_waiters_per_org", 256) + return gate + + +async def test_weighted_immediate_grant_refusal_other_org_and_reuse(monkeypatch) -> None: + gate = _gate(monkeypatch) + work = admission.VectorWork("one", ("private",), 2) + other = admission.VectorWork("two", ("private",), 2) + assert not admission._without_wait.get() + with admission._without_admission_wait(): + first, _ = await gate.enter(work, 1) + try: + assert gate.population == 2 and first.held == 2 + # Clearing statement attribution cannot clear the owner's wait policy. + with admission.vector_work("", (), 0): + with admission._without_admission_wait(): + with pytest.raises(VectorAdmissionError) as caught: + await gate.enter(work, 1) + assert admission._without_wait.get() + assert caught.value.extra == {"reason": "org_queue", "retry_after_s": 1} + assert first.waiting == 0 and not first.queue and gate.population == 2 + second, _ = await gate.enter(other, 1) + assert second.held == 2 and gate.population == 4 + gate.leave(other, second) + finally: + gate.leave(work, first) + reused, _ = await gate.enter(work, 1) + gate.leave(work, reused) + assert not admission._without_wait.get() + assert admission.current_work() is None + assert gate.population == 0 and gate.orgs == {} + + +async def test_sensitive_work_does_not_jump_an_older_weighted_fifo_head(monkeypatch) -> None: + gate = _gate(monkeypatch) + one = admission.VectorWork("one", ("private",), 1) + two = admission.VectorWork("one", ("private",), 2) + held, _ = await gate.enter(one, 1) + older = asyncio.create_task(gate.enter(two, 1)) + try: + await asyncio.sleep(0) + assert held.held == 1 and held.waiting == 2 and len(held.queue) == 1 + with admission._without_admission_wait(): + with pytest.raises(VectorAdmissionError) as caught: + await gate.enter(one, 1) + assert caught.value.extra["reason"] == "org_queue" + assert held.held == 1 and held.waiting == 2 and len(held.queue) == 1 + assert gate.population == 3 and not older.done() + finally: + older.cancel() + result = await asyncio.gather(older, return_exceptions=True) + gate.leave(one, held) + assert isinstance(result[0], asyncio.CancelledError) + assert gate.population == 0 and gate.orgs == {} + + +async def test_inherited_policy_resets_before_ordinary_wait_and_cancellation(monkeypatch) -> None: + gate = _gate(monkeypatch) + full = admission.VectorWork("one", ("private",), 2) + one = admission.VectorWork("one", ("private",), 1) + held, _ = await gate.enter(full, 1) + tasks: list[asyncio.Task[Any]] = [] + try: + with admission._without_admission_wait(): + sensitive = asyncio.create_task(gate.enter(one, 1)) + tasks.append(sensitive) + with pytest.raises(VectorAdmissionError): + await sensitive + assert not admission._without_wait.get() + ordinary = asyncio.create_task(gate.enter(one, 1)) + tasks.append(ordinary) + await asyncio.sleep(0) + assert not ordinary.done() + assert held.waiting == 1 and gate.population == 3 + ordinary.cancel() + with pytest.raises(asyncio.CancelledError): + await ordinary + assert held.waiting == 0 and gate.population == 2 + finally: + for task in tasks: + if not task.done(): + task.cancel() + await asyncio.gather(*tasks, return_exceptions=True) + gate.leave(full, held) + assert not admission._without_wait.get() + assert gate.population == 0 and gate.orgs == {} + + +@pytest.mark.parametrize("quantized_retry", [False, True], ids=["plain", "quantized-retry"]) +async def test_public_query_holds_nonwaiting_charge_through_pool_wait_and_retry( + monkeypatch: pytest.MonkeyPatch, quantized_retry: bool +) -> None: + from types import SimpleNamespace + + gate = _gate(monkeypatch) + ch, driver = client(monkeypatch, pool=1) + calls: list[tuple[str, dict[str, Any]]] = [] + by_owner: dict[int, list[str]] = {} + state = quantized_setting.quantized_codes_state() + quantized_setting.set_quantized_codes_supported(True) + first = asyncio.Event() + release = asyncio.Event() + + async def query(sql: str, **kwargs: Any) -> Any: + calls.append((sql, kwargs["settings"])) + owner = kwargs["parameters"]["owner"] + attempts = by_owner.setdefault(owner, []) + attempts.append(sql) + first.set() + await release.wait() + if quantized_retry and owner == 1 and len(attempts) == 1: + raise DatabaseError("Code: 115. Unknown setting vector_search_use_quantized_codes") + return SimpleNamespace(result_rows=[(1,)], column_names=["value"], summary={}) + + monkeypatch.setattr(driver, "query", query) + plain = "SELECT {owner:UInt8} * 0 + 1 AS value SETTINGS max_threads=1" + sql = plain + if quantized_retry: + sql += ", vector_search_use_quantized_codes = 1" + + async def read(owner: int) -> list[dict[str, Any]]: + with admission._without_admission_wait(), admission.vector_work("one", "private"): + return await ch.query_all(sql, {"owner": owner}) + + tasks = [asyncio.create_task(read(1))] + try: + async with asyncio.timeout(1): + await first.wait() + tasks.append(asyncio.create_task(read(2))) + await settled() + assert len(calls) == 1 + assert gate.population == 2 and gate.orgs["one"].held == 2 + assert gate.orgs["one"].waiting == 0 and not gate.orgs["one"].queue + assert ch._slots is not None and ch._slots._value == 0 + with pytest.raises(VectorAdmissionError) as caught: + await read(3) + assert caught.value.extra["reason"] == "org_queue" + assert gate.population == 2 and len(calls) == 1 + release.set() + assert await asyncio.gather(*tasks) == [[{"value": 1}], [{"value": 1}]] + assert len(calls) == (3 if quantized_retry else 2) + assert by_owner == {1: [sql, plain] if quantized_retry else [sql], 2: [sql]} + if quantized_retry: + assert not quantized_setting.quantized_codes_supported() + for _, caps in calls: + assert caps["max_rows_to_read"] == settings.vector_scan_max_rows_to_read + assert caps["read_overflow_mode"] == "throw" + assert 0 < caps["max_execution_time"] <= settings.ch_caller_shaped_max_execution_s + assert 0 < caps["max_memory_usage"] <= settings.ch_caller_shaped_max_memory_bytes + finally: + release.set() + for task in tasks: + if not task.done(): + task.cancel() + await asyncio.gather(*tasks, return_exceptions=True) + quantized_setting.set_quantized_codes_supported(state) + assert all(task.done() for task in tasks) + assert ch._slots is not None and ch._slots._value == 1 + assert ch._inflight == {} and ch._retired == {} + assert gate.population == 0 and gate.orgs == {} + assert not admission._without_wait.get() and admission.current_work() is None + + +async def test_policy_scope_resets_when_its_owner_is_cancelled() -> None: + started = asyncio.Event() + released = asyncio.Event() + restored: list[bool] = [] + + async def owner() -> None: + try: + with admission._without_admission_wait(): + assert admission._without_wait.get() + started.set() + await released.wait() + finally: + restored.append(admission._without_wait.get()) + + task = asyncio.create_task(owner()) + try: + async with asyncio.timeout(1): + await started.wait() + assert not admission._without_wait.get() + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + finally: + released.set() + if not task.done(): + task.cancel() + await asyncio.gather(task, return_exceptions=True) + assert restored == [False] + assert not admission._without_wait.get() From e81a519ae94ccdcdb87e1ba4ce55b42299181c1e Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Wed, 18 Nov 2026 07:36:10 +0000 Subject: [PATCH 08/28] Propagate vector capacity refusals through evidence readers Preserve typed refusals through identifier, document and widening owners while retaining ordinary optional-read fallbacks. Join both temporal operands before returning the first failure, including through repeated cancellation. Add a native row-budget CI case. Signed-off-by: zlareb1 --- server/knowledge/app/source_passages.py | 7 + server/knowledge/app/source_search_passes.py | 5 + server/memory/app/agent_rules.py | 5 +- server/memory/app/ask/context_blocks.py | 3 + server/memory/app/ask/identifiers.py | 19 +- server/memory/app/retrieval/derived.py | 26 +- server/memory/app/stack.py | 3 + server/memory/app/stack_knowledge.py | 3 + .../integration/test_vector_native_budget.py | 64 +++ .../test_vector_capacity_owner_escapes.py | 518 ++++++++++++++++++ 10 files changed, 645 insertions(+), 8 deletions(-) create mode 100644 tests/integration/test_vector_native_budget.py create mode 100644 tests/unit/test_vector_capacity_owner_escapes.py diff --git a/server/knowledge/app/source_passages.py b/server/knowledge/app/source_passages.py index 56005f50fb..b58d513939 100644 --- a/server/knowledge/app/source_passages.py +++ b/server/knowledge/app/source_passages.py @@ -8,6 +8,7 @@ from dataclasses import dataclass, field, replace from server.core.diagnostics import private_work +from server.core.errors import VectorCapacityError from server.knowledge.api.models_knowledge import _SearchHit from server.knowledge.app import knowledge_search from server.knowledge.app.knowledge_search import CheckedSearch, SearchRequest @@ -92,6 +93,8 @@ async def _fallback(self) -> list[_SearchHit]: if self._wide is not None: try: wide = (await self._wide.ordinary()).results + except VectorCapacityError: + raise except Exception: # noqa: BLE001 — the first complete ordinary pass still answers self._wide = None self.hits = await self._select(first, wide) @@ -119,6 +122,8 @@ async def current(self) -> list[_SearchHit]: if self._wide is not None: try: wide = (await self._wide.current()).results + except VectorCapacityError: + raise except Exception: # noqa: BLE001 — discard mixed work if an optional renewal fails return await self._fallback() if not self.fresh(): @@ -163,6 +168,8 @@ async def wider(width: int) -> list[_SearchHit]: if width > top_k: try: await wider(width) + except VectorCapacityError: + raise except Exception: # noqa: BLE001 — a failed optional wide pass does not replace the first wide = None result = CheckedPassages(first, wide, agent, top_k, keep, []) diff --git a/server/knowledge/app/source_search_passes.py b/server/knowledge/app/source_search_passes.py index c615c8033f..f7f269c250 100644 --- a/server/knowledge/app/source_search_passes.py +++ b/server/knowledge/app/source_search_passes.py @@ -6,6 +6,7 @@ from server.core.ch import sequential_reads, set_query_tag from server.core.diagnostics import private_work +from server.core.errors import VectorCapacityError from server.knowledge.app.search import ChunkHit, HybridSearchQuery from server.knowledge.app.source_access import Admitted, ReadPlan from server.knowledge.app.source_ranking import RankedHit, ordinary_families, rank @@ -95,6 +96,8 @@ async def collect( ordinary=ordinary_families({leg: ordinary.get(leg, []) for leg in legs}), ) mirrored = await hydrate(self.plan, candidates, model=query.embedding_model) + except VectorCapacityError: + raise except Exception: # noqa: BLE001 — a failed conditional pass must disclose no partial mirrors self.plan.failed = True self._passes.append(_Pass(ordinary, mirrored, legs, fallback, top_k, lexical_weight)) @@ -118,6 +121,8 @@ async def finish(self) -> list[list[RankedHit]]: if self.proofs.invalid: raise ValueError("invalid final search admission") return [selected.ranked(checked) for selected in self._passes] + except VectorCapacityError: + raise except Exception: # noqa: BLE001 — final candidate failure invalidates every earlier mirror pass self.plan.failed = True self.proofs = SearchProofs(self.plan, "", ()) diff --git a/server/memory/app/agent_rules.py b/server/memory/app/agent_rules.py index 1c09de7ddc..d68f8ced7c 100644 --- a/server/memory/app/agent_rules.py +++ b/server/memory/app/agent_rules.py @@ -9,6 +9,7 @@ import logging from collections.abc import Awaitable, Callable +from server.core.errors import VectorCapacityError from server.memory.domain import assistant_rules log = logging.getLogger(__name__) @@ -39,7 +40,7 @@ async def served_hits[H]( """The first `top_k` knowledge hits that serve `agent` (and that `keep` keeps, when given). When a dropped hit took a place among `hits` and more can be asked for, `widen(n)` searches again for `n` hits; if that search fails, the hits served from - the first pass answer.""" + the first pass answer. Capacity refusals propagate.""" def serves(h: H) -> bool: return assistant_rules.serves_agent(tags_of(h), agent) and (keep is None or keep(h)) @@ -53,6 +54,8 @@ def serves(h: H) -> bool: try: wider = await widen(wider_k) served = [h for h in wider if serves(h)] + except VectorCapacityError: + raise except Exception as e: # noqa: BLE001 — the first pass's served hits still answer log.warning("knowledge search widened past dropped hits failed: %s", e) return served[:top_k] diff --git a/server/memory/app/ask/context_blocks.py b/server/memory/app/ask/context_blocks.py index 8ac4189a37..a9a09bfb11 100644 --- a/server/memory/app/ask/context_blocks.py +++ b/server/memory/app/ask/context_blocks.py @@ -7,6 +7,7 @@ from dataclasses import dataclass from typing import Any +from server.core.errors import VectorCapacityError from server.knowledge.app.source_corpus import LostCorpus from server.knowledge.app.source_passages import CheckedPassages, collect from server.memory.app.ask.source_context import PreparedKnowledge, prepare_knowledge @@ -116,6 +117,8 @@ async def knowledge_passages( return AskKnowledge(passages) except LostCorpus: raise + except VectorCapacityError: + raise except Exception: # noqa: BLE001 — an unavailable additive plane contributes no passages logger.warning("knowledge plane on ask unavailable") return AskKnowledge() diff --git a/server/memory/app/ask/identifiers.py b/server/memory/app/ask/identifiers.py index 6a6892f5f6..7943cdb4ab 100644 --- a/server/memory/app/ask/identifiers.py +++ b/server/memory/app/ask/identifiers.py @@ -9,6 +9,7 @@ from collections.abc import Awaitable, Callable from typing import Any +from server.core.errors import VectorCapacityError from server.memory.api.models_search import _MemorySearchHit from server.memory.app.ask.question_shapes import ( is_decisive_question, @@ -110,7 +111,8 @@ def ordinal_line_markers(query_text: str) -> list[str]: async def _mention_window_chunks(search_input: Any, question: str, ranked: list[Any]) -> list[Any]: """Chunks from the sessions dated inside the question's resolved relative window (±1 day), cosine-ranked, as injectable hits. Empty - when the question resolves to no date. Fail-open.""" + when the question resolves to no date. Ordinary failures return no chunks; + capacity refusals propagate.""" from server.settings import settings as _s3000 _n = int(_s3000.memory_ask_mention_window_chunks) @@ -149,6 +151,8 @@ async def _mention_window_chunks(search_input: Any, question: str, ranked: list[ agent=getattr(search_input, "agent", "*"), user_id=getattr(search_input, "user_id", None), ) + except VectorCapacityError: + raise except Exception as e: # noqa: BLE001 — additive, never fatal logger.debug("mention-window chunks skipped: %s", e) return [] @@ -336,7 +340,8 @@ async def _chase_identifiers( the identifiers the previous hop's memories named; chased rows rank just under the direct injections (0.9) so the direct hit stays first. Stops when a hop names nothing new, adds nothing new, or `cap` rows are held. - Returns (rows added, hops walked). Fail-open: an error ends the walk.""" + Returns (rows added, hops walked). Ordinary errors end the walk; + capacity refusals propagate.""" added: list[Any] = [] seen_all = list(seen) walked = 0 @@ -348,6 +353,8 @@ async def _chase_identifiers( seen_all.extend(idents) try: rows = await _inject_identifier_memories(search_input, idents, have + added) + except VectorCapacityError: + raise except Exception as e: # noqa: BLE001 — additive, never fatal logger.debug("identifier chase stopped: %s", e) break @@ -471,6 +478,8 @@ async def _read_decisive_chunks( user_id=getattr(search_input, "user_id", None), markers=list(_DECISIVE_MARKERS), ) + except VectorCapacityError: + raise except Exception as e: # noqa: BLE001 — additive logger.debug("decisive chunk injection failed: %s", e) return [] @@ -518,6 +527,8 @@ async def _read_latest_word( user_id=getattr(search_input, "user_id", None), order="recency", ) + except VectorCapacityError: + raise except Exception as e: # noqa: BLE001 — additive logger.debug("recent identifier injection failed: %s", e) return rows @@ -614,6 +625,8 @@ async def _inject_identifier_memories( agent=getattr(search_input, "agent", "*"), user_id=getattr(search_input, "user_id", None), ) + except VectorCapacityError: + raise except Exception as e: # noqa: BLE001 — chunks are additive too logger.debug("identifier chunk injection failed: %s", e) rows = list(rows) + await _read_decisive_chunks( @@ -622,6 +635,8 @@ async def _inject_identifier_memories( rows = list(rows) + await _read_latest_word( search_input, idents, case_insensitive, _lim_settings ) + except VectorCapacityError: + raise except Exception as e: # noqa: BLE001 — additive, never fatal logger.debug("identifier injection failed: %s", e) return [] diff --git a/server/memory/app/retrieval/derived.py b/server/memory/app/retrieval/derived.py index 9185c33c49..6f51bc3c4e 100644 --- a/server/memory/app/retrieval/derived.py +++ b/server/memory/app/retrieval/derived.py @@ -12,6 +12,7 @@ from typing import Any from server.core.errors import VectorCapacityError +from server.core.shielded import to_its_end from server.core.vectors.admission import vector_work from server.core.vectors.search_settings import vector_search_settings from server.memory.app.search_llm import expand_verb_synonyms @@ -494,8 +495,8 @@ async def fetch_temporal_diff( elif len(operand_vecs) != 2: return None - resolved = await asyncio.gather( - *[ + tasks = [ + asyncio.create_task( _resolve_operand_event_date( ch, org=org, @@ -506,9 +507,24 @@ async def fetch_temporal_diff( embedding_model=embedding_model, run_id=run_id, ) - for v in operand_vecs - ] - ) + ) + for v in operand_vecs + ] + try: + resolved = await asyncio.gather(*tasks) + except BaseException: + for task in tasks: + task.cancel() + + async def stopped() -> None: + await asyncio.gather(*tasks, return_exceptions=True) + + try: + await to_its_end(stopped(), name="temporal-operand-cleanup") + except asyncio.CancelledError: + # The first operand failure stays the answer after every sibling settles. + pass + raise found = [r for r in resolved if r is not None] if len(found) != len(resolved): return None diff --git a/server/memory/app/stack.py b/server/memory/app/stack.py index b780d62599..6eb35ba0a2 100644 --- a/server/memory/app/stack.py +++ b/server/memory/app/stack.py @@ -11,6 +11,7 @@ from server.core.ch import get_client from server.core.diagnostics import private_work +from server.core.errors import VectorCapacityError from server.identity.app.request_scope import request_scope from server.identity.domain.read_scope import scope_and from server.knowledge.app.source_access import ReadPlan @@ -354,6 +355,8 @@ async def run_checked_stack(input: MemoryStackInput, plan: ReadPlan) -> CheckedS knowledge = await stack_knowledge.collect( input.org, input.namespace, input.agent, seed, int(input.knowledge_top_k), plan ) + except VectorCapacityError: + raise except Exception as e: # noqa: BLE001 — a failed knowledge section must not fail the stack log.warning("memory.stack: knowledge fan-out failed (non-fatal): %s", e) return CheckedStack(fill, knowledge, plan.marked) diff --git a/server/memory/app/stack_knowledge.py b/server/memory/app/stack_knowledge.py index d5b15317d9..56cc113f5b 100644 --- a/server/memory/app/stack_knowledge.py +++ b/server/memory/app/stack_knowledge.py @@ -5,6 +5,7 @@ from dataclasses import dataclass from server import embedding +from server.core.errors import VectorCapacityError from server.knowledge.app import pg_serving, search from server.knowledge.app.knowledge_search import not_noise from server.knowledge.app.search import ChunkHit, HybridSearchQuery @@ -106,6 +107,8 @@ async def collect_pass(width: int, fallback: list[ChunkHit]) -> None: try: wider = await ordinary(width) await collect_pass(width, wider) + except VectorCapacityError: + raise except Exception: # noqa: BLE001 — a wider ordinary failure must preserve the usable first pass wider = None pools = await passes.finish() diff --git a/tests/integration/test_vector_native_budget.py b/tests/integration/test_vector_native_budget.py new file mode 100644 index 0000000000..7aff71037d --- /dev/null +++ b/tests/integration/test_vector_native_budget.py @@ -0,0 +1,64 @@ +"""Native vector row ceilings refuse complete reads and release their credits.""" + +from __future__ import annotations + +import pytest +from clickhouse_connect.driver.exceptions import DatabaseError + +from server.core import ch +from server.core.ch_errors import error_code +from server.core.errors import VectorAdmissionError +from server.core.vectors import admission +from server.settings import settings +from tests.integration._resources import unique_org + +pytestmark = pytest.mark.usefixtures("ch_target_required") + + +async def test_native_row_ceiling_refuses_the_read_and_releases_credits( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(settings, "vector_scan_max_rows_to_read", 16) + monkeypatch.setattr(settings, "vector_scan_max_concurrent_per_org", 2) + monkeypatch.setattr(settings, "vector_scan_max_waiters_per_org", 2) + monkeypatch.setattr(settings, "ch_pool_maxsize", 2) + monkeypatch.setattr(settings, "ch_pool_wait_ms", 500) + org = unique_org("vector-native-budget") + sql = "SELECT sum(number) AS total FROM numbers({rows:UInt64})" + client = ch.CHClient() + gate = admission._admission + population_before = gate.population + pool_before = ch.pool_reading() + work_before = admission.current_work() + try: + with ch.query_tag("itest.vector-native-budget"), admission.vector_work("", (), 0): + await client.connect() + assert await client.query_all(sql, {"rows": 64}) == [{"total": 2016}] + with admission.vector_work(org, "native-budget", weight=2): + result: object = None + with pytest.raises(VectorAdmissionError) as caught: + result = await client.query_all(sql, {"rows": 64}) + assert result is None + assert type(caught.value) is VectorAdmissionError + assert caught.value.code == "VECTOR_CAPACITY_REFUSED" + assert caught.value.status_code == 429 + assert caught.value.extra == {"reason": "rows", "retry_after_s": 1} + assert caught.value.headers() == {"Retry-After": "1"} + assert isinstance(caught.value.__cause__, DatabaseError) + assert error_code(caught.value.__cause__) == 158 + assert gate.population == population_before + assert org not in gate.orgs + assert ch.pool_reading() == pool_before + assert client._inflight == {} + assert client._retired == {} + + assert await client.query_all(sql, {"rows": 4}) == [{"total": 6}] + assert admission.current_work() is None + assert gate.population == population_before + assert org not in gate.orgs + assert ch.pool_reading() == pool_before + assert client._inflight == {} + assert client._retired == {} + finally: + await client.disconnect() + assert admission.current_work() == work_before diff --git a/tests/unit/test_vector_capacity_owner_escapes.py b/tests/unit/test_vector_capacity_owner_escapes.py new file mode 100644 index 0000000000..898a30a466 --- /dev/null +++ b/tests/unit/test_vector_capacity_owner_escapes.py @@ -0,0 +1,518 @@ +"""A refused native read fails its owner; ordinary optional failures still fall back.""" + +from __future__ import annotations + +import asyncio +import re +from dataclasses import asdict +from datetime import date +from types import SimpleNamespace +from typing import Any + +import pytest + +from server.core import pg, request_work +from server.core.errors import ( + VectorAdmissionError, + VectorCapacityError, + VectorInputTooLargeError, + VectorOverloadedError, +) +from server.core.vectors.admission import current_work +from server.knowledge.api.models_knowledge import KnowledgeSearchInput, _SearchHit +from server.knowledge.app import knowledge_search, source_passages, source_search_passes +from server.knowledge.app.knowledge_search import SearchRequest +from server.knowledge.app.search import ChunkHit, HybridSearchQuery +from server.knowledge.app.source_access import ReadPlan +from server.memory.api.models_lifecycle import MemoryStackInput +from server.memory.app import stack, stack_knowledge +from server.memory.app.ask import context_blocks, identifiers +from server.memory.app.ask.source_context import PreparedKnowledge +from server.memory.app.retrieval import derived +from server.settings import settings +from tests.unit.ch_fake import RecordingCH + +BODY = "The complete ordinary passage stays intact.\nIts second line is retained too." +FACT = { + "memory_key": "fact:known", + "memory_type": "semantic", + "content_excerpt": "The complete identifier fact cites 109741 and retains every word.", + "created_at": "2025-01-01 12:00:00", + "valid_at": "2025-01-01 12:00:00", +} +NS = "acl-" + "1" * 24 +SID = "11111111-1111-4111-8111-111111111111" +NEXT = "0b5f2a1c-7e3d-4f60-9a2b-1c2d3e4f5a6b" + + +def _failure(status: int, capacity: bool) -> Exception: + if not capacity: + return RuntimeError("ordinary optional read unavailable") + if status == 429: + return VectorAdmissionError("native row budget refused", reason="rows", retry_after_s=7) + if status == 422: + return VectorInputTooLargeError("complete vector input refused") + assert status == 503 + return VectorOverloadedError("native pool refused", reason="pool", retry_after_s=7) + + +def _assert_refusal(actual: Exception, expected: Exception, status: int) -> None: + assert actual is expected + assert actual.status_code == status + code = {429: "VECTOR_CAPACITY_REFUSED", 422: "VECTOR_INPUT_TOO_LARGE", 503: "OVERLOADED"} + detail = {"code": code[status], "message": str(expected)} + if status != 422: + detail.update(reason="rows" if status == 429 else "pool", retry_after_s=7) + assert actual.to_detail() == detail + assert actual.headers() == (None if status == 422 else {"Retry-After": "7"}) + + +def _chunk_row(*, excluded: bool = False) -> dict[str, Any]: + return { + "namespace": "docs", + "source_uri": "ordinary:excluded" if excluded else "ordinary:complete", + "chunk_idx": 1 if excluded else 0, + "doc_version": 10, + "content": "Another bot's complete rule." if excluded else BODY, + "context": "stored context", + "tags": ["source-kind:assistant_turn", "agent:other"] if excluded else ["doc"], + "chunk_metadata": "{}", + "distance": 0.3 if excluded else 0.2, + "agent": "-", + "session_id": "-", + "source_class": "", + } + + +def _chunk_hit() -> ChunkHit: + return ChunkHit( + "docs", + "ordinary:complete", + 0, + BODY, + ["doc"], + 0.2, + doc_version=10, + context="stored context", + score=0.8, + ) + + +def _passage_hit() -> _SearchHit: + return _SearchHit( + namespace="docs", + source_uri="ordinary:complete", + chunk_idx=0, + content=BODY, + tags=["doc"], + distance=0.2, + similarity=0.8, + score=0.8, + ) + + +@pytest.fixture +def native(monkeypatch: pytest.MonkeyPatch): + """Use real handler/store code; substitute only CH and the embedder.""" + token = request_work._current.set(None) + request_work.begin() + state = SimpleNamespace(site="", failure=None, refused=[], vector_widths=[]) + + class Embedder: + model_name = "fake" + + async def embed(self, _text: str) -> list[float]: + return [1.0, 0.0] + + def answer(sql: str, params: dict[str, Any]) -> list[dict[str, Any]]: + is_fact = "FROM loom.memory_embeddings" in sql and "cosineDistance" in sql + is_chunk = "FROM loom.memory_chunks" in sql and "cosineDistance" in sql + is_knowledge = "FROM loom.chunks" in sql and "cosineDistance" in sql + is_mirror = "FORMAT JSONEachRow" in sql + if is_knowledge and not is_mirror: + state.vector_widths.append(params["top_k"]) + refusal = ( + (state.site in {"fact", "chase"} and is_fact) + or (state.site in {"chunk", "mention", "decisive"} and is_chunk) + or (state.site == "mirror" and is_mirror) + or (state.site in {"ask-first", "stack-first"} and is_knowledge) + or ( + state.site in {"passages-wide", "stack-wide", "lazy-wide"} + and is_knowledge + and params["top_k"] > 2 + ) + ) + if refusal: + work = current_work() + assert work is not None and work.org == "acme" and work.weight == 1 + assert work.namespaces == ((NS,) if is_mirror else ("docs",)) + state.refused.append((sql, dict(params))) + raise state.failure + if is_fact: + return [FACT] + if is_chunk: + return [] + if is_knowledge: + rows = [_chunk_row()] + return rows + [_chunk_row(excluded=True)] if state.site == "lazy-wide" else rows + if "FROM loom.memory FINAL" in sql: + return [] # The real profile, pinned and recent sections have no corpus rows. + raise AssertionError(f"unrouted native read: {' '.join(sql.split())[:200]}") + + ch = RecordingCH().on(re.compile(r".*", re.DOTALL), answer) + ch.install(monkeypatch, "server.memory.app.stack", "server.knowledge.store.source_candidates") + monkeypatch.setattr("server.embedding.get_embedder", Embedder) + monkeypatch.setattr(knowledge_search, "get_embedder", Embedder) + monkeypatch.setattr(pg, "enabled", lambda: False) + monkeypatch.setattr(settings, "knowledge_gaps_enabled", False) + monkeypatch.setattr(settings, "knowledge_noise_filter", False) + monkeypatch.setattr(settings, "memory_ask_identifier_inject_chunks", 4) + monkeypatch.setattr(settings, "memory_ask_decisive_chunks", 4) + monkeypatch.setattr(settings, "memory_ask_mention_window_chunks", 4) + state.ch = ch + try: + yield state + finally: + request_work._current.reset(token) + + +@pytest.mark.parametrize( + "site,status", + [("fact", 429), ("chunk", 422), ("mention", 503), ("decisive", 429), ("chase", 503)], + ids=["fact-429", "chunk-422", "mention-503", "decisive-429", "chase-503"], +) +@pytest.mark.parametrize("capacity", [True, False], ids=["refusal", "ordinary-fallback"]) +async def test_identifier_owners_keep_complete_read_refusals(native, site, status, capacity): + native.site, native.failure = site, _failure(status, capacity) + question = "What do we know about 109741?" + if site == "mention": + question = "We talked about a movie last week. Which one?" + elif site == "decisive": + question = "What caused the bug in 109741?" + request = SimpleNamespace( + org="acme", + namespace="docs", + agent="reader", + user_id="-", + scope="private", + observation_date="2025-01-20", + query=question, + ) + if site == "mention": + call = identifiers._mention_window_chunks(request, question, []) + elif site == "decisive": + call = identifiers._read_decisive_chunks(request, ["109741"], [1.0, 0.0], False, settings) + elif site == "chase": + frontier = [SimpleNamespace(memory_key="start", content_excerpt=f"Value: {NEXT}")] + call = identifiers._chase_identifiers( + request, + seen=["109741"], + frontier=frontier, + ranked=frontier, + hops=2, + cap=8, + ) + else: + call = identifiers._inject_identifier_memories(request, ["109741"], []) + try: + if capacity: + with pytest.raises(VectorCapacityError) as caught: + await asyncio.wait_for(call, timeout=1) + _assert_refusal(caught.value, native.failure, status) + else: + result = await asyncio.wait_for(call, timeout=1) + if site == "chunk": + assert len(result) == 1 + hit = result[0] + assert ( + hit.memory_key, + hit.content_excerpt, + hit.rank_score, + hit.similarity, + hit.distance, + hit.source, + hit.valid_at, + ) == ( + "fact:known", + FACT["content_excerpt"], + 1.0, + 1.0, + 0.0, + "fact", + "2025-01-01 12:00:00", + ) + else: + assert result == (([], 1) if site == "chase" else []) + finally: + assert len(native.refused) == 1 + _sql, params = native.refused[0] + assert params["qv"] == [1.0, 0.0] + if site == "decisive": + assert "root cause" in params["marks"] + elif site == "mention": + assert params["d0"] < params["d1"] + elif site == "chase": + assert params["id0"] == NEXT + + +@pytest.mark.parametrize( + "site,status", + [ + ("mirror", 429), + ("ask-first", 422), + ("passages-wide", 503), + ("stack-first", 503), + ("stack-wide", 422), + ("lazy-wide", 429), + ], + ids=[ + "mirror-429", + "ask-first-422", + "passages-wide-503", + "stack-first-503", + "stack-wide-422", + "lazy-wide-429", + ], +) +@pytest.mark.parametrize("capacity", [True, False], ids=["refusal", "ordinary-fallback"]) +async def test_document_owners_keep_complete_read_refusals(native, site, status, capacity): + native.site, native.failure = site, _failure(status, capacity) + plan = ReadPlan("acme", marked=site != "lazy-wide") + request = SearchRequest( + KnowledgeSearchInput( + org="acme", + namespace="docs", + query="question", + top_k=2, + search_mode="semantic", + namespace_scope="this", + rerank="none", + collapse_sections=False, + narrow_spans=False, + ), + ("docs",), + (), + ) + if site == "mirror": + plan._bindings = ((NS, SID, "docs"),) + passes = source_search_passes.SearchPasses(plan) + ordinary = _chunk_hit() + + async def collect_pass(): + await passes.collect( + HybridSearchQuery("acme", ("docs",), "fake", [1.0, 0.0], 2), + {"semantic": [ordinary]}, + legs=("semantic",), + fallback=[ordinary], + top_k=2, + ) + return await passes.finish() + + call = collect_pass() + elif site == "ask-first": + call = context_blocks.knowledge_passages( + SimpleNamespace(agent="reader"), + "question", + prepared=PreparedKnowledge(request, plan), + ) + elif site == "passages-wide": + call = source_passages.collect(request, plan, "reader") + elif site == "stack-first": + call = stack.run_checked_stack( + MemoryStackInput( + org="acme", + namespace="docs", + agent="reader", + seed_query="question", + include_knowledge=True, + knowledge_top_k=2, + procedure_limit=0, + pinned_limit=0, + recent_limit=0, + ), + plan, + ) + else: + call = stack_knowledge.collect("acme", "docs", "reader", "question", 2, plan) + try: + if capacity: + with pytest.raises(VectorCapacityError) as caught: + await asyncio.wait_for(call, timeout=1) + _assert_refusal(caught.value, native.failure, status) + else: + result = await asyncio.wait_for(call, timeout=1) + if site == "mirror": + assert [[asdict(hit.chunk) for hit in pool] for pool in result] == [ + [asdict(_chunk_hit())] + ] + assert not result[0][0].mirrored + assert plan.failed + elif site == "ask-first": + assert result.passages is None and await result.current() == ("", 0) + elif site == "passages-wide": + assert [hit.model_dump() for hit in await result.current()] == [ + _passage_hit().model_dump() + ] + elif site == "stack-first": + assert (await result.current()).model_dump() == { + "entries": [], + "tokens_used": 0, + "tokens_budget": 4000, + "truncated": False, + "section_counts": { + name: 0 + for name in ("profile", "pinned", "recent_recall", "procedure", "knowledge") + }, + } + else: + assert [asdict(hit) for hit in await result.current()] == [asdict(_chunk_hit())] + finally: + assert len(native.refused) == 1 + if site in {"passages-wide", "stack-wide", "lazy-wide"}: + assert native.vector_widths == [2, 8] + elif site in {"ask-first", "stack-first"}: + assert native.vector_widths == [2] + + +def _temporal_call(ch): + return derived.fetch_temporal_diff( + ch, + org="acme", + namespace="docs", + agent="reader", + user_id="-", + operand_vecs=[[0.0], [1.0]], + embedding_model="fake", + ) + + +@pytest.mark.parametrize( + "failed_operand,status", [(0, 429), (1, 503)], ids=["first-429", "second-503"] +) +async def test_temporal_refusal_joins_the_other_native_operand(failed_operand, status): + sibling_started, sibling_settled = asyncio.Event(), asyncio.Event() + failure = _failure(status, True) + native_tasks = set() + diff_calls = [] + + class CH: + async def query_all(self, sql, params=None): + if "dateDiff" in sql: + diff_calls.append(params) + return [{"days": 1}] + assert "cosineDistance" in sql + native_tasks.add(asyncio.current_task()) + if params["q"] == [float(failed_operand)]: + await sibling_started.wait() + raise failure + sibling_started.set() + try: + await asyncio.Event().wait() + finally: + sibling_settled.set() + + caller = asyncio.create_task(_temporal_call(CH())) + try: + with pytest.raises(VectorCapacityError) as caught: + await asyncio.wait_for(asyncio.shield(caller), timeout=1) + _assert_refusal(caught.value, failure, status) + assert sibling_started.is_set() + assert sibling_settled.is_set(), ( + "the owning handler returned while its native sibling was live" + ) + assert all(task.done() for task in native_tasks) + assert diff_calls == [] + finally: + for task in native_tasks | {caller}: + task.cancel() + await asyncio.wait_for( + asyncio.gather(*native_tasks, caller, return_exceptions=True), timeout=1 + ) + + +async def test_temporal_first_refusal_survives_repeated_cancellation_while_joining(): + started, cleanup_started, release, settled = (asyncio.Event() for _ in range(4)) + failure = _failure(422, True) + native_tasks = set() + + class CH: + async def query_all(self, sql, params=None): + assert "cosineDistance" in sql + native_tasks.add(asyncio.current_task()) + if params["q"] == [0.0]: + await started.wait() + raise failure + started.set() + try: + await asyncio.Event().wait() + finally: + cleanup_started.set() + await release.wait() + settled.set() + + caller = asyncio.create_task(_temporal_call(CH())) + cleanup_wait = asyncio.create_task(cleanup_started.wait()) + try: + done, _ = await asyncio.wait( + {caller, cleanup_wait}, timeout=1, return_when=asyncio.FIRST_COMPLETED + ) + assert done, "the operand pair made no progress" + if caller.done(): + assert caller.exception() is failure + assert cleanup_started.is_set(), "first refusal returned without joining its sibling" + assert not caller.done() and not settled.is_set() + caller.cancel() + await asyncio.sleep(0) + caller.cancel() + await asyncio.sleep(0) + assert not caller.done() and not settled.is_set() + release.set() + with pytest.raises(VectorInputTooLargeError) as caught: + await asyncio.wait_for(asyncio.shield(caller), timeout=1) + _assert_refusal(caught.value, failure, 422) + assert settled.is_set() and all(task.done() for task in native_tasks) + finally: + release.set() + for task in native_tasks | {caller, cleanup_wait}: + if not task.done(): + task.cancel() + await asyncio.wait_for( + asyncio.gather(*native_tasks, caller, cleanup_wait, return_exceptions=True), + timeout=1, + ) + + +async def test_temporal_two_native_operands_keep_the_complete_interval_result(): + calls = [] + + class CH: + async def query_all(self, sql, params=None): + calls.append((sql, dict(params or {}))) + if "dateDiff" in sql: + assert params == {"a": date(2025, 1, 2), "b": date(2025, 1, 5)} + return [{"days": 3}] + assert "cosineDistance" in sql + first = params["q"] == [0.0] + return [ + { + "event_date": date(2025, 1, 2 if first else 5), + "distance": 0.2, + "excerpt": "Complete first event." if first else "Complete second event.", + "memory_key": "first" if first else "second", + "anchored": True, + } + ] + + result = await asyncio.wait_for(_temporal_call(CH()), timeout=1) + assert result == { + "interval_days": 3, + "from_date": "2025-01-02", + "to_date": "2025-01-05", + "from_excerpt": "Complete first event.", + "to_excerpt": "Complete second event.", + "to_observation": False, + "from_event_dated": True, + "to_event_dated": True, + "all_event_dated": True, + } + assert len(calls) == 3 From 0b5cdd02d3e50a5b686249d7ce53011a56d0f607 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Wed, 18 Nov 2026 07:52:01 +0000 Subject: [PATCH 09/28] Propagate vector refusals through outer read owners Optional context and grounding fallbacks must preserve whole-read capacity errors. Join owned siblings and keep the first refusal when cleanup is cancelled. Signed-off-by: zlareb1 --- server/code/app/answers.py | 7 +- server/memory/app/ask/handler.py | 5 +- server/memory/app/ask/pipeline.py | 11 +- server/memory/app/code_grounding.py | 74 ++- server/memory/app/context_pack.py | 37 +- .../unit/test_vector_outer_owner_refusals.py | 484 ++++++++++++++++++ 6 files changed, 594 insertions(+), 24 deletions(-) create mode 100644 tests/unit/test_vector_outer_owner_refusals.py diff --git a/server/code/app/answers.py b/server/code/app/answers.py index d69c9251a7..340977f476 100644 --- a/server/code/app/answers.py +++ b/server/code/app/answers.py @@ -49,6 +49,7 @@ from server.code.app.search import CodeSearchResult from server.code.domain.units import CodeUnit from server.code.store.index import CodeRepo +from server.core.errors import VectorCapacityError from server.observability.genai import staged from server.settings.budgets import llm_max_tokens @@ -439,7 +440,7 @@ async def _inject_memory_context(question: str) -> MemoryContext: memory.ask's code enrichment. Gated by an explicit memory-namespace binding (code_ask_memory_namespace); a no-op when unset. Returns short excerpts with their keys to ground code.ask's synthesis in WHY the code is the way it is. - Fail-open to no excerpts.""" + Ordinary read failures leave no excerpts.""" from server.settings import settings as _s mem_ns = _s.code_ask_memory_namespace.strip() @@ -476,7 +477,9 @@ async def _inject_memory_context(question: str) -> MemoryContext: include_tags=True, # a consolidated row is told by its tags ) ) - except Exception as e: # noqa: BLE001 — additive, never fatal + except VectorCapacityError: + raise + except Exception as e: # noqa: BLE001 — ordinary enrichment failures are optional logger.debug("code.ask memory enrichment failed: %s", e) return MemoryContext() if getattr(out, "abstain", False): diff --git a/server/memory/app/ask/handler.py b/server/memory/app/ask/handler.py index a8df0cab19..872e118fe9 100644 --- a/server/memory/app/ask/handler.py +++ b/server/memory/app/ask/handler.py @@ -14,6 +14,7 @@ import logging from typing import TYPE_CHECKING, Any +from server.core.errors import VectorCapacityError from server.core.llm_free import refuse_synthesis from server.core.naming import named from server.memory.api.models_ask import MemoryAskInput, MemoryAskOutput @@ -495,7 +496,9 @@ async def search(top_k: int) -> list[Any]: route_tier="strong", route_reason=ask_routing.route_reason(synth.documents), ) - except Exception as e: # noqa: BLE001 - the memory path is always available + except VectorCapacityError: + raise + except Exception as e: # noqa: BLE001 — ordinary route failures leave the memory path available log.debug("memory.ask synthesize route skipped: %s", e) return None diff --git a/server/memory/app/ask/pipeline.py b/server/memory/app/ask/pipeline.py index 9cf0572abe..96b4c2fa8d 100644 --- a/server/memory/app/ask/pipeline.py +++ b/server/memory/app/ask/pipeline.py @@ -467,6 +467,7 @@ async def ask( # Every exit below — an early answer, a precedent, an error — leaves the # knowledge read cancelled; after a normal await it is already done. + failed = False try: # `question` is the text the model answers; search_input.query is what # retrieval embedded (the stem, when the caller's query carried an answer @@ -577,6 +578,14 @@ async def ask( generate, knowledge_context, marked=prepared_knowledge.plan.marked ) + except BaseException: + failed = True + raise finally: if not _knowledge_task.done(): - await cancel_and_drain(_knowledge_task) + try: + await cancel_and_drain(_knowledge_task) + except asyncio.CancelledError: + # Join the child, then preserve an already propagating failure. + if not failed: + raise diff --git a/server/memory/app/code_grounding.py b/server/memory/app/code_grounding.py index 7ee7a7392b..fd7e8c70e2 100644 --- a/server/memory/app/code_grounding.py +++ b/server/memory/app/code_grounding.py @@ -11,8 +11,9 @@ entry point. Every step is gated by an explicit code-namespace BINDING (`_resolve_code_target` -— never guesses a repo) and by its own setting, and every step is additive / -fail-open, so a deployment with no bound code index is a complete no-op. +— never guesses a repo) and by its own setting. Ordinary read failures leave the pool +available; capacity refusals fail the complete read. A deployment with no bound code +index is a complete no-op. """ from __future__ import annotations @@ -23,6 +24,8 @@ from typing import Any from server.code.domain.anchor import local_name +from server.core.errors import VectorCapacityError +from server.core.shielded import to_its_end from server.memory.store.code_links import LinkRow logger = logging.getLogger(__name__) @@ -144,7 +147,7 @@ async def _inject_code_context( When both hold, run code.search against that EXPLICIT (org, namespace) and render the top units as `[code index @ ] path:line — symbol …` grounding hits, carrying the authoritative source location + signature + - recent PRs the memory corpus usually lacks. Fail-open to [].""" + recent PRs the memory corpus usually lacks. Ordinary failures return [].""" from server.code.domain import idents _hop_cap = int(settings.memory_ask_code_hop_symbols) @@ -250,15 +253,37 @@ async def _search() -> Any | None: consumer=GROUNDING_CONSUMER, ) ) - except Exception as e: # noqa: BLE001 — additive, never fatal + except VectorCapacityError: + raise + except Exception as e: # noqa: BLE001 — ordinary enrichment failures are optional _warn_once("code-context search fallback failed: %s", e) return None # Hop symbols ride the same concurrent round, one # authoritative definition each, appended after the question's rows. - gathered = await asyncio.gather( - *(_symbol(sym) for sym in exact), *(_symbol(sym) for sym in hop_syms), _search() - ) + tasks = [ + asyncio.create_task(work) + for work in ( + *(_symbol(sym) for sym in exact), + *(_symbol(sym) for sym in hop_syms), + _search(), + ) + ] + try: + gathered = await asyncio.gather(*tasks) + except BaseException: + for task in tasks: + task.cancel() + + async def stopped() -> None: + await asyncio.gather(*tasks, return_exceptions=True) + + try: + await to_its_end(stopped(), name="code-context-read-cleanup") + except asyncio.CancelledError: + # Cleanup cancellation cannot replace the first read failure. + pass + raise symbol_outs, out = list(gathered[: len(exact)]), gathered[-1] hop_outs = list(gathered[len(exact) : len(exact) + len(hop_syms)]) # The exact definitions first, the search fills. @@ -688,14 +713,33 @@ async def _noop_note() -> Any | None: if not target[1]: return synth_pool - gated, code_hits = await asyncio.gather( - _gate_future_prs(synth_pool, search_input, settings, code_target=target) - if settings.memory_ask_pr_temporal_gate - else _noop_gate(), - _inject_code_context(query, search_input, settings, pool=synth_pool, code_target=target) - if settings.memory_ask_code_enrichment - else _noop_hits(), - ) + tasks = [ + asyncio.create_task( + _gate_future_prs(synth_pool, search_input, settings, code_target=target) + if settings.memory_ask_pr_temporal_gate + else _noop_gate() + ), + asyncio.create_task( + _inject_code_context(query, search_input, settings, pool=synth_pool, code_target=target) + if settings.memory_ask_code_enrichment + else _noop_hits() + ), + ] + try: + gated, code_hits = await asyncio.gather(*tasks) + except BaseException: + for task in tasks: + task.cancel() + + async def stopped() -> None: + await asyncio.gather(*tasks, return_exceptions=True) + + try: + await to_its_end(stopped(), name="code-grounding-round-cleanup") + except asyncio.CancelledError: + # Cleanup cancellation cannot replace the first grounding failure. + pass + raise if len(gated) != len(synth_pool): synth_pool = gated if code_hits: diff --git a/server/memory/app/context_pack.py b/server/memory/app/context_pack.py index dc6b8744f9..e0199824f5 100644 --- a/server/memory/app/context_pack.py +++ b/server/memory/app/context_pack.py @@ -16,10 +16,11 @@ from dataclasses import dataclass, field from typing import TYPE_CHECKING, Any, cast -from server.core.background import gather_or_cancel from server.core.diagnostics import private_work +from server.core.errors import VectorCapacityError from server.core.naming import SENTINEL as _WM_SENTINEL from server.core.naming import named +from server.core.shielded import to_its_end from server.identity.app.request_scope import request_scope from server.memory.api.models_context import ContextBlock, ContextPackInput, ContextPackOutput from server.memory.api.models_lifecycle import MemoryStackInput @@ -892,7 +893,9 @@ async def _pack_runs(st: _PackState) -> None: async def _optional_section(name: str, work: Awaitable[Any]) -> Any: try: return await work - except Exception as error: # noqa: BLE001 — a failed optional section leaves other sections available + except VectorCapacityError: + raise + except Exception as error: # noqa: BLE001 — an ordinary section failure leaves other sections available log.warning("context.pack: section %s failed: %s", name, error) return None @@ -915,9 +918,25 @@ async def _gather_sections( _pack_runs(st), ) names = [name for name in _DEFAULT_SECTION_ORDER if name != "decisions"] - results = await gather_or_cancel( - *(_optional_section(name, task) for name, task in zip(names, work, strict=True)) - ) + tasks = [ + asyncio.create_task(_optional_section(name, task)) + for name, task in zip(names, work, strict=True) + ] + try: + results = await asyncio.gather(*tasks) + except BaseException: + for task in tasks: + task.cancel() + + async def stopped() -> None: + await asyncio.gather(*tasks, return_exceptions=True) + + try: + await to_its_end(stopped(), name="context-pack-section-cleanup") + except asyncio.CancelledError: + # Cleanup cancellation cannot replace the first section failure. + pass + raise return cast("CheckedStack | None", results[5]), cast("CheckedPassages | None", results[8]) @@ -1091,11 +1110,15 @@ async def _fallback(self) -> ContextPackOutput: if self._direct is not None: try: direct = await self._direct.ordinary() + except VectorCapacityError: + raise except Exception: # noqa: BLE001 — a failed ordinary section is omitted without retry self._direct = None if self._stack is not None: try: stack = await self._stack.ordinary() + except VectorCapacityError: + raise except Exception: # noqa: BLE001 — a failed ordinary section is omitted without retry self._stack = None self._output = _render_checked_pack(self._prefix, stack, direct) @@ -1124,11 +1147,15 @@ async def current(self) -> ContextPackOutput: if self._direct is not None: try: direct = await self._direct.current() + except VectorCapacityError: + raise except Exception: # noqa: BLE001 — a failed optional section cannot expose a partial mixed pack self._direct, failed = None, True if self._stack is not None: try: stack = await self._stack.current() + except VectorCapacityError: + raise except Exception: # noqa: BLE001 — a failed optional section cannot expose a partial mixed pack self._stack, failed = None, True if failed or not self.fresh(): diff --git a/tests/unit/test_vector_outer_owner_refusals.py b/tests/unit/test_vector_outer_owner_refusals.py new file mode 100644 index 0000000000..6627295dff --- /dev/null +++ b/tests/unit/test_vector_outer_owner_refusals.py @@ -0,0 +1,484 @@ +"""A complete-read refusal survives optional outer owners and owned cleanup.""" + +from __future__ import annotations + +import asyncio +from types import SimpleNamespace as NS + +import pytest + +from server.core.errors import ( + VectorAdmissionError, + VectorCapacityError, + VectorInputTooLargeError, + VectorOverloadedError, +) +from server.knowledge.app.source_access import ReadPlan +from server.memory.app.ask.source_context import PreparedKnowledge +from server.settings import settings +from tests.unit.test_vector_capacity_owner_escapes import native as native + +BODY = "The entire ordinary answer remains available.\nThe second line remains too." + + +def failure(status): + if status == 429: + return VectorAdmissionError("whole read refused", reason="rows", retry_after_s=7) + if status == 422: + return VectorInputTooLargeError("whole input refused") + assert status == 503 + return VectorOverloadedError("whole process refused", reason="pool", retry_after_s=7) + + +def assert_refusal(actual, expected, status): + assert actual is expected + assert actual.status_code == status + detail = { + "code": {429: "VECTOR_CAPACITY_REFUSED", 422: "VECTOR_INPUT_TOO_LARGE", 503: "OVERLOADED"}[ + status + ], + "message": str(expected), + } + if status != 422: + detail.update(reason="rows" if status == 429 else "pool", retry_after_s=7) + assert actual.to_detail() == detail + assert actual.headers() == (None if status == 422 else {"Retry-After": "7"}) + + +async def join_owned(tasks): + for task in tasks: + if not task.done(): + task.cancel() + await asyncio.wait_for(asyncio.gather(*tasks, return_exceptions=True), 1) + + +@pytest.mark.parametrize( + "site,status", + [ + ("memories", 429), + ("procedures", 422), + ("knowledge", 503), + ("memory_stack", 429), + ("conversations", 503), + ], +) +@pytest.mark.parametrize("capacity", [True, False], ids=["refusal", "ordinary-fallback"]) +async def test_public_pack_preserves_requested_section_refusal(monkeypatch, site, status, capacity): + from server.knowledge.app.knowledge_search import SearchRequest + from server.memory.api.models_context import ContextPackInput + from server.memory.app import context_pack + from tests.unit.checked_pack_fakes import install_ordinary_pack + + install_ordinary_pack(monkeypatch) + expected = failure(status) if capacity else RuntimeError("ordinary section unavailable") + calls = [] + + async def profile(inp): + assert (inp.org, inp.namespace) == ("acme", "docs") + return NS(rendered=BODY, labels=["ordinary"], section_count=1) + + async def refused(inp, *rest, **kwargs): + actual = inp.input if isinstance(inp, SearchRequest) else inp + assert (actual.org, actual.namespace) == ("acme", "docs") + assert ( + getattr(actual, "query", None) + or getattr(actual, "situation", None) + or getattr(actual, "seed_query", None) + ) == "What is known about 109741?" + calls.append(actual) + raise expected + + monkeypatch.setattr("server.memory.app.profile.handle_profile_render", profile) + targets = { + "memories": "server.memory.app.search.handle_memory_search", + "procedures": "server.memory.app.procedure.handle_procedure_match", + "knowledge": "server.knowledge.app.source_passages.collect", + "memory_stack": "server.memory.app.stack.run_checked_stack", + "conversations": "server.knowledge.app.conversations.handle_conversation_search", + } + monkeypatch.setattr(targets[site], refused) + inp = ContextPackInput( + agent="reader", + org="acme", + namespace="docs", + query="What is known about 109741?", + only_sections=["profile", site], + include_knowledge=site in {"knowledge", "memory_stack"}, + include_conversations=site == "conversations", + ) + try: + if capacity: + with pytest.raises(VectorCapacityError) as caught: + await context_pack.handle_context_pack(inp) + assert_refusal(caught.value, expected, status) + else: + out = await context_pack.handle_context_pack(inp) + assert out.sections == {"profile": BODY} + assert [(b.section, b.rendered, b.citations, b.source_count) for b in out.blocks] == [ + ("profile", BODY, ["profile:ordinary"], 1) + ] + assert out.section_count == 1 and out.truncated is False + assert BODY in out.rendered + finally: + assert len(calls) == 1 + + +async def test_public_pack_joins_other_requested_section_before_refusal(monkeypatch): + from server.memory.api.models_context import ContextPackInput + from server.memory.app.context_pack import handle_context_pack + + started, settled = asyncio.Event(), asyncio.Event() + tasks = set() + expected = failure(429) + + async def profile(inp): + tasks.add(asyncio.current_task()) + started.set() + try: + await asyncio.Event().wait() + finally: + settled.set() + + async def search(inp): + assert (inp.org, inp.namespace, inp.query) == ("acme", "docs", "109741") + await started.wait() + raise expected + + monkeypatch.setattr("server.memory.app.profile.handle_profile_render", profile) + monkeypatch.setattr("server.memory.app.search.handle_memory_search", search) + caller = asyncio.create_task( + handle_context_pack( + ContextPackInput( + agent="reader", + org="acme", + namespace="docs", + query="109741", + only_sections=["profile", "memories"], + include_knowledge=False, + ) + ) + ) + tasks.add(caller) + try: + await asyncio.wait({caller}, timeout=1) + assert settled.is_set(), ( + "a refused pack did not cancel and join its other requested section" + ) + with pytest.raises(VectorCapacityError) as caught: + await asyncio.wait_for(asyncio.shield(caller), 1) + assert_refusal(caught.value, expected, 429) + assert all(t.done() for t in tasks) + finally: + await join_owned(tasks) + + +@pytest.mark.parametrize("capacity", [True, False], ids=["refusal", "ordinary-fallback"]) +async def test_code_answer_cannot_synthesize_after_bound_memory_refusal(monkeypatch, capacity): + from server.code.app import answers + + expected = failure(503) if capacity else RuntimeError("ordinary memory failure") + reads, syntheses = [], [] + monkeypatch.setattr(settings, "code_ask_memory_namespace", "docs") + monkeypatch.setattr(settings, "code_ask_memory_org", "acme") + + async def allow(*args, **kwargs): + return True + + async def search(inp): + assert (inp.org, inp.namespace, inp.query, inp.namespace_scope, inp.include_tags) == ( + "acme", + "docs", + "Why FooBar?", + "this", + True, + ) + reads.append(inp) + raise expected + + async def synthesize(llm, question, units, *, memories, history): + syntheses.append((question, units, memories, history)) + return BODY + + monkeypatch.setattr("server.auth.authz.row_in_scope", allow) + monkeypatch.setattr("server.memory.app.search.handle_memory_search", search) + monkeypatch.setattr(answers, "_synthesize", synthesize) + monkeypatch.setattr(answers, "_memory_context_cacheable", allow) + ctx = NS(repo=None, org="acme", ns="code", question="Why FooBar?", model="fake") + retrieval = NS(units=[]) + try: + if capacity: + with pytest.raises(VectorCapacityError) as caught: + await answers._grounded_answer(ctx, None, retrieval, ["Prior full turn"]) + assert_refusal(caught.value, expected, 503) + assert syntheses == [] + else: + out = await answers._grounded_answer(ctx, None, retrieval, ["Prior full turn"]) + assert out.answer == BODY and out.source == "synthesized" + assert out.citations == [] and out.memory_keys == [] and out.memory_namespace == "" + assert syntheses == [("Why FooBar?", [], [], ["Prior full turn"])] + finally: + assert len(reads) == 1 + + +@pytest.mark.parametrize("capacity", [True, False], ids=["refusal", "ordinary-fallback"]) +async def test_memory_grounding_preserves_refusal_and_full_ordinary_pool(monkeypatch, capacity): + from server.memory.app import code_grounding as grounding + + expected = failure(422) if capacity else RuntimeError("ordinary code search failure") + calls = [] + pool = [NS(memory_key="known", content_excerpt=BODY)] + + async def target(*args): + return "acme", "code" + + async def search(inp): + assert (inp.org, inp.namespace, inp.query, inp.consumer, inp.top_k) == ( + "acme", + "code", + "where is helper_fn", + "memory.ask", + 3, + ) + calls.append(inp) + raise expected + + monkeypatch.setattr(grounding, "_authorized_code_target", target) + monkeypatch.setattr("server.code.domain.idents.embedded_idents", lambda query: {"helper_fn"}) + monkeypatch.setattr("server.code.app.search.handler.handle_code_search", search) + cfg = settings.model_copy( + update={ + "memory_ask_code_hop_symbols": 0, + "memory_ask_pr_temporal_gate": False, + "memory_ask_code_enrichment": True, + "memory_ask_pr_grounding": False, + "memory_ask_code_staleness": False, + } + ) + try: + if capacity: + with pytest.raises(VectorCapacityError) as caught: + await grounding.apply_code_grounding( + "where is helper_fn", NS(org="acme", namespace="docs"), pool, cfg + ) + assert_refusal(caught.value, expected, 422) + else: + out = await grounding.apply_code_grounding( + "where is helper_fn", NS(org="acme", namespace="docs"), pool, cfg + ) + assert out is pool and out[0].content_excerpt == BODY + finally: + assert len(calls) == 1 + + +async def test_code_grounding_joins_inner_symbol_and_outer_gate_on_refusal(monkeypatch): + from server.memory.app import code_grounding as grounding + + inner_started, outer_started = asyncio.Event(), asyncio.Event() + inner_settled, outer_settled = asyncio.Event(), asyncio.Event() + tasks = set() + expected = failure(429) + + async def target(*args): + return "acme", "code" + + async def block(started, settled): + tasks.add(asyncio.current_task()) + started.set() + try: + await asyncio.Event().wait() + finally: + settled.set() + + async def symbol(inp): + assert inp.name == "FooBar" and inp.consumer == "memory.ask" + return await block(inner_started, inner_settled) + + async def gate(*args, **kwargs): + return await block(outer_started, outer_settled) + + async def search(inp): + assert (inp.org, inp.namespace, inp.consumer) == ("acme", "code", "memory.ask") + await inner_started.wait() + await outer_started.wait() + raise expected + + monkeypatch.setattr(grounding, "_authorized_code_target", target) + monkeypatch.setattr(grounding, "_gate_future_prs", gate) + monkeypatch.setattr("server.code.domain.idents.embedded_idents", lambda query: {"FooBar"}) + monkeypatch.setattr("server.code.app.symbol.handle_code_symbol", symbol) + monkeypatch.setattr("server.code.app.search.handler.handle_code_search", search) + cfg = settings.model_copy( + update={ + "memory_ask_code_hop_symbols": 0, + "memory_ask_pr_temporal_gate": True, + "memory_ask_code_enrichment": True, + "memory_ask_pr_grounding": False, + "memory_ask_code_staleness": False, + } + ) + caller = asyncio.create_task( + grounding.apply_code_grounding("FooBar", NS(org="acme", namespace="docs"), [], cfg) + ) + tasks.add(caller) + try: + await asyncio.wait({caller}, timeout=1) + assert inner_settled.is_set() and outer_settled.is_set(), ( + "refused grounding did not cancel and join both owned fanout levels" + ) + with pytest.raises(VectorCapacityError) as caught: + await asyncio.wait_for(asyncio.shield(caller), 1) + assert_refusal(caught.value, expected, 429) + assert all(t.done() for t in tasks) + finally: + await join_owned(tasks) + + +@pytest.mark.parametrize("wide", [False, True], ids=["initial", "lazy-wide"]) +@pytest.mark.parametrize("capacity", [True, False], ids=["refusal", "ordinary-fallback"]) +async def test_corpus_route_preserves_knowledge_refusal(monkeypatch, wide, capacity): + from server.knowledge.api.models_knowledge import KnowledgeSearchInput + from server.knowledge.app.knowledge_search import SearchRequest + from server.memory.api.models_ask import MemoryAskInput + from server.memory.app.ask import handler + from tests.unit.ask_knowledge_fake import checked_reply + + expected = failure(429) if capacity else RuntimeError("ordinary corpus failure") + widths = [] + + async def search(request, plan): + assert (request.input.org, request.input.namespace, request.input.query) == ( + "acme", + "docs", + "Summarize these reports", + ) + assert plan.org == "acme" + widths.append(request.input.top_k) + if wide and len(widths) == 1: + # A full first page consists entirely of another agent's rules, so real + # served_hits must widen before selecting the corpus documents. + return checked_reply( + NS( + results=[ + NS( + source_uri="doc:excluded", + tags=["source-kind:assistant_turn", "agent:other"], + ) + for _ in range(request.input.top_k) + ] + ) + ) + raise expected + + monkeypatch.setattr("server.knowledge.app.knowledge_search.run_checked_search", search) + inp = MemoryAskInput( + org="acme", + namespace="docs", + agent="reader", + query="Summarize these reports", + knowledge=False, + ) + prepared = PreparedKnowledge( + SearchRequest( + KnowledgeSearchInput(org="acme", namespace="docs", query=inp.query), ("docs",), () + ), + ReadPlan("acme"), + ) + try: + if capacity: + with pytest.raises(VectorCapacityError) as caught: + await handler._maybe_route_to_synthesize(inp, prepared) + assert_refusal(caught.value, expected, 429) + else: + assert await handler._maybe_route_to_synthesize(inp, prepared) is None + finally: + assert widths[0] == 20 + assert len(widths) == (2 if wide else 1) + if wide: + assert widths[1] > widths[0] + + +@pytest.mark.parametrize("mode", ["refusal", "cancel-only", "success"]) +async def test_ask_pipeline_preserves_first_refusal_during_repeated_cleanup_cancel( + native, monkeypatch, mode +): + from server.memory.api.models_search import MemorySearchInput + from server.memory.app.ask import identifiers, pipeline + from server.memory.app.ask.result import MemoryAskResult + + native.site, native.failure = "fact", failure(503) + started, cleaning, release, settled = (asyncio.Event() for _ in range(4)) + tasks = set() + ordinary = MemoryAskResult( + answer=BODY, source="retrieval_only", memory_keys=["complete"], results=[] + ) + + async def knowledge(*args, **kwargs): + tasks.add(asyncio.current_task()) + started.set() + try: + await asyncio.Event().wait() + finally: + cleaning.set() + await release.wait() + settled.set() + + async def fill(run, **kwargs): + await started.wait() + if mode == "cancel-only": + await asyncio.Event().wait() + if mode == "refusal": + # Actual corrected identifier owner and real store SQL dispatch; the + # existing RecordingCH fixture asserts native attribution before raise. + await identifiers._inject_identifier_memories(run.search_input, ["109741"], []) + return False, False, False + + async def early(*args, **kwargs): + return ordinary, False + + monkeypatch.setattr(pipeline, "knowledge_passages", knowledge) + monkeypatch.setattr( + pipeline, "_ask_pool_size", lambda inp, *args, **kwargs: NS(search_input=inp) + ) + monkeypatch.setattr(pipeline, "_fill_pool", fill) + monkeypatch.setattr(pipeline, "_early_answer", early) + request = MemorySearchInput(org="acme", namespace="docs", query="What is known about 109741?") + caller = asyncio.create_task( + pipeline.ask( + request, llm=None, prepared_knowledge=PreparedKnowledge(None, ReadPlan("acme")) + ) + ) + tasks.add(caller) + waiter = asyncio.create_task(cleaning.wait()) + tasks.add(waiter) + try: + await asyncio.wait_for(started.wait(), 1) + if mode == "cancel-only": + caller.cancel() + done, _ = await asyncio.wait( + {caller, waiter}, timeout=1, return_when=asyncio.FIRST_COMPLETED + ) + assert done and cleaning.is_set(), "ask returned without starting owned knowledge cleanup" + assert not caller.done() and not settled.is_set() + if mode != "success": + caller.cancel() + await asyncio.sleep(0) + caller.cancel() + await asyncio.sleep(0) + assert not caller.done() and not settled.is_set() + release.set() + if mode == "refusal": + with pytest.raises(VectorCapacityError) as caught: + await asyncio.wait_for(asyncio.shield(caller), 1) + assert_refusal(caught.value, native.failure, 503) + elif mode == "cancel-only": + with pytest.raises(asyncio.CancelledError): + await asyncio.wait_for(asyncio.shield(caller), 1) + else: + assert await asyncio.wait_for(asyncio.shield(caller), 1) is ordinary + assert ordinary.answer == BODY and ordinary.memory_keys == ["complete"] + assert settled.is_set() + assert all(t.done() for t in tasks) + finally: + release.set() + await join_owned(tasks) + assert len(native.refused) == (1 if mode == "refusal" else 0) From 66f468ca94d594d778358ba33d3dadc7fcddc52f Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Wed, 18 Nov 2026 08:15:14 +0000 Subject: [PATCH 10/28] Keep capacity refusals through bisect and tracker owners A refused ranking read cannot become absent evidence. Collect duplicate-check and bisect children before returning the original failure, including through repeated cancellation. Signed-off-by: zlareb1 --- server/code/app/bisect.py | 24 +- server/code/app/bisect_fix.py | 7 +- server/code/app/bisect_fusion.py | 9 +- server/tracker/app/dupcheck.py | 18 +- .../unit/test_vector_bisect_owner_refusals.py | 433 ++++++++++++++++++ .../test_vector_duplicate_related_pr_owner.py | 233 ++++++++++ 6 files changed, 712 insertions(+), 12 deletions(-) create mode 100644 tests/unit/test_vector_bisect_owner_refusals.py create mode 100644 tests/unit/test_vector_duplicate_related_pr_owner.py diff --git a/server/code/app/bisect.py b/server/code/app/bisect.py index 5c80b41b3f..666dffc6b1 100644 --- a/server/code/app/bisect.py +++ b/server/code/app/bisect.py @@ -90,6 +90,7 @@ from server.code.domain.time_window import _iso, _parse_when from server.code.store.index import CodeRepo from server.core.errors import ValidationError +from server.core.shielded import to_its_end from server.settings import settings from server.tracker.app import links as lk from server.tracker.domain import bisect as bi @@ -702,6 +703,7 @@ async def _handle_bisect(input: Any, started: float) -> Any: dup_task = _start_dupcheck_fix(input, org, ns, frames) if dup_task is not None: owned.append(dup_task) + failed = False try: return await _run_bisect( input, @@ -718,12 +720,24 @@ async def _handle_bisect(input: Any, started: float) -> Any: dup_task=dup_task, owned=owned, ) + except BaseException: + failed = True + raise finally: - # The duplicate check and the fusion's window reads run as tasks beside the scan: an - # early failure or a cancelled call must not leave them reading. - for task in owned: - if not task.done(): - task.cancel() + # A failed or cancelled scan keeps its owned reads until retirement completes. + pending = [task for task in owned if not task.done()] + for task in pending: + task.cancel() + if owned and (failed or pending): + + async def stopped() -> None: + await asyncio.gather(*owned, return_exceptions=True) + + try: + await to_its_end(stopped(), name="bisect-read-cleanup") + except asyncio.CancelledError: + if not failed: + raise async def _run_bisect( diff --git a/server/code/app/bisect_fix.py b/server/code/app/bisect_fix.py index 04e2a8b17d..97b0ce2d3a 100644 --- a/server/code/app/bisect_fix.py +++ b/server/code/app/bisect_fix.py @@ -21,6 +21,7 @@ from server.code.app.spans import code_span from server.code.domain.index import blame_introduced as intro from server.code.store.index import CodeRepo +from server.core.errors import VectorCapacityError from server.settings import settings from server.tracker.domain import bisect as bi from server.tracker.domain import bisect_fusion as fu @@ -147,7 +148,7 @@ async def _dupcheck_fix(task: Any, started: float | None = None) -> dict[str, An """The matched issue's fix, or {} when there is none. The check started at the top of the call; this waits for what is left of its budget (`code_bisect_dupcheck_timeout_s` from `started`). Past it the - bisect answers with the fusion alone and says so (`status: timeout`); any + bisect answers with the fusion alone and says so (`status: timeout`); an ordinary error answers as it would without the check.""" if task is None: return {} @@ -159,7 +160,9 @@ async def _dupcheck_fix(task: Any, started: float | None = None) -> dict[str, An if _expired("dupcheck"): return {"status": "deadline", "budget_s": budget} return {"status": "timeout", "budget_s": budget} - except Exception as e: # noqa: BLE001 — fail open + except VectorCapacityError: + raise + except Exception as e: # noqa: BLE001 — ordinary duplicate-check errors keep the scan available logger.info("bisect: duplicate-check fix skipped: %s", e) return {} diff --git a/server/code/app/bisect_fusion.py b/server/code/app/bisect_fusion.py index e051cbc4ac..a5638ceb4f 100644 --- a/server/code/app/bisect_fusion.py +++ b/server/code/app/bisect_fusion.py @@ -2,8 +2,8 @@ The diff scan's candidates, fused with the window's vector, lexical, path, subtree and recency ranks, and the suspects the report names -(`server/tracker/domain/bisect_fusion.py` holds the pure parts). Every read -fails open: an error leaves the scan's candidates as they were. +(`server/tracker/domain/bisect_fusion.py` holds the pure parts). Ordinary read failures +leave the scan's candidates available; capacity refusals fail the complete read. """ from __future__ import annotations @@ -16,6 +16,7 @@ from typing import Any from server.core.background import create_deferred_task +from server.core.errors import VectorCapacityError from server.core.llm_free import reads_llm_free from server.llm import customer from server.settings import settings @@ -58,7 +59,9 @@ async def _vector_order( kinds=["pr"], numbers=numbers, ) - except Exception as e: # noqa: BLE001 — the vector leg is optional + except VectorCapacityError: + raise + except Exception as e: # noqa: BLE001 — ordinary vector failures leave other signals available logger.info("bisect fusion: vector read failed: %s", e) return [] return [int(h["number"]) for h in hits] diff --git a/server/tracker/app/dupcheck.py b/server/tracker/app/dupcheck.py index 372a02a793..31cbb93315 100644 --- a/server/tracker/app/dupcheck.py +++ b/server/tracker/app/dupcheck.py @@ -119,6 +119,7 @@ async def execute_duplicate_check(plan: DuplicateCheckPlan) -> TrackerDuplicateC wants_prs = bool(input.related_prs) and not plan.explicit pr_task = _start_pr_check(input.org, plan.code_namespace, title, body) if wants_prs else None deadline = dl.start_deadline() + failed = False try: inputs = [input.model_copy(update={"namespace": ns}) for ns in plan.targets] or [input] out = await _checked(inputs, wall, int(input.top_k)) @@ -132,10 +133,23 @@ async def execute_duplicate_check(plan: DuplicateCheckPlan) -> TrackerDuplicateC title, cache=plan.cache, ) + except BaseException: + failed = True + raise finally: dl.end_deadline(deadline) - if pr_task is not None and not pr_task.done(): - pr_task.cancel() + if pr_task is not None and (failed or not pr_task.done()): + if not pr_task.done(): + pr_task.cancel() + + async def stopped() -> None: + await asyncio.gather(pr_task, return_exceptions=True) + + try: + await to_its_end(stopped(), name="tracker.related-pr-cleanup") + except asyncio.CancelledError: + if not failed: + raise def _start_pr_check(org: str, code_ns: str, title: str, body: str) -> Any: diff --git a/tests/unit/test_vector_bisect_owner_refusals.py b/tests/unit/test_vector_bisect_owner_refusals.py new file mode 100644 index 0000000000..9a21f15a3c --- /dev/null +++ b/tests/unit/test_vector_bisect_owner_refusals.py @@ -0,0 +1,433 @@ +"""Bisect refuses a missing vector signal instead of ranking a smaller read.""" + +from __future__ import annotations + +import asyncio +from datetime import datetime +from types import SimpleNamespace + +import pytest + +from server.core.errors import VectorAdmissionError, VectorCapacityError, VectorOverloadedError +from server.core.vectors.admission import current_work +from server.settings import settings +from tests.unit.ch_fake import RecordingCH + +ORG, CODE, TRACKER = "acme", "code-example", "tracker-example" +WALL = datetime(2026, 9, 1) +TEXT = "Opaque symptom\nThe entire report body remains unchanged." +PATH = "src/Engine.cpp" +ROWS = [ + { + "pr_number": 1001, + "title": "Alpha", + "author": "first", + "closed_at": "2026-08-20 00:00:00", + "minor_version": "26.8", + "affected_file_paths": [], + "url": "https://example.test/pull/1001", + }, + { + "pr_number": 1002, + "title": "Beta", + "author": "second", + "closed_at": "2026-08-10 00:00:00", + "minor_version": "26.8", + "affected_file_paths": [], + "url": "https://example.test/pull/1002", + }, +] + + +def refused(status): + if status == 429: + return VectorAdmissionError("complete tracker read refused", reason="rows", retry_after_s=7) + assert status == 503 + return VectorOverloadedError("complete tracker read refused", reason="pool", retry_after_s=7) + + +def assert_refusal(actual, expected, status): + assert actual is expected and actual.status_code == status + assert actual.to_detail() == { + "code": "VECTOR_CAPACITY_REFUSED" if status == 429 else "OVERLOADED", + "message": str(expected), + "reason": "rows" if status == 429 else "pool", + "retry_after_s": 7, + } + assert actual.headers() == {"Retry-After": "7"} + + +async def join_owned(tasks): + for task in tasks: + if not task.done(): + task.cancel() + await asyncio.wait_for(asyncio.gather(*tasks, return_exceptions=True), 1) + + +def candidate(number, *, score, why=(), scan=False, named=False): + row = next(r for r in ROWS if r["pr_number"] == number) + out = { + "pr_number": number, + "title": row["title"], + "author": row["author"], + "merged_at": row["closed_at"], + "minor_version": "26.8", + "matched_paths": [PATH] if scan else [], + "matched_symbols": [], + "touches_affected": scan, + "kind": "change", + "has_test": None, + "url": row["url"], + "score": score, + "merge_commit_sha": "", + "merge_sha_source": "", + } + if why: + out["why"] = list(why) + if named: + out["named_cause_by"] = [{"pr_number": 9000, "role": "fix"}] + return out + + +@pytest.mark.parametrize( + "wide,status", [(False, 429), (True, 503)], ids=["initial-fusion", "wider-fusion"] +) +@pytest.mark.parametrize("mode", ["refusal", "ordinary", "complete"]) +async def test_consumed_bisect_fusion_keeps_native_refusals(monkeypatch, wide, status, mode): + from server.code.app import bisect_fusion + + expected = refused(status) if mode == "refusal" else RuntimeError("ordinary tracker outage") + native_calls, window_calls = [], [] + + def answer(sql, params): + assert "cosineDistance" in sql and "FROM loom.qa_item_embeddings FINAL" in sql + assert "org = {org:String}" in sql and "namespace = {ns:String}" in sql + assert "number IN {nums:Array(UInt64)}" in sql and "tombstoned_at IS NULL" in sql + assert params == { + "org": ORG, + "ns": TRACKER, + "q": [1.0, 0.0], + "dim": 2, + "limit": 200, + "kinds": ["pr"], + "nums": [1001, 1002], + } + work = current_work() + assert work is not None and (work.org, work.namespaces, work.weight) == (ORG, (TRACKER,), 1) + native_calls.append((sql, dict(params))) + if mode != "complete": + raise expected + return [ + { + "kind": "pr", + "number": n, + "source_id": str(n), + "component": "", + "is_open": 0, + "created_at": WALL, + "distance": distance, + } + for n, distance in [(1002, 0.1), (1001, 0.2)] + ] + + RecordingCH().on("FROM loom.qa_item_embeddings", answer).install( + monkeypatch, "server.tracker.store.embed" + ) + monkeypatch.setattr(settings, "vector_search_quantized_codes", False) + + async def embed(text): + assert text == TEXT + return [1.0, 0.0] + + class Source: + async def window_prs(self, org, namespace, versions, *, before, after): + assert (org, namespace, before, after) == (ORG, TRACKER, WALL, None) + assert versions == (["26.8", "26.9"] if wide else ["26.8"]) + window_calls.append(list(versions)) + return [dict(r) for r in ROWS] + + monkeypatch.setattr(bisect_fusion, "_embed", embed) + tasks = set() + + async def run(): + if wide: + pool, rows = await bisect_fusion.wide_extras( + Source(), + ORG, + TRACKER, + good="26.7", + bad="26.8", + master=False, + versions=["26.8"], + report_at=WALL, + in_window=set(), + text=TEXT, + paths=[], + ) + summary = None + else: + task = bisect_fusion.start_prefetch( + Source(), ORG, TRACKER, versions=["26.8"], before=WALL, report_at=WALL, text=TEXT + ) + assert task is not None + tasks.add(task) + pool, summary, rows = await bisect_fusion.fusion_stage( + task, [], report_at=WALL, before=WALL, text=TEXT, paths=[] + ) + return pool, summary, rows + + try: + if mode == "refusal": + with pytest.raises(VectorCapacityError) as caught: + await run() + assert_refusal(caught.value, expected, status) + else: + pool, summary, rows = await run() + order = [1002, 1001] if mode == "complete" else [1001, 1002] + why = ["vector", "recency"] if mode == "complete" else ["recency"] + assert [c.as_dict() for c in pool] == [ + candidate( + n, + score=0.0 if wide else (1.0 if i == 0 else 0.5), + why=(["wider_window", *why] if wide else why), + ) + for i, n in enumerate(order) + ] + assert rows == {r["pr_number"]: r for r in ROWS} + if not wide: + assert summary == { + "window_prs": 2, + "signals": { + "scan": 0, + "vector": 2 if mode == "complete" else 0, + "lexical": 0, + "exact_paths": 0, + "subtree": 0, + "recency": 2, + }, + "suspects": {"named": [], "mentioned": []}, + "margin": 0.016 if mode == "complete" else 0.048, + "weak": True, + } + finally: + await join_owned(tasks) + assert len(native_calls) == 1 and len(window_calls) == 1 + assert current_work() is None + + +@pytest.fixture +def public_bisect(monkeypatch): + from server.code.app import bisect + + calls, tasks = [], set() + + class Repo: + async def merge_commit_of(self, org, namespace, number): + assert (org, namespace, number) == (ORG, CODE, 9000) + calls.append(("blame-ledger", number)) + return "" + + class Source: + async def causes_named_by(self, org, namespace, number): + assert (org, namespace, number) == (ORG, TRACKER, 9000) + calls.append(("fix-named", number)) + return [1002] + + async def authorized(org, namespace, *, hides_answer): + assert (org, namespace, hides_answer) == (ORG, CODE, False) + return TRACKER + + async def named_or_prefetch(*args, **kwargs): + return [], None + + async def scan(source, org, namespace, tracker, plan, facts, **kwargs): + assert (org, namespace, tracker) == (ORG, CODE, TRACKER) + assert plan.paths == [PATH] and plan.versions == ["26.8", "26.9"] + calls.append(("scan", list(plan.paths))) + return ( + [{**r, "merged_at": r["closed_at"], "matched_paths": [PATH]} for r in ROWS], + True, + "tracker", + ) + + async def attach(repo, org, namespace, ranked): + assert (org, namespace) == (ORG, CODE) + calls.append(("attach", [c.pr_number for c in ranked])) + + monkeypatch.setattr(bisect, "require_code_intel", lambda: None) + monkeypatch.setattr(bisect, "resolve_code_identity", lambda org, namespace: (org, namespace)) + monkeypatch.setattr(bisect, "CodeRepo", Repo) + monkeypatch.setattr("server.tracker.app.binding.authorized_tracker_namespace", authorized) + monkeypatch.setattr("server.tracker.app.source.tracker_source", lambda org, namespace: Source()) + monkeypatch.setattr(bisect, "_candidate_rows", scan) + monkeypatch.setattr(bisect, "_named_or_prefetch", named_or_prefetch) + monkeypatch.setattr(bisect, "_attach_merge_shas", attach) + monkeypatch.setattr(bisect, "new_trace_run_id", lambda: "fixture-trace") + monkeypatch.setattr(bisect, "schedule_code_trace", lambda **kwargs: None) + return SimpleNamespace(module=bisect, calls=calls, tasks=tasks) + + +def bisect_input(): + from server.code.api.models_history import CodeBisectCandidatesInput + + # _CodeIdentity gives all five identity fields defaults. Spell them out here; + # the bounded model's remaining fields all have defaults and this shape is valid. + return CodeBisectCandidatesInput( + org=ORG, + namespace=CODE, + agent="reader", + user_id="-", + consumer="", + paths=[PATH], + text=TEXT, + good_version="26.7", + bad_version="26.8", + before="2026-09-01 00:00:00", + marks=False, + ) + + +def install_duplicate_child(monkeypatch, state, mode, before_raise=None): + from server.tracker.api.models import TrackerDuplicateCheckOutput + + expected = ( + refused(503) if mode == "refusal" else RuntimeError("ordinary duplicate-check outage") + ) + + async def check(inp): + state.tasks.add(asyncio.current_task()) + assert (inp.org, inp.namespace, inp.title, inp.body) == ( + ORG, + CODE, + "Opaque symptom", + "The entire report body remains unchanged.", + ) + assert inp.related_prs is False and inp.rerank is False + assert inp.frames == [] and inp.paths == [PATH] and inp.as_of == "2026-09-01 00:00:00" + state.calls.append(("duplicate", inp.namespace)) + if before_raise is not None: + await before_raise.wait() + if mode != "complete": + raise expected + return TrackerDuplicateCheckOutput( + verdict="duplicate_of", + items=[ + { + "kind": "issue", + "number": 8000, + "namespace": TRACKER, + "state": "closed", + "fixed_in": "26.9", + "fixed_in_source": "release_tag", + "links": [{"number": 9000, "state": "merged"}], + } + ], + ) + + monkeypatch.setattr("server.tracker.app.dupcheck.handle_tracker_duplicate_check", check) + return expected + + +@pytest.mark.parametrize("mode", ["refusal", "ordinary", "complete"]) +async def test_public_bisect_keeps_duplicate_fix_refusal(monkeypatch, public_bisect, mode): + state = public_bisect + expected = install_duplicate_child(monkeypatch, state, mode) + try: + if mode == "refusal": + with pytest.raises(VectorCapacityError) as caught: + await state.module.handle_code_bisect_candidates(bisect_input()) + assert_refusal(caught.value, expected, 503) + assert not any(c[0] == "attach" for c in state.calls) + else: + out = await state.module.handle_code_bisect_candidates(bisect_input()) + complete = mode == "complete" + order = [1002, 1001] if complete else [1001, 1002] + assert out.candidates == [ + candidate(n, score=1.0, scan=True, named=complete and n == 1002) for n in order + ] + assert (out.verdict, out.causes, out.epistemic, out.candidates_from) == ( + "candidates", + [], + "lower_bound", + "tracker", + ) + assert out.paths == [PATH] and out.frames == [] and out.patterns == [] + assert out.versions_searched == ["26.8", "26.9"] + assert out.good_version == "26.7" and out.bad_version == "26.8" + assert out.window == "clean_to_bad_plus_slack" and out.tracker_namespace == TRACKER + assert ( + out.fusion == {} + and out.trace_run_id == "fixture-trace" + and out.tracker_unavailable is False + ) + assert out.widened_by == (["dupcheck_fix"] if complete else []) + assert out.fix_blame == ( + {"fix_pr": 9000, "skipped": "fix_not_in_ledger", "named_by_fix": [1002]} + if complete + else {} + ) + assert out.dupcheck_fix == ( + { + "status": "matched", + "issue": 8000, + "fix_pr": 9000, + "match": "duplicate", + "namespace": TRACKER, + "note": "the report is a duplicate of #8000, fixed by #9000; the candidates below lead with what that fix's changed lines and cause statement point at", + } + if complete + else {} + ) + assert ("attach", order) in state.calls + finally: + await join_owned(state.tasks) + assert len([c for c in state.calls if c[0] == "duplicate"]) == 1 + + +async def test_public_bisect_joins_owned_prefetch_before_duplicate_refusal( + monkeypatch, public_bisect +): + state = public_bisect + started, cleaning, release, settled = (asyncio.Event() for _ in range(4)) + expected = install_duplicate_child(monkeypatch, state, "refusal", started) + + async def pending(): + started.set() + try: + await asyncio.Event().wait() + finally: + cleaning.set() + await release.wait() + settled.set() + + async def named_or_prefetch(*args, **kwargs): + task = asyncio.create_task(pending()) + state.tasks.add(task) + return [], task + + monkeypatch.setattr(state.module, "_named_or_prefetch", named_or_prefetch) + caller = asyncio.create_task(state.module.handle_code_bisect_candidates(bisect_input())) + waiter = asyncio.create_task(cleaning.wait()) + state.tasks.update((caller, waiter)) + try: + done, _ = await asyncio.wait( + {caller, waiter}, timeout=1, return_when=asyncio.FIRST_COMPLETED + ) + assert done and cleaning.is_set(), "duplicate refusal did not begin owned prefetch cleanup" + assert not caller.done() and not settled.is_set(), ( + "bisect returned before owned prefetch finished" + ) + caller.cancel() + await asyncio.sleep(0) + caller.cancel() + await asyncio.sleep(0) + assert not caller.done() and not settled.is_set() + release.set() + with pytest.raises(VectorCapacityError) as caught: + await asyncio.wait_for(asyncio.shield(caller), 1) + assert_refusal(caught.value, expected, 503) + assert settled.is_set() + assert all(t.done() for t in state.tasks) + finally: + release.set() + await join_owned(state.tasks) diff --git a/tests/unit/test_vector_duplicate_related_pr_owner.py b/tests/unit/test_vector_duplicate_related_pr_owner.py new file mode 100644 index 0000000000..b59829df46 --- /dev/null +++ b/tests/unit/test_vector_duplicate_related_pr_owner.py @@ -0,0 +1,233 @@ +"""A duplicate-check owner joins its optional PR reader before returning.""" + +from __future__ import annotations + +import asyncio +from types import SimpleNamespace + +import pytest + +from server.core.errors import VectorCapacityError +from server.core.vectors.admission import current_work +from server.operations.registry import OPERATIONS_BY_NAME +from server.settings import settings +from server.tracker.api.models import TrackerDuplicateCheckInput +from server.tracker.app import dupcheck +from server.tracker.domain.similar import Query +from server.tracker.store.embed import TrackerEmbedRepo +from tests.unit.test_vector_read_admission import Driver, NativeBudgetError, client + +TITLE = "stream reader retirement retains complete request scope" +BODY = "The complete draft body remains available.\nIts second line remains too." + + +@pytest.mark.parametrize("mode", ["refusal", "repeat-cancel", "cancel-only", "success"]) +async def test_duplicate_check_joins_related_pr_reader(monkeypatch, mode): + ch, _ = client(monkeypatch) + pr_started, checking, cleaning, release, retired = (asyncio.Event() for _ in range(5)) + owned = set() + seen = SimpleNamespace(refusal=None, detail=None, headers=None, candidate_calls=0, pr_calls=0) + native_error = NativeBudgetError(158) + + class NativeDriver(Driver): + async def query(self, sql, **kwargs): + self.started += 1 + self.settings.append(kwargs["settings"]) + work = current_work() + assert work is not None + assert (work.org, work.namespaces, work.weight) == ("one-org", ("tracker-example",), 1) + assert "FROM loom.qa_item_embeddings FINAL" in sql + assert "cosineDistance" in sql + assert "org = {org:String}" in sql and "namespace = {ns:String}" in sql + assert kwargs["parameters"] == { + "org": "one-org", + "ns": "tracker-example", + "q": [1.0, 0.0], + "dim": 2, + "limit": 50, + "kinds": ["issue"], + } + assert kwargs["settings"]["max_rows_to_read"] == settings.vector_scan_max_rows_to_read + assert kwargs["settings"]["read_overflow_mode"] == "throw" + if mode in {"refusal", "repeat-cancel"}: + raise native_error + return SimpleNamespace(result_rows=[], column_names=[], summary={}) + + driver = NativeDriver() + monkeypatch.setattr(ch, "_client", driver) + repo = TrackerEmbedRepo(ch) + + class Source: + async def duplicate_gold(self, org, namespace, *, limit): + assert (org, namespace, limit) == ("one-org", "tracker-example", 2000) + return [] + + async def candidate_reads(src, inp, *, wall, status): + seen.candidate_calls += 1 + assert isinstance(src, Source) + assert (inp.org, inp.namespace, inp.title, inp.body, wall) == ( + "one-org", + "tracker-example", + TITLE, + BODY, + None, + ) + await pr_started.wait() + checking.set() + if mode == "cancel-only": + await asyncio.Event().wait() + try: + rows = await repo.nearest( + inp.org, inp.namespace, vector=[1.0, 0.0], dim=2, kinds=["issue"] + ) + except VectorCapacityError as error: + seen.refusal = error + seen.detail, seen.headers = error.to_detail(), error.headers() + raise + assert rows == [] + status["vector"] = "empty" + return { + "q": Query(text=TITLE + "\n" + BODY, title=TITLE, terms=["stream", "retirement"]), + "item": None, + "ranked": [], + "record": {}, + "meta": {}, + "draft_paths": [], + "leg_heads": {}, + "fingerprint": [], + } + + async def no_fit(org, namespace): + assert (org, namespace) == ("one-org", "tracker-example") + return None + + async def related_pr(inp): + seen.pr_calls += 1 + assert (inp.org, inp.namespace, inp.title, inp.body) == ( + "one-org", + "code-example", + TITLE, + BODY, + ) + task = asyncio.current_task() + assert task is not None + owned.add(task) + pr_started.set() + if mode == "success": + retired.set() + return { + "verdict": "related", + "items": [ + { + "pr_number": 109741, + "title": TITLE, + "state": "open", + "relation": "related", + "score": 0.75, + } + ], + } + try: + await asyncio.Event().wait() + finally: + cleaning.set() + await release.wait() + retired.set() + + monkeypatch.setattr(dupcheck, "candidates", candidate_reads) + monkeypatch.setattr(dupcheck, "cached_fit", no_fit) + monkeypatch.setattr(dupcheck.tsource, "tracker_source", lambda org, namespace: Source()) + monkeypatch.setattr("server.code.app.pr_check.handle_code_pr_check", related_pr) + draft = TrackerDuplicateCheckInput( + org="one-org", namespace="code-example", title=TITLE, body=BODY, rerank=False + ) + assert draft.related_prs is True + plan = dupcheck.DuplicateCheckPlan(draft, ("tracker-example",), "code-example", False, False) + assert OPERATIONS_BY_NAME["tracker.duplicate_check"].name == "tracker.duplicate_check" + caller = asyncio.create_task(dupcheck.execute_duplicate_check(plan)) + owned.add(caller) + try: + await asyncio.wait_for(pr_started.wait(), 1) + if mode == "success": + out = await asyncio.wait_for(asyncio.shield(caller), 1) + assert out.verdict == "none" and out.epistemic == "uncalibrated" + assert out.items == [] and out.closest == [] and out.nearest == [] + assert out.legs == {"vector": "empty"} + assert out.query == { + "number": 0, + "known": False, + "terms": ["stream", "retirement"], + "tokens": 0, + "frames": [], + "frames_source": "", + "mentions": [], + "component": "", + "versions": [], + "error_code": 0, + "paths": [], + "before": "", + "record": "skipped", + } + assert out.related_prs == [ + { + "number": 109741, + "title": TITLE, + "state": "open", + "relation": "related", + "score": 0.75, + } + ] + assert ( + out.answer + == "no reported issue matches; a PR may already address this: #109741 (open)" + ) + else: + await asyncio.wait_for(checking.wait(), 1) + if mode == "cancel-only": + caller.cancel() + await asyncio.wait_for(cleaning.wait(), 1) + assert not caller.done(), ( + "duplicate_check returned while its related-PR reader was still retiring" + ) + assert not retired.is_set() + if mode in {"repeat-cancel", "cancel-only"}: + caller.cancel() + await asyncio.sleep(0) + caller.cancel() + await asyncio.sleep(0) + assert not caller.done() and not retired.is_set() + release.set() + if mode == "cancel-only": + with pytest.raises(asyncio.CancelledError): + await asyncio.wait_for(asyncio.shield(caller), 1) + else: + with pytest.raises(VectorCapacityError) as caught: + await asyncio.wait_for(asyncio.shield(caller), 1) + assert caught.value is seen.refusal + assert caught.value.__cause__ is native_error + assert caught.value.status_code == 429 + assert ( + caught.value.to_detail() + == seen.detail + == { + "code": "VECTOR_CAPACITY_REFUSED", + "message": "the complete vector read exceeded its native budget", + "reason": "rows", + "retry_after_s": 1, + } + ) + assert caught.value.headers() == seen.headers == {"Retry-After": "1"} + assert retired.is_set() and all(task.done() for task in owned) + finally: + release.set() + for task in owned: + if not task.done(): + task.cancel() + await asyncio.wait_for(asyncio.gather(*owned, return_exceptions=True), 1) + assert seen.candidate_calls == seen.pr_calls == 1 + assert driver.started == (0 if mode == "cancel-only" else 1) + assert driver.active == 0 + assert ch._inflight == {} and ch._retired == {} + if mode in {"refusal", "repeat-cancel"}: + assert isinstance(seen.refusal, VectorCapacityError) + assert seen.refusal.status_code == 429 From a1beb614456e509e0419fb9ca3bc935e51363ac5 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Wed, 18 Nov 2026 08:32:39 +0000 Subject: [PATCH 11/28] Reconnect metadata stays scalar and bisect retains discarded reads Register speculative reads before the graph await so the public owner joins their cleanup. Keep advisory connection metadata outside the interrupted statement charge, and preserve ask failures in the existing cancellation helper. Signed-off-by: zlareb1 --- server/code/app/bisect.py | 8 +- server/core/ch.py | 8 +- server/memory/app/ask/pipeline.py | 8 +- server/memory/app/ask/source_context.py | 7 +- .../unit/test_bisect_named_prefetch_owner.py | 352 ++++++++++++++++++ .../unit/test_vector_bisect_owner_refusals.py | 1 + tests/unit/test_vector_reconnect_metadata.py | 248 ++++++++++++ 7 files changed, 617 insertions(+), 15 deletions(-) create mode 100644 tests/unit/test_bisect_named_prefetch_owner.py create mode 100644 tests/unit/test_vector_reconnect_metadata.py diff --git a/server/code/app/bisect.py b/server/code/app/bisect.py index 666dffc6b1..c0439c9fe9 100644 --- a/server/code/app/bisect.py +++ b/server/code/app/bisect.py @@ -622,6 +622,7 @@ async def _named_or_prefetch( introduced: str, before: Any, report_text: str, + owned: list[asyncio.Future[Any]], ) -> tuple[list[bi.Candidate], Any]: """The graph's named cause, else the fusion's window reads started early. @@ -641,13 +642,15 @@ async def _named_or_prefetch( text=report_text, ) ) + if prefetch is not None: + owned.append(prefetch) named: list[bi.Candidate] = [] if source is not None and facts.qa_ok and facts.caused and not introduced and not input.rescan: try: named = await _graph_named(source, org, tracker_ns, facts.caused) except BaseException: if prefetch is not None: - prefetch.cancel() # the caller never receives it to cancel + prefetch.cancel() # the outer owner retains it through cleanup raise if named and prefetch is not None: prefetch.cancel() @@ -811,9 +814,8 @@ async def _run_bisect( introduced=introduced, before=before, report_text=report_text, + owned=owned, ) - if prefetch is not None: - owned.append(prefetch) fix_pr, widening = await _resolve_fix(input, facts, widening, source, org, tracker_ns) plan = _plan( input, resolved=resolved, widening=widening, frames=frames, patterns=patterns, facts=facts diff --git a/server/core/ch.py b/server/core/ch.py index f7bc90c229..23b786d819 100644 --- a/server/core/ch.py +++ b/server/core/ch.py @@ -690,8 +690,8 @@ async def _probe_vector_state(client: CHClient) -> None: """The vector read state (does every replica know the codes setting, which columns are coded, which tables carry patch parts) is probed by whichever process connects first, not only by the server's lifespan, so CLIs and - benches build the same reads as the server. Once per process; a reconnect - skips it. Unit tests (fake clients) replace this function.""" + benches build the same reads as the server. Successful probes are reused; + a reconnect retries unset advisory state. Unit tests replace this function.""" from server.core.vectors.codes_state import ensure_probed await ensure_probed(client) @@ -1005,7 +1005,9 @@ async def connect(self) -> None: await _turn_off_unsafe_condition_cache(client) self._client = client self._connection_config = configuration - await _probe_vector_state(self) + # Metadata stays scalar while the interrupted statement retains its charge. + with vector_work("", (), 0): + await _probe_vector_state(self) async def disconnect(self) -> None: self._ensure_state() diff --git a/server/memory/app/ask/pipeline.py b/server/memory/app/ask/pipeline.py index 96b4c2fa8d..6183b30ff3 100644 --- a/server/memory/app/ask/pipeline.py +++ b/server/memory/app/ask/pipeline.py @@ -582,10 +582,4 @@ async def ask( failed = True raise finally: - if not _knowledge_task.done(): - try: - await cancel_and_drain(_knowledge_task) - except asyncio.CancelledError: - # Join the child, then preserve an already propagating failure. - if not failed: - raise + await cancel_and_drain(_knowledge_task, preserve_failure=failed) diff --git a/server/memory/app/ask/source_context.py b/server/memory/app/ask/source_context.py index 0ff4d076cf..750b41fa66 100644 --- a/server/memory/app/ask/source_context.py +++ b/server/memory/app/ask/source_context.py @@ -57,8 +57,11 @@ async def prepare_knowledge( return PreparedKnowledge(request, plan) -async def cancel_and_drain(task: asyncio.Task[Any]) -> None: +async def cancel_and_drain(task: asyncio.Task[Any], *, preserve_failure: bool = False) -> None: """Join an owned child before another attempt or the request can finish.""" + if task.done(): + return + import anyio from server.core.ch import set_query_tag @@ -80,7 +83,7 @@ async def cancel_and_drain(task: asyncio.Task[Any]) -> None: break if not task.cancelled(): task.exception() - if interrupted is not None: + if interrupted is not None and not preserve_failure: raise interrupted diff --git a/tests/unit/test_bisect_named_prefetch_owner.py b/tests/unit/test_bisect_named_prefetch_owner.py new file mode 100644 index 0000000000..04100c7bb2 --- /dev/null +++ b/tests/unit/test_bisect_named_prefetch_owner.py @@ -0,0 +1,352 @@ +"""A named bisect owns speculative reads until they retire or are handed over.""" + +from __future__ import annotations + +import asyncio +from datetime import datetime +from types import SimpleNamespace + +import pytest + +from server.code.api.models_history import CodeBisectCandidatesInput +from server.code.app import bisect, bisect_fusion +from server.code.app.bisect_facts import IssueFacts +from server.core.errors import VectorAdmissionError +from server.core.vectors.admission import current_work +from server.settings import settings +from tests.unit.ch_fake import RecordingCH + +ORG, CODE, TRACKER = "acme", "code-example", "tracker-example" +PATH, TEXT = "src/Engine.cpp", "The complete reported symptom." +WALL = datetime(2026, 9, 1) +ROW = { + "pr_number": 1001, + "title": "Named engine change", + "author": "writer", + "merged_at": "2026-08-20 00:00:00", + "closed_at": "2026-08-20 00:00:00", + "minor_version": "26.8", + "affected_file_paths": [PATH], + "has_test": 1, + "url": "https://example.test/pull/1001", +} + + +def input_model(): + # _CodeIdentity and every remaining field have defaults. All identity fields + # are nevertheless explicit here; issue_number disables unrelated dupcheck work. + return CodeBisectCandidatesInput( + org=ORG, + namespace=CODE, + agent="reader", + user_id="-", + consumer="", + paths=[PATH], + text=TEXT, + issue_number=8000, + good_version="26.7", + bad_version="26.8", + before="2026-09-01 00:00:00", + marks=False, + ) + + +async def join_owned(tasks): + for task in tasks: + if not task.done(): + task.cancel() + await asyncio.wait_for(asyncio.gather(*tasks, return_exceptions=True), 1) + + +def assert_named_body(out): + assert out.candidates == [ + { + "pr_number": 1001, + "title": ROW["title"], + "author": ROW["author"], + "merged_at": ROW["merged_at"], + "minor_version": "26.8", + "matched_paths": [], + "matched_symbols": [], + "touches_affected": False, + "kind": "change", + "has_test": True, + "url": ROW["url"], + "score": 3.0, + "merge_commit_sha": "", + "merge_sha_source": "", + "named_by": "caused_by_edge", + } + ] + assert (out.verdict, out.causes, out.epistemic, out.candidates_from) == ( + "graph_named", + ["caused_by_edge"], + "lower_bound", + "none", + ) + assert out.named_by_graph == [1001] and out.paths == [PATH] + assert out.frames == [] and out.patterns == [] and out.resolved == [] + assert out.versions_searched == ["26.8", "26.9"] + assert out.good_version == "26.7" and out.bad_version == "26.8" + assert out.window == "clean_to_bad_plus_slack" + assert out.issue == {"number": 8000, "title": "Reported issue"} + assert out.tracker_namespace == TRACKER and out.tracker_unavailable is False + assert out.fusion == {} and out.dupcheck_fix == {} and out.fix_blame == {} + assert out.trace_run_id == "fixture-trace" and out.partial is False + + +@pytest.fixture +def named_owner(monkeypatch): + state = SimpleNamespace( + tasks=set(), + prefetches=[], + graph_calls=[], + window_calls=[], + attached=[], + started=asyncio.Event(), + cleaning=asyncio.Event(), + release=asyncio.Event(), + settled=asyncio.Event(), + graph_error=RuntimeError("graph item read failed"), + unused_refusal=VectorAdmissionError( + "unused speculative vector refused", reason="rows", retry_after_s=7 + ), + graph_mode="named", + caused=[1001], + scanned=asyncio.Event(), + immediate=False, + raise_during_cleanup=True, + ) + real_start = bisect.start_prefetch + + def start(*args, **kwargs): + task = real_start(*args, **kwargs) + assert task is not None + state.tasks.add(task) + state.prefetches.append(task) + return task + + class Source: + async def window_prs(self, org, namespace, versions, *, before, after): + assert (org, namespace, versions, before, after) == ( + ORG, + TRACKER, + ["26.8", "26.9"], + WALL, + None, + ) + state.tasks.add(asyncio.current_task()) + state.window_calls.append((org, namespace, list(versions))) + state.started.set() + if state.immediate: + return [dict(ROW)] + try: + await state.release.wait() + return [dict(ROW)] + finally: + state.cleaning.set() + await state.release.wait() + state.settled.set() + if state.raise_during_cleanup: + raise state.unused_refusal + + async def items_by_number(self, org, namespace, numbers): + assert (org, namespace, numbers) == (ORG, TRACKER, [1001]) + state.graph_calls.append(list(numbers)) + await state.started.wait() + if state.immediate: + # Observe completion without consuming the unused task's error. + done, _ = await asyncio.wait({state.prefetches[-1]}, timeout=1) + assert done + if state.graph_mode == "error": + raise state.graph_error + return {1001: dict(ROW)} + + state.source = Source() + + async def authorize(org, namespace, *, hides_answer): + assert (org, namespace, hides_answer) == (ORG, CODE, True) + return TRACKER + + async def issue(source, org, namespace, number, **kwargs): + assert source is state.source and (org, namespace, number) == (ORG, TRACKER, 8000) + return IssueFacts( + fact={"number": 8000, "title": "Reported issue"}, + bad="26.8", + good="26.7", + opened_at=WALL, + caused=list(state.caused), + qa_ok=True, + ), None + + async def scan(source, org, namespace, tracker, plan, facts, **kwargs): + assert source is state.source and (org, namespace, tracker) == (ORG, CODE, TRACKER) + assert kwargs["scan"] is (not bool(state.caused)) + state.scanned.set() + return [], True, "none" + + async def attach(repo, org, namespace, candidates): + assert (org, namespace) == (ORG, CODE) + state.attached.append([c.pr_number for c in candidates]) + + async def embed(text): + assert text == TEXT + return [1.0, 0.0] if state.immediate else [] + + monkeypatch.setattr(bisect, "require_code_intel", lambda: None) + monkeypatch.setattr(bisect, "resolve_code_identity", lambda org, namespace: (org, namespace)) + monkeypatch.setattr(bisect, "CodeRepo", object) + monkeypatch.setattr("server.tracker.app.binding.authorized_tracker_namespace", authorize) + monkeypatch.setattr( + "server.tracker.app.source.tracker_source", lambda org, namespace: state.source + ) + monkeypatch.setattr(bisect, "_issue_stage", issue) + monkeypatch.setattr(bisect, "_candidate_rows", scan) + monkeypatch.setattr(bisect, "_attach_merge_shas", attach) + monkeypatch.setattr(bisect, "schedule_code_trace", lambda **kwargs: None) + monkeypatch.setattr(bisect, "new_trace_run_id", lambda: "fixture-trace") + monkeypatch.setattr(bisect, "start_prefetch", start) + monkeypatch.setattr(bisect_fusion, "_embed", embed) + return state + + +@pytest.mark.parametrize("outcome", ["error", "named"]) +@pytest.mark.parametrize("repeat_cancel", [False, True], ids=["held-cleanup", "repeated-cancel"]) +async def test_public_bisect_joins_prefetch_discarded_by_real_named_helper( + named_owner, outcome, repeat_cancel +): + state = named_owner + state.graph_mode = outcome + caller = asyncio.create_task(bisect.handle_code_bisect_candidates(input_model())) + waiter = asyncio.create_task(state.cleaning.wait()) + state.tasks.update((caller, waiter)) + try: + done, _ = await asyncio.wait( + {caller, waiter}, timeout=1, return_when=asyncio.FIRST_COMPLETED + ) + assert done and state.started.is_set() + assert not caller.done(), ( + "public bisect returned before the named helper's discarded prefetch retired" + ) + assert state.cleaning.is_set() and not state.settled.is_set() + if repeat_cancel: + caller.cancel() + await asyncio.sleep(0) + caller.cancel() + await asyncio.sleep(0) + assert not caller.done() and not state.settled.is_set() + state.release.set() + if outcome == "error": + with pytest.raises(RuntimeError) as caught: + await asyncio.wait_for(asyncio.shield(caller), 1) + assert caught.value is state.graph_error + assert state.attached == [] + elif repeat_cancel: + # There was no earlier failure: a cancelled caller still receives + # cancellation, after the unused child has finished its cleanup. + with pytest.raises(asyncio.CancelledError): + await asyncio.wait_for(asyncio.shield(caller), 1) + else: + assert_named_body(await asyncio.wait_for(asyncio.shield(caller), 1)) + assert state.attached == [[1001]] + assert state.settled.is_set() + assert all(task.done() for task in state.tasks) + finally: + state.release.set() + await join_owned(state.tasks) + assert state.graph_calls == [[1001]] + assert len(state.window_calls) == 1 and len(state.prefetches) == 1 + + +async def test_public_bisect_accepts_ordinary_pending_prefetch_handover(named_owner): + state = named_owner + state.caused = [] + state.raise_during_cleanup = False + caller = asyncio.create_task(bisect.handle_code_bisect_candidates(input_model())) + state.tasks.add(caller) + try: + await asyncio.wait_for(state.started.wait(), 1) + await asyncio.wait_for(state.scanned.wait(), 1) + assert len(state.prefetches) == 1 and not state.prefetches[0].done() + assert not caller.done() and state.graph_calls == [] + assert not state.cleaning.is_set() + state.release.set() + out = await asyncio.wait_for(asyncio.shield(caller), 1) + assert out.candidates == [ + { + "pr_number": 1001, + "title": ROW["title"], + "author": ROW["author"], + "merged_at": ROW["merged_at"], + "minor_version": "26.8", + "matched_paths": [PATH], + "matched_symbols": [], + "touches_affected": True, + "kind": "change", + "has_test": None, + "url": ROW["url"], + "score": 1.0, + "merge_commit_sha": "", + "merge_sha_source": "", + "why": ["exact_paths", "subtree", "recency"], + } + ] + assert out.verdict == "candidates" and out.causes == [] and out.named_by_graph == [] + assert out.paths == [PATH] and out.versions_searched == ["26.8", "26.9"] + assert out.fusion == { + "window_prs": 1, + "signals": { + "scan": 0, + "vector": 0, + "lexical": 0, + "exact_paths": 1, + "subtree": 1, + "recency": 1, + }, + "suspects": {"named": [], "mentioned": []}, + "margin": 1.0, + "weak": False, + } + assert state.attached == [[1001]] and state.settled.is_set() + assert all(t.done() for t in state.tasks) + finally: + state.release.set() + await join_owned(state.tasks) + assert len(state.prefetches) == 1 and len(state.window_calls) == 1 + + +async def test_public_named_body_ignores_completed_unused_vector_refusal(named_owner, monkeypatch): + state = named_owner + state.immediate = True + native_calls = [] + + def answer(sql, params): + assert "FROM loom.qa_item_embeddings FINAL" in sql and "cosineDistance" in sql + assert params == { + "org": ORG, + "ns": TRACKER, + "q": [1.0, 0.0], + "dim": 2, + "limit": 200, + "kinds": ["pr"], + "nums": [1001], + } + work = current_work() + assert work is not None and (work.org, work.namespaces, work.weight) == (ORG, (TRACKER,), 1) + native_calls.append((sql, dict(params))) + raise state.unused_refusal + + RecordingCH().on("FROM loom.qa_item_embeddings", answer).install( + monkeypatch, "server.tracker.store.embed" + ) + monkeypatch.setattr(settings, "vector_search_quantized_codes", False) + try: + out = await bisect.handle_code_bisect_candidates(input_model()) + assert_named_body(out) + assert state.attached == [[1001]] + assert len(state.prefetches) == 1 and state.prefetches[0].done() + assert state.prefetches[0].exception() is state.unused_refusal + finally: + await join_owned(state.tasks) + assert len(native_calls) == 1 and state.graph_calls == [[1001]] + assert current_work() is None diff --git a/tests/unit/test_vector_bisect_owner_refusals.py b/tests/unit/test_vector_bisect_owner_refusals.py index 9a21f15a3c..a267e913ab 100644 --- a/tests/unit/test_vector_bisect_owner_refusals.py +++ b/tests/unit/test_vector_bisect_owner_refusals.py @@ -403,6 +403,7 @@ async def pending(): async def named_or_prefetch(*args, **kwargs): task = asyncio.create_task(pending()) state.tasks.add(task) + kwargs["owned"].append(task) return [], task monkeypatch.setattr(state.module, "_named_or_prefetch", named_or_prefetch) diff --git a/tests/unit/test_vector_reconnect_metadata.py b/tests/unit/test_vector_reconnect_metadata.py new file mode 100644 index 0000000000..9a7b8d6baa --- /dev/null +++ b/tests/unit/test_vector_reconnect_metadata.py @@ -0,0 +1,248 @@ +"""Reconnect metadata stays scalar while the statement retains its work charge.""" + +from __future__ import annotations + +import asyncio +from types import SimpleNamespace + +import pytest +from clickhouse_connect.driver.exceptions import OperationalError + +from server.core import ch as ch_mod +from server.core.schema.vector_codec_ddl import CODEC_SETTINGS_SQL +from server.core.vectors import admission, codes_state, quantized_setting +from server.settings import settings +from tests.unit.codec_states import dict_rows + +_REAL_PROBE = ch_mod._probe_vector_state +_REAL_CONDITION_PROBE = ch_mod._turn_off_unsafe_condition_cache +SQL = """SELECT memory_key, content_excerpt, +cosineDistance(embedding, {q:Array(Float32)}) AS distance, +cosineDistance(memory_key_embedding, {q:Array(Float32)}) AS key_distance +FROM loom.memory_embeddings FINAL +WHERE org = {org:String} AND namespace = {ns:String} +ORDER BY distance ASC LIMIT 2""" +PARAMS = {"org": "one-org", "ns": "private", "q": [1.0, 0.0]} +ROWS = [ + { + "memory_key": "one", + "content_excerpt": "The complete first passage.", + "distance": 0.1, + "key_distance": 0.2, + }, + { + "memory_key": "two", + "content_excerpt": "The complete second passage.\nIts next line remains.", + "distance": 0.3, + "key_distance": 0.4, + }, +] + + +def result(rows): + names = list(rows[0]) if rows else [] + return SimpleNamespace( + result_rows=[tuple(row[name] for name in names) for row in rows], + column_names=names, + summary={}, + ) + + +@pytest.mark.parametrize("boot_failed", [True, False], ids=["failed-advisory-boot", "healthy-boot"]) +@pytest.mark.parametrize("vector", [True, False], ids=["weight-two", "scalar-control"]) +async def test_reconnect_metadata_does_not_reenter_its_outer_org_charge( + monkeypatch, boot_failed, vector +): + for name, value in { + "ch_pool_maxsize": 2, + "ch_pool_wait_ms": 5000, + "vector_scan_max_concurrent_per_org": 2, + "vector_scan_max_waiters_per_org": 2, + }.items(): + monkeypatch.setattr(settings, name, value) + gate = admission._Admission() + monkeypatch.setattr(admission, "_admission", gate) + monkeypatch.setattr(ch_mod, "_probe_vector_state", _REAL_PROBE) + monkeypatch.setattr(ch_mod, "_turn_off_unsafe_condition_cache", _REAL_CONDITION_PROBE) + + async def schema_ready(*args, **kwargs): + return None + + monkeypatch.setattr(ch_mod, "guard_pending_schema", schema_ready) + # The real support probe returns false, so no read-triggered refresh can + # repair failed boot state before reconnect gets the chance to retry it. + quantized_setting.set_quantized_codes_supported(None) + replacement_created, metadata_started, release_metadata = (asyncio.Event() for _ in range(3)) + native_calls, metadata_calls, factory_work, version_calls = [], [], [], [] + transient = OperationalError("Network Error: Server disconnected") + expected_work = admission.VectorWork("one-org", ("private",), 2) if vector else None + outer_charge = 2 if vector else 0 + + def charged_snapshot(): + org = gate.orgs.get("one-org") + return gate.population, (org.held, org.waiting, len(org.queue)) if org is not None else None + + def assert_outer_charge(): + assert charged_snapshot() == (outer_charge, (2, 0, 0) if vector else None) + + class NativeDriver: + def __init__(self, generation): + self.generation, self.closed, self.vector_calls = generation, False, 0 + + async def query(self, sql, parameters=None, settings=None, **kwargs): + parameters, native_settings = dict(parameters or {}), dict(settings or {}) + work = admission.current_work() + if native_settings.get("log_comment") == "ch.condition_cache_probe": + assert "version" in sql and parameters == {} + assert native_settings == {"log_comment": "ch.condition_cache_probe"} + version_calls.append((self.generation, work, charged_snapshot())) + assert work == (expected_work if self.generation == 1 else None) + return result([{"version": "26.8.1.1"}]) + if sql == SQL: + self.vector_calls += 1 + native_calls.append((self.generation, sql, parameters, native_settings, work)) + assert work == expected_work + assert parameters == PARAMS + assert_outer_charge() + if vector: + assert ( + native_settings["max_rows_to_read"] + == settings_global.vector_scan_max_rows_to_read + ) + assert native_settings["read_overflow_mode"] == "throw" + else: + assert "max_rows_to_read" not in native_settings + if self.generation == 0: + raise transient + return result(ROWS) + kind = ( + "codec" + if sql == CODEC_SETTINGS_SQL + else "columns" + if sql == codes_state.CODED_COLUMNS_SQL + else "support" + ) + metadata_calls.append( + (self.generation, kind, work, native_settings, charged_snapshot()) + ) + assert work is None + assert "max_rows_to_read" not in native_settings + if kind == "support": + assert self.generation == 0 + assert "clusterAllReplicas" in sql + assert parameters == {"n": "vector_search_use_quantized_codes"} + return result([{"replicas": 1, "with_setting": 0}]) + if kind == "columns": + assert parameters == {"db": "loom"} + else: + assert parameters == {} + if self.generation == 0 and boot_failed: + raise RuntimeError("initial advisory metadata read unavailable") + if self.generation == 1: + assert_outer_charge() + metadata_started.set() + await release_metadata.wait() + if kind == "codec": + return result(dict_rows(enable_quantized_codec="on")) + assert kind == "columns" + return result( + [ + { + "table": "memory_embeddings", + "name": "embedding", + "compression_codec": "CODEC(Quantized('rabitq', 2, 0))", + "patch_parts": 0, + } + ] + ) + + async def close(self): + self.closed = True + + # Keep the fake driver's native settings argument separate from the real settings. + settings_global = settings + drivers = [] + + async def create_native(**kwargs): + assert kwargs["connector_limit"] == 2 + ch_mod.STREAM_SLOTS + assert kwargs["tz_source"] == "server" + driver = NativeDriver(len(drivers)) + drivers.append(driver) + factory_work.append(admission.current_work()) + if driver.generation == 1: + assert_outer_charge() + replacement_created.set() + return driver + + monkeypatch.setattr(ch_mod.clickhouse_connect, "get_async_client", create_native) + ch = ch_mod.CHClient() + caller = None + try: + await ch.connect() + assert quantized_setting.quantized_codes_state() is False + assert (codes_state._state.probed_at is None) is boot_failed + assert (codes_state._state.codec_available is None) is boot_failed + assert [(g, kind) for g, kind, *_ in metadata_calls] == [ + (0, "support"), + (0, "codec"), + (0, "columns"), + ] + + async def read(): + with admission.vector_work("one-org", "private", 2 if vector else 0): + rows = await ch.query_all(SQL, PARAMS) + assert admission.current_work() == expected_work + return rows + + caller = asyncio.create_task(read()) + await asyncio.wait_for(replacement_created.wait(), 1) + if boot_failed: + # The factory and first metadata query run in the same task without + # a yield. It can suspend at this driver gate only after admission. + assert metadata_started.is_set(), ( + "reconnect metadata queued behind its own vector charge", + charged_snapshot(), + ) + assert not caller.done() + assert_outer_charge() + release_metadata.set() + else: + assert not metadata_started.is_set() + assert await asyncio.wait_for(asyncio.shield(caller), 1) == ROWS + assert factory_work == [None, expected_work] + assert version_calls == [ + (0, None, (0, None)), + (1, expected_work, (outer_charge, (2, 0, 0) if vector else None)), + ] + assert [(g, kind) for g, kind, *_ in metadata_calls] == [ + (0, "support"), + (0, "codec"), + (0, "columns"), + ] + ([(1, "codec"), (1, "columns")] if boot_failed else []) + assert all(work is None for _, _, work, _, _ in metadata_calls) + assert ( + codes_state._state.probed_at is not None and codes_state._state.codec_available is True + ) + assert codes_state.stored_width("memory_embeddings", "embedding") == 2 + assert codes_state._state.refresh is None + assert len(native_calls) == 2 + assert native_calls[0][1:3] == native_calls[1][1:3] == (SQL, PARAMS) + assert [call[4] for call in native_calls] == [expected_work, expected_work] + assert ch.generation == 1 and drivers[0].closed + assert gate.population == 0 and gate.orgs == {} + assert ch._inflight == {} and ch._retired == {} + assert ch._slots is not None and ch._slots._value == 2 + finally: + release_metadata.set() + if caller is not None and not caller.done(): + caller.cancel() + if caller is not None: + await asyncio.wait_for(asyncio.gather(caller, return_exceptions=True), 1) + await ch.disconnect() + for driver in drivers: + if not driver.closed: + await driver.close() + quantized_setting.set_quantized_codes_supported(None) + assert all(driver.closed for driver in drivers) + assert gate.population == 0 and gate.orgs == {} + assert admission.current_work() is None From 6cf711bc51a70b966d92435c51dccf2a848a595c Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Wed, 18 Nov 2026 08:47:46 +0000 Subject: [PATCH 12/28] Cancelled waiters retain replacement org accounting Remove an empty org entry only while it still names the same accounting object. An older cancelled waiter can finish cleanup after a new request has already acquired replacement credits. Signed-off-by: zlareb1 --- server/core/vectors/admission.py | 2 +- .../unit/test_vector_org_generation_forget.py | 261 ++++++++++++++++++ 2 files changed, 262 insertions(+), 1 deletion(-) create mode 100644 tests/unit/test_vector_org_generation_forget.py diff --git a/server/core/vectors/admission.py b/server/core/vectors/admission.py index ca75fee656..933864e8ea 100644 --- a/server/core/vectors/admission.py +++ b/server/core/vectors/admission.py @@ -183,7 +183,7 @@ async def enter(self, work: VectorWork, wait_s: float) -> tuple[_Org, _Waiter | raise def _forget(self, name: str, org: _Org) -> None: - if not org.held and not org.queue: + if self.orgs.get(name) is org and not org.held and not org.queue: self.orgs.pop(name, None) def leave(self, work: VectorWork, org: _Org) -> None: diff --git a/tests/unit/test_vector_org_generation_forget.py b/tests/unit/test_vector_org_generation_forget.py new file mode 100644 index 0000000000..080ab3b938 --- /dev/null +++ b/tests/unit/test_vector_org_generation_forget.py @@ -0,0 +1,261 @@ +"""Cancelled waiters cannot forget a replacement org's live native reads.""" + +from __future__ import annotations + +import asyncio +from types import SimpleNamespace + +import pytest + +from server.core import ch as ch_module +from server.core.vectors import admission +from server.settings import settings + +ORG, NS = "same-org", "private" +BODY = "The complete first row.\nIts second line is retained." + + +def rows(label): + return [ + {"batch": label, "body": BODY, "score": 0.25}, + {"batch": label, "body": BODY + "\nAnother complete row.", "score": 0.75}, + ] + + +class NativeDriver: + def __init__(self): + self.release = {label: asyncio.Event() for label in ("B", "C", "D", "E")} + self.started = {label: asyncio.Event() for label in ("B", "C", "D", "E")} + self.order = [] + self.active = 0 + self.peak = 0 + self.work = [] + self.native_settings = [] + self.org_at_dispatch = {} + + async def query(self, sql, *, parameters, settings, **kwargs): + label = parameters["owner"] + work = admission.current_work() + assert work is not None and (work.org, work.namespaces) == (ORG, (NS,)) + assert parameters == {"owner": label, "org": ORG, "ns": NS, "q": [1.0, 0.0]} + assert sql.count("cosineDistance") == work.weight + assert "org = {org:String}" in sql and "namespace = {ns:String}" in sql + assert settings["read_overflow_mode"] == "throw" + assert settings["max_rows_to_read"] == 1_000_000 + assert 0 < settings["max_execution_time"] <= 30 + assert 0 < settings["max_memory_usage"] <= 4_000_000_000 + self.work.append(work) + self.native_settings.append(dict(settings)) + self.org_at_dispatch[label] = admission._admission.orgs[ORG] + self.order.append(label) + self.active += work.weight + self.peak = max(self.peak, self.active) + self.started[label].set() + try: + await self.release[label].wait() + result = rows(label) + return SimpleNamespace( + column_names=["batch", "body", "score"], + result_rows=[tuple(r.values()) for r in result], + summary={}, + ) + finally: + self.active -= work.weight + + +@pytest.fixture +def owned_client(monkeypatch): + gate = admission._Admission() + monkeypatch.setattr(admission, "_admission", gate) + monkeypatch.setattr(settings, "vector_scan_max_concurrent_per_org", 2) + monkeypatch.setattr(settings, "vector_scan_max_waiters_per_org", 8) + monkeypatch.setattr(settings, "vector_scan_max_rows_to_read", 1_000_000) + monkeypatch.setattr(settings, "ch_caller_shaped_max_execution_s", 30) + monkeypatch.setattr(settings, "ch_caller_shaped_max_memory_bytes", 4_000_000_000) + monkeypatch.setattr(settings, "ch_pool_maxsize", 8) + monkeypatch.setattr(settings, "ch_pool_wait_ms", 5000) + + async def schema_ready(*args, **kwargs): + return None + + monkeypatch.setattr(ch_module, "guard_pending_schema", schema_ready) + client, driver = ch_module.CHClient(), NativeDriver() + monkeypatch.setattr(client, "_client", driver) + + async def read(label, weight): + probes = " + ".join( + f"cosineDistance(embedding_{axis}, {{q:Array(Float32)}})" for axis in range(weight) + ) + sql = f"SELECT batch, body, {probes} AS score FROM fixture_vectors WHERE org = {{org:String}} AND namespace = {{ns:String}} AND batch = {{owner:String}}" + with admission.vector_work(ORG, NS, weight): + return await client.query_all( + sql, {"owner": label, "org": ORG, "ns": NS, "q": [1.0, 0.0]} + ) + + return SimpleNamespace(gate=gate, client=client, driver=driver, read=read) + + +async def cleanup(state, tasks): + for release in state.driver.release.values(): + release.set() + for task in tasks: + if not task.done(): + task.cancel() + await asyncio.wait_for(asyncio.gather(*tasks, return_exceptions=True), 1) + assert state.driver.active == 0 + assert state.gate.population == 0 and state.gate.orgs == {} + assert state.client._slots is not None and state.client._slots._value == 8 + assert state.client._inflight == {} and state.client._retired == {} + assert admission.current_work() is None + + +async def test_cancelled_waiter_cleanup_cannot_erase_replacement_org_under_real_client( + owned_client, +): + state = owned_client + gate, driver = state.gate, state.driver + full = admission.VectorWork(ORG, (NS,), 2) + old, _ = await gate.enter(full, 5) + left = False + tasks = [] + try: + cancelled = asyncio.create_task(state.read("B", 1)) + tasks.append(cancelled) + await asyncio.sleep(0) + assert old.held == 2 and old.waiting == 1 and len(old.queue) == 1 + assert gate.population == 3 and driver.order == [] + cancelled.cancel() + assert old.queue[0].ready.cancelled() and not cancelled.done() + # No suspension between cancellation and releasing the actual old lease. + # leave removes its cancelled head and forgets only this empty generation. + gate.leave(full, old) + left = True + assert ORG not in gate.orgs and gate.population == 1 + + async def later_read(): + # This task is scheduled after the cancelled waiter's cleanup, and a + # direct await of its completed task adds no scheduling turn. + try: + await cancelled + except asyncio.CancelledError: + pass + return await state.read("D", 2) + + later = asyncio.create_task(later_read()) + tasks.append(later) + + async def observe(): + try: + replacement = driver.org_at_dispatch["C"] + current = gate.orgs[ORG] + snapshot = { + "native_active_weight": driver.active, + "native_order": list(driver.order), + "same_replacement_generation": current is replacement, + "held": current.held, + "waiting": current.waiting, + "population": gate.population, + } + assert snapshot == { + "native_active_weight": 2, + "native_order": ["C"], + "same_replacement_generation": True, + "held": 2, + "waiting": 2, + "population": 4, + }, ( + "cancelled old waiter erased replacement ownership and exceeded the real C=2 native cap" + ) + assert cancelled.cancelled() and replacement is not old + finally: + driver.release["C"].set() + driver.release["D"].set() + + observer = asyncio.create_task(observe()) + tasks.append(observer) + # This call runs inline until the driver's Event: C is admitted and live + # before the event loop resumes B's already-scheduled cancellation cleanup. + first_body = await state.read("C", 2) + second_body = await later + assert first_body == rows("C") and second_body == rows("D") + assert gate.population == 0 and gate.orgs == {} + driver.release["E"].set() + assert await state.read("E", 2) == rows("E") + assert driver.order == ["C", "D", "E"] + assert [work.weight for work in driver.work] == [2, 2, 2] + await observer + assert driver.peak == 2 + finally: + if not left: + gate.leave(full, old) + await cleanup(state, tasks) + + +async def test_cancelled_waiter_without_replacement_keeps_its_current_owner(owned_client): + state = owned_client + gate, driver = state.gate, state.driver + full = admission.VectorWork(ORG, (NS,), 2) + old, _ = await gate.enter(full, 5) + left = False + tasks = [] + try: + cancelled = asyncio.create_task(state.read("B", 1)) + tasks.append(cancelled) + await asyncio.sleep(0) + assert old.waiting == 1 and gate.population == 3 + cancelled.cancel() + with pytest.raises(asyncio.CancelledError): + await cancelled + assert gate.orgs[ORG] is old and old.held == 2 + assert old.waiting == 0 and not old.queue and gate.population == 2 + assert driver.order == [] + gate.leave(full, old) + left = True + assert gate.population == 0 and gate.orgs == {} + driver.release["C"].set() + assert await state.read("C", 2) == rows("C") + assert driver.order == ["C"] and driver.peak == 2 + finally: + if not left: + gate.leave(full, old) + await cleanup(state, tasks) + + +async def test_ordinary_weighted_fifo_preserves_complete_native_bodies(owned_client): + state = owned_client + gate, driver = state.gate, state.driver + full = admission.VectorWork(ORG, (NS,), 2) + old, _ = await gate.enter(full, 5) + left = False + tasks = [] + try: + first = asyncio.create_task(state.read("B", 1)) + second = asyncio.create_task(state.read("C", 1)) + tasks.extend((first, second)) + await asyncio.sleep(0) + assert old.waiting == 2 and [w.weight for w in old.queue] == [1, 1] + assert gate.population == 4 and driver.order == [] + gate.leave(full, old) + left = True + later = asyncio.create_task(state.read("D", 2)) + tasks.append(later) + await asyncio.sleep(0) + assert driver.order == ["B", "C"] and driver.active == 2 + assert gate.orgs[ORG] is old and old.held == 2 and old.waiting == 2 + assert gate.population == 4 + driver.release["B"].set() + assert await first == rows("B") + assert driver.order == ["B", "C"] and driver.active == 1 + assert old.held == 1 and old.waiting == 2 and not later.done() + driver.release["C"].set() + assert await second == rows("C") + await asyncio.wait_for(driver.started["D"].wait(), 1) + assert gate.orgs[ORG] is old and old.held == 2 and old.waiting == 0 + driver.release["D"].set() + assert await later == rows("D") + assert driver.order == ["B", "C", "D"] and driver.peak == 2 + assert [work.weight for work in driver.work] == [1, 1, 2] + finally: + if not left: + gate.leave(full, old) + await cleanup(state, tasks) From f748c53514620c94fb00acb762187396852c5c2d Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Wed, 18 Nov 2026 09:04:00 +0000 Subject: [PATCH 13/28] Classify reconnect metadata as scalar work The dispatch census must account for the reconnect probe that clears inherited vector attribution. A focused check inspects the actual connect method while the complete source census remains in CI. Signed-off-by: zlareb1 --- tests/unit/test_vector_dispatch_contract.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/tests/unit/test_vector_dispatch_contract.py b/tests/unit/test_vector_dispatch_contract.py index 772123ef63..03ce7a0325 100644 --- a/tests/unit/test_vector_dispatch_contract.py +++ b/tests/unit/test_vector_dispatch_contract.py @@ -82,6 +82,7 @@ (), ), "server/core/ch.py:CHClient._query_attempts": (0, (("''", "()", "0"),), (), ()), + "server/core/ch.py:CHClient.connect": (0, (("''", "()", "0"),), (), ()), "server/core/ch.py:CHClient._raw_stream_query": (0, (("''", "()", "0"),), (), ()), "server/core/ch.py:CHClient.raw_stream": (0, (("''", "()", "0"),), (), ()), "server/identity/app/namespace_catalog.py:handle_namespace_search_categories": ( @@ -742,6 +743,25 @@ def _unclassified( return failures +def test_connect_metadata_marker_has_a_scalar_classification() -> None: + path = "server/core/ch.py" + root = Path(__file__).resolve().parents[2] + tree = ast.parse((root / path).read_text()) + client = next( + node for node in tree.body if isinstance(node, ast.ClassDef) and node.name == "CHClient" + ) + client.body = [ + node + for node in client.body + if isinstance(node, ast.AsyncFunctionDef) and node.name == "connect" + ] + assert len(client.body) == 1 + owner = path + ":CHClient.connect" + classified = {name: row for name, row in _CLASSIFIED.items() if name == owner} + failures = _unclassified({path: ast.unparse(client)}, classified, set()) + assert not failures, "\n".join(failures) + + def test_first_party_vector_dispatches_have_a_classification() -> None: """Whole source contract: run in CI, not the local small focused cohort.""" root = Path(__file__).resolve().parents[2] From 36b87bf3469acee96be84d03144758b2e664e911 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Wed, 18 Nov 2026 20:41:16 +0000 Subject: [PATCH 14/28] Keep vector admission on ranked mirror discovery The shared stream reader also serves scalar document reads. Scope vector work at its discovery caller through stream close, preserve the parser contract, and align the SDK and dispatch declarations with their intended public surface. Signed-off-by: zlareb1 --- docs/architecture/clickhouse.md | 1 - sdk/typescript/test/package/exports.test.ts | 1 + server/knowledge/app/source_search.py | 22 ++--- server/knowledge/store/source_candidates.py | 8 +- server/memory/app/ask/identifiers.py | 23 +++-- server/memory/store/lookups.py | 2 +- tests/unit/snapshots/sdk_public_api.json | 1 + .../unit/test_vector_admission_boundaries.py | 20 +++-- tests/unit/test_vector_dispatch_contract.py | 13 +-- tests/unit/test_vector_mirror_stream_scope.py | 88 +++++++++++++++++++ 10 files changed, 135 insertions(+), 44 deletions(-) create mode 100644 tests/unit/test_vector_mirror_stream_scope.py diff --git a/docs/architecture/clickhouse.md b/docs/architecture/clickhouse.md index 104a67b02b..35072e231b 100644 --- a/docs/architecture/clickhouse.md +++ b/docs/architecture/clickhouse.md @@ -1022,7 +1022,6 @@ rows. When any replica is older, every Loom session turns the cache off when it - [Troubleshooting](../operations/troubleshooting.md): too many parts, slow inserts. - [Performance tuning](../guides/performance-tuning.md): traffic classes, vector knobs. - ## Vector read admission Vector dispatches carry their target org, authorized namespaces and probe weight. diff --git a/sdk/typescript/test/package/exports.test.ts b/sdk/typescript/test/package/exports.test.ts index 0867701ab8..d199a0d607 100644 --- a/sdk/typescript/test/package/exports.test.ts +++ b/sdk/typescript/test/package/exports.test.ts @@ -62,6 +62,7 @@ const ROOT_EXPORTS = [ "TimeoutError", "Transport", "ValidationError", + "VectorCapacityRefused", "WrongCell", "accessToken", "backoffDelayMs", diff --git a/server/knowledge/app/source_search.py b/server/knowledge/app/source_search.py index ad8e689fe2..257ea2e183 100644 --- a/server/knowledge/app/source_search.py +++ b/server/knowledge/app/source_search.py @@ -9,6 +9,7 @@ from server.core.ch import sequential_reads, set_query_tag from server.core.diagnostics import private_work from server.core.errors import VectorCapacityError +from server.core.vectors.admission import vector_work from server.core.vectors.private_first import each from server.knowledge.app.search import HybridSearchQuery, _common_where from server.knowledge.app.source_access import Admitted, ReadPlan @@ -80,17 +81,18 @@ async def take(batch: list[dict[str, Any]]) -> None: sql, params = store.statement(where, params, leg=leg, access_keys=coarse) # Open and consume in this coroutine: acquiring all streams first can # deadlock when selected legs exceed the available stream slots. - async with store.candidates(sql, params) as stream: - batch = [] - async for row in stream: - batch.append(row) - if len(batch) == 64: + with vector_work(query.org, mirrors): + async with store.candidates(sql, params) as stream: + batch = [] + async for row in stream: + batch.append(row) + if len(batch) == 64: + await take(batch) + batch = [] + if len(out) >= limit: + break + if batch and len(out) < limit: await take(batch) - batch = [] - if len(out) >= limit: - break - if batch and len(out) < limit: - await take(batch) except VectorCapacityError: raise except Exception: # noqa: BLE001 — an unconditional discovery failure invalidates every mirror leg diff --git a/server/knowledge/store/source_candidates.py b/server/knowledge/store/source_candidates.py index f371221587..995ac6e08c 100644 --- a/server/knowledge/store/source_candidates.py +++ b/server/knowledge/store/source_candidates.py @@ -9,7 +9,6 @@ from server.core import bm25 from server.core.ch import CHClient, get_client, set_query_tag -from server.core.vectors.admission import vector_work from server.knowledge.domain.live_chunks import live_chunk_sql Leg = Literal["semantic", "lexical", "keys"] @@ -177,7 +176,6 @@ async def candidates( sql: str, params: dict[str, Any], *, ch: CHClient | None = None ) -> AsyncIterator[AsyncIterator[dict[str, Any]]]: set_query_tag("knowledge.mirror.candidates") - with vector_work(str(params["org"]), list(params["nss"])): - async with (ch or get_client()).raw_stream(sql, params) as body: - async with aclosing(_rows(body)) as rows: - yield rows + async with (ch or get_client()).raw_stream(sql, params) as body: + async with aclosing(_rows(body)) as rows: + yield rows diff --git a/server/memory/app/ask/identifiers.py b/server/memory/app/ask/identifiers.py index 7943cdb4ab..8fb0ba264b 100644 --- a/server/memory/app/ask/identifiers.py +++ b/server/memory/app/ask/identifiers.py @@ -534,6 +534,19 @@ async def _read_latest_word( return rows +def _note_identifier_injections(out: list[Any]) -> None: + """Record injection counts without failing the answer.""" + try: + from server.core import request_work as _rw + + _n_chunk_rows = sum( + 1 for h in out if str(getattr(h, "memory_key", "")).startswith("chunk:") + ) + _rw.note_injected(facts=len(out) - _n_chunk_rows, chunks=_n_chunk_rows) + except Exception: # noqa: BLE001 — a counter never fails an answer + pass + + async def _inject_identifier_memories( search_input: Any, idents: list[str], ranked: list[Any], *, case_insensitive: bool = False ) -> list[Any]: @@ -669,13 +682,5 @@ async def _inject_identifier_memories( continue hit._row_identity = row_identity(r) # the row read (none for a chunk), for the authority out.append(hit) - try: - from server.core import request_work as _rw - - _n_chunk_rows = sum( - 1 for h in out if str(getattr(h, "memory_key", "")).startswith("chunk:") - ) - _rw.note_injected(facts=len(out) - _n_chunk_rows, chunks=_n_chunk_rows) - except Exception: # noqa: BLE001 — a counter never fails an answer - pass + _note_identifier_injections(out) return out diff --git a/server/memory/store/lookups.py b/server/memory/store/lookups.py index 343bca397e..7535634305 100644 --- a/server/memory/store/lookups.py +++ b/server/memory/store/lookups.py @@ -269,7 +269,7 @@ async def find_chunks_by_identifier( FROM loom.memory_chunks FINAL WHERE {where_sql} AND ({_CHUNK_KEY}) IN ( - SELECT DISTINCT {_CHUNK_KEY} FROM loom.memory_chunks WHERE {where_sql} + SELECT DISTINCT {_CHUNK_KEY} FROM loom.memory_chunks WHERE {where_sql} ) ORDER BY {order_by} LIMIT {{lim:UInt32}} diff --git a/tests/unit/snapshots/sdk_public_api.json b/tests/unit/snapshots/sdk_public_api.json index 30811709bc..36dc51e37a 100644 --- a/tests/unit/snapshots/sdk_public_api.json +++ b/tests/unit/snapshots/sdk_public_api.json @@ -3874,6 +3874,7 @@ "SyncAgent", "TokenClaims", "ValidationError", + "VectorCapacityRefused", "WrongCell", "__version__", "decode_token_claims", diff --git a/tests/unit/test_vector_admission_boundaries.py b/tests/unit/test_vector_admission_boundaries.py index 20c4ce9a0a..02fae04543 100644 --- a/tests/unit/test_vector_admission_boundaries.py +++ b/tests/unit/test_vector_admission_boundaries.py @@ -10,6 +10,7 @@ from server.core import ch as ch_module from server.core import ch_schema_guard from server.core.schema import status +from server.core.vectors.admission import vector_work from server.knowledge.store import source_candidates from server.memory.app.search import ann_entity_boosts from server.settings import settings @@ -69,15 +70,16 @@ async def test_mirror_stream_clears_only_consumer_attribution_and_closes_in_its_ driver = StreamingDriver() driver.release.set() monkeypatch.setattr(ch, "_client", driver) - async with source_candidates.candidates( - "SELECT namespace, cosineDistance(embedding, {q:Array(Float32)}) AS distance FROM loom.chunks", - {"org": "one-org", "nss": ["private"], "q": [1.0, 0.0]}, - ch=ch, - ) as rows: - assert await anext(rows) == {"namespace": "private", "chunk_idx": 0} - assert driver.stream.opened and not driver.stream.closed - assert await ch.query_all("SELECT 1") == [] - assert "max_rows_to_read" not in driver.settings[-1] + with vector_work("one-org", ["private"]): + async with source_candidates.candidates( + "SELECT namespace, cosineDistance(embedding, {q:Array(Float32)}) AS distance FROM loom.chunks", + {"org": "one-org", "nss": ["private"], "q": [1.0, 0.0]}, + ch=ch, + ) as rows: + assert await anext(rows) == {"namespace": "private", "chunk_idx": 0} + assert driver.stream.opened and not driver.stream.closed + assert await ch.query_all("SELECT 1") == [] + assert "max_rows_to_read" not in driver.settings[-1] assert driver.stream.closed assert driver.stream_settings["max_rows_to_read"] == 1_000_000 assert driver.stream_settings["read_overflow_mode"] == "throw" diff --git a/tests/unit/test_vector_dispatch_contract.py b/tests/unit/test_vector_dispatch_contract.py index 03ce7a0325..54c6740c6b 100644 --- a/tests/unit/test_vector_dispatch_contract.py +++ b/tests/unit/test_vector_dispatch_contract.py @@ -27,6 +27,7 @@ "query_columns", "raw_bytes", "raw_stream", + "candidates", "_rows", "_chunk_query", "_call_retrying_empty_block", @@ -117,16 +118,10 @@ ), "server/knowledge/app/source_search.py:collect_leg": ( 0, - (), - (), + (("query.org", "mirrors", "1"),), + (("candidates", ("query.org", "mirrors", "1")),), (("server/knowledge/store/source_candidates.py:statement", 1),), ), - "server/knowledge/store/source_candidates.py:candidates": ( - 0, - (("str(params['org'])", "list(params['nss'])", "1"),), - (("_rows", ("str(params['org'])", "list(params['nss'])", "1")),), - (), - ), "server/knowledge/store/source_candidates.py:statement": (1, (), (), ()), "server/memory/app/conflicts.py:_foreign_pinned_rows": ( 0, @@ -214,7 +209,7 @@ (), (("server/memory/app/retrieval/fact_rank.py:_bm25_hybrid_rank_score", 1),), ), - "server/memory/app/retrieval/fact_sql.py:_fact_distance_expr": (3, (), (), ()), + "server/memory/app/retrieval/fact_sql.py:_fact_distance_expr": (2, (), (), ()), "server/memory/app/retrieval/fact_sql.py:_fact_leg_sql": ( 2, (), diff --git a/tests/unit/test_vector_mirror_stream_scope.py b/tests/unit/test_vector_mirror_stream_scope.py new file mode 100644 index 0000000000..980cf0b621 --- /dev/null +++ b/tests/unit/test_vector_mirror_stream_scope.py @@ -0,0 +1,88 @@ +"""Mirror vector discovery owns attribution; the shared parser also serves scalar reads.""" + +from __future__ import annotations + +import asyncio +from contextlib import asynccontextmanager +from types import SimpleNamespace + +import pytest + +from server.core.errors import VectorAdmissionError +from server.core.vectors.admission import VectorWork, current_work +from server.knowledge.app import source_search as app +from server.knowledge.app.search import HybridSearchQuery +from server.knowledge.app.source_access import Admitted + + +@pytest.mark.parametrize("outcome", ["success", "refusal", "cancellation"]) +async def test_vector_discovery_scope_covers_open_consume_and_close(monkeypatch, outcome): + mirror = "acl-" + "1" * 24 + sid = "11111111-1111-4111-8111-111111111111" + expected = VectorWork("acme", (mirror,), 1) + candidate = { + "namespace": mirror, + "source_uri": "gdrive:file#" + sid, + "chunk_idx": 0, + "doc_version": 1, + "revision": 0, + "distance": 0.25, + "lexical": 0.0, + "matched": [], + } + plan = SimpleNamespace( + org="acme", + discovery_sources={(mirror, sid): "docs"}, + discovery_reader=None, + failed=False, + unread=False, + ) + admitted = Admitted( + "docs", candidate, {"namespace": mirror, "source_uri": candidate["source_uri"]} + ) + phases = [] + refusal = VectorAdmissionError("discovery refused", reason="rows") + + async def check(batch, *, chunks): + assert chunks and batch == [candidate] + return [admitted] + + plan.check = check + monkeypatch.setattr(app.pg, "enabled", lambda: False) + + @asynccontextmanager + async def stream(sql, params): + assert "cosineDistance" in sql + assert params["org"] == "acme" and params["nss"] == [mirror] + phases.append(("open", current_work())) + + async def rows(): + phases.append(("consume", current_work())) + if outcome == "refusal": + raise refusal + if outcome == "cancellation": + raise asyncio.CancelledError + yield candidate + + try: + yield rows() + finally: + phases.append(("close", current_work())) + + monkeypatch.setattr(app.store, "candidates", stream) + query = HybridSearchQuery("acme", ("docs",), "fake", [1.0], 2) + assert current_work() is None + if outcome == "refusal": + with pytest.raises(VectorAdmissionError) as caught: + await app.collect_leg(plan, query, "semantic", limit=2) + assert caught.value is refusal + elif outcome == "cancellation": + with pytest.raises(asyncio.CancelledError): + await app.collect_leg(plan, query, "semantic", limit=2) + else: + result = await app.collect_leg(plan, query, "semantic", limit=2) + assert result == [admitted] + assert result[0].candidate == candidate + assert phases == [("open", expected), ("consume", expected), ("close", expected)] + assert not plan.failed and not plan.unread + assert current_work() is None From f26273b819ea624476515d923f656ed74e5ccc88 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Wed, 18 Nov 2026 20:58:24 +0000 Subject: [PATCH 15/28] Keep key-component refreshes outside vector admission Component-only key scoring reads existing pool keys without vectors. Preserve scalar availability under occupied vector credits while retaining admission and native limits for cosine-ranked reads. Signed-off-by: zlareb1 --- server/memory/app/search/key_match.py | 2 +- tests/unit/test_vector_dispatch_contract.py | 4 +- .../unit/test_vector_key_components_scope.py | 169 ++++++++++++++++++ 3 files changed, 172 insertions(+), 3 deletions(-) create mode 100644 tests/unit/test_vector_key_components_scope.py diff --git a/server/memory/app/search/key_match.py b/server/memory/app/search/key_match.py index ca4a0d0a59..161754d82b 100644 --- a/server/memory/app/search/key_match.py +++ b/server/memory/app/search/key_match.py @@ -337,7 +337,7 @@ async def _rows_for_ns(ns: str) -> tuple[str, list[dict[str, Any]], bool]: return ns, [], True sql = key_match_sql(" AND ".join(where)) try: - with vector_work(input.org, ns): + with vector_work(input.org, ns, 0 if component_only else 1): return ns, await _ch.query_all(sql, params=params), False except VectorCapacityError: raise diff --git a/tests/unit/test_vector_dispatch_contract.py b/tests/unit/test_vector_dispatch_contract.py index 54c6740c6b..92ed2e0528 100644 --- a/tests/unit/test_vector_dispatch_contract.py +++ b/tests/unit/test_vector_dispatch_contract.py @@ -290,8 +290,8 @@ ), "server/memory/app/search/key_match.py:_run_key_match_pool._rows_for_ns": ( 0, - (("input.org", "ns", "1"),), - (("query_all", None), ("query_all", ("input.org", "ns", "1"))), + (("input.org", "ns", "0 if component_only else 1"),), + (("query_all", None), ("query_all", ("input.org", "ns", "0 if component_only else 1"))), (("server/memory/app/search/sql.py:key_match_sql", 1),), ), "server/memory/app/search/sql.py:anchor_pool_sql": (2, (), (), ()), diff --git a/tests/unit/test_vector_key_components_scope.py b/tests/unit/test_vector_key_components_scope.py new file mode 100644 index 0000000000..32b3b2310c --- /dev/null +++ b/tests/unit/test_vector_key_components_scope.py @@ -0,0 +1,169 @@ +"""Key-only component scoring stays scalar when vector credits are occupied.""" + +from __future__ import annotations + +from copy import deepcopy +from types import SimpleNamespace + +import pytest + +from server.core import ch as ch_module +from server.core.errors import VectorAdmissionError +from server.core.vectors import admission +from server.memory.api.models_search import MemorySearchInput +from server.memory.app.search import key_match as app +from server.memory.app.search.sql import _key_match_where, key_match_component_sql, key_match_sql +from server.settings import settings +from tests.unit.test_vector_read_admission import client + + +@pytest.mark.parametrize("component_only", [True, False], ids=["key-components", "ranked-vector"]) +@pytest.mark.parametrize("busy", [False, True], ids=["free-credits", "occupied-credits"]) +async def test_key_match_owns_only_its_actual_vector_read(monkeypatch, component_only, busy): + gate = admission._Admission() + monkeypatch.setattr(admission, "_admission", gate) + monkeypatch.setattr(settings, "vector_scan_max_concurrent_per_org", 2) + monkeypatch.setattr(settings, "vector_scan_max_waiters_per_org", 4) + monkeypatch.setattr(settings, "memory_search_key_match_min_strength", 0.2) + monkeypatch.setattr(settings, "memory_search_per_key_cap", 3) + monkeypatch.setattr(app, "key_match_query_stems", lambda _query: ["task", "worktre"]) + monkeypatch.setattr(app, "key_match_gate", lambda **_kwargs: "pool_key") + ch, _unused_driver = client(monkeypatch, pool=2) + calls = [] + native_rows = [ + { + "memory_key": "task_worktree", + "strength": 0.9, + "agent": "a", + "user_id": "u", + "session_id": "s1", + }, + { + "memory_key": "task_path", + "strength": 0.5, + "agent": "a", + "user_id": "u", + "session_id": "s1", + }, + ] + + class Driver: + async def query(self, sql, *, parameters, settings, **_kwargs): + calls.append((sql, deepcopy(parameters), deepcopy(settings), admission.current_work())) + rows = [{"any_key": 1}] if "SELECT 1 AS any_key" in sql else deepcopy(native_rows) + names = list(rows[0]) + return SimpleNamespace( + column_names=names, + result_rows=[tuple(row[k] for k in names) for row in rows], + summary={}, + ) + + monkeypatch.setattr(ch, "_client", Driver()) + monkeypatch.setattr(ch_module, "get_client", lambda: ch) + hits = [ + SimpleNamespace( + namespace="private", + memory_key=row["memory_key"], + content_excerpt=f"Complete {index} body.\nIts next line stays.", + rank_score=score, + score_components={"rank_score_raw": score}, + source="fact", + session_id="s1", + _row_identity=("a", "u", "s1"), + ) + for index, (row, score) in enumerate(zip(native_rows, [0.8, 0.7], strict=True)) + ] + before = [deepcopy(vars(hit)) for hit in hits] + inp = MemorySearchInput( + org="one-org", + namespace="private", + agent="a", + user_id="u", + session_id="s1", + query="task worktree", + top_k=2, + key_match_mode="gated" if component_only else "always", + search_mode="semantic", + ) + ctx = SimpleNamespace( + input=inp, + all_hits=hits, + actual_work={}, + namespaces=["private"], + query_vec=[0.1, 0.2], + embedder=SimpleNamespace(model_name="fake"), + endpoint=None, + query_shape="", + ) + occupied = admission.VectorWork("one-org", ("private",), 2) + holder = (await gate.enter(occupied, 1))[0] if busy else None + refused = busy and not component_only + try: + with admission._without_admission_wait(): + if refused: + with pytest.raises(VectorAdmissionError) as caught: + await app._run_key_match_pool(ctx) + assert caught.value.code == "VECTOR_CAPACITY_REFUSED" + assert caught.value.status_code == 429 + assert caught.value.extra == {"reason": "org_queue", "retry_after_s": 1} + assert caught.value.headers() == {"Retry-After": "1"} + else: + await app._run_key_match_pool(ctx) + assert ctx.all_hits == hits and len(ctx.all_hits) == 2 + expected = deepcopy(before) + if not refused: + for hit, row in zip(expected, native_rows, strict=True): + hit["score_components"]["key_match"] = row["strength"] + assert ctx.actual_work["key_match_added"] == 0 + assert ctx.actual_work["key_match_precheck_skipped"] == 0 + assert [vars(hit) for hit in ctx.all_hits] == expected + assert [hit.rank_score for hit in ctx.all_hits] == [0.8, 0.7] + _assert_dispatched_queries(calls, ctx, component_only, refused) + assert gate.population == (2 if busy else 0) + assert ch._inflight == {} and ch._retired == {} + assert ch._slots is not None and ch._slots._value == 2 + assert admission.current_work() is None + finally: + if holder is not None: + gate.leave(occupied, holder) + assert gate.population == 0 and gate.orgs == {} + + +def _assert_dispatched_queries(calls, ctx, component_only, refused): + if component_only: + assert ctx.actual_work["key_match_gate"] == "pool_key" + assert ctx.actual_work["key_match_component_only"] is True + else: + assert calls[0][0].strip().endswith("LIMIT 1") + assert calls[0][1] == {"org": "one-org", "ns": "private", "stems": ["task", "worktre"]} + assert calls[0][3] is None and "max_rows_to_read" not in calls[0][2] + if not refused: + sql, params, native, work = calls[-1] + expected_params = { + "org": "one-org", + "ns": "private", + "stems": ["task", "worktre"], + "limit": 2 if component_only else 20, + "per_key": 3, + "model": "fake", + "min_strength": 0.2, + "agent": "a", + "rs_user_id": "u", + "rs_session_id": "s1", + } + if component_only: + expected_params["pool_keys"] = ["task_path", "task_worktree"] + else: + expected_params["qv"] = [0.1, 0.2] + assert params == expected_params + builder = key_match_component_sql if component_only else key_match_sql + assert sql == builder(" AND ".join(_key_match_where(ctx.input, dict(expected_params)))) + assert ("cosineDistance" in sql) is (not component_only) + assert work == ( + None if component_only else admission.VectorWork("one-org", ("private",), 1) + ) + assert ("max_rows_to_read" in native) is (not component_only) + if not component_only: + assert native["max_rows_to_read"] == settings.vector_scan_max_rows_to_read + assert native["read_overflow_mode"] == "throw" + assert len(calls) == (1 if component_only or refused else 2) From 783468e8222553819a03d7a77e87af886da8bdcb Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Wed, 18 Nov 2026 21:44:13 +0000 Subject: [PATCH 16/28] Compare ordered retrieval across mixed builds Use a fixed fake-provider corpus and native query-setting witnesses to check that vector admission preserves all three retrieval APIs. Signed-off-by: zlareb1 --- .github/workflows/ci.yml | 2 +- bench/retrieval_determinism.py | 117 +++++++++++ scripts/gates/mixed_version.py | 250 ++++++++++++++++++++++- tests/unit/test_retrieval_determinism.py | 164 +++++++++++++++ 4 files changed, 526 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5afeb625df..4c3c7a80ea 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1083,7 +1083,7 @@ jobs: # first parent of its merge commit, the target branch as the merge was made. The # target branch's tip can carry a newer migration by the time the run starts. - name: Mixed-version gate - run: python scripts/gates/mixed_version.py --require --base HEAD~1 + run: python scripts/gates/mixed_version.py --require --base HEAD~1 --ranking-neutral - name: Published build ordinary-event compatibility run: python scripts/gates/mixed_version.py --require --base 1cbaf0411114512957b96ec5ea7bbb8058828543 diff --git a/bench/retrieval_determinism.py b/bench/retrieval_determinism.py index 62dd683ad3..06fbd95613 100644 --- a/bench/retrieval_determinism.py +++ b/bench/retrieval_determinism.py @@ -28,11 +28,14 @@ import argparse import json +import math import os import statistics import sys +import time import urllib.parse import urllib.request +from collections.abc import Callable from dataclasses import dataclass, field from typing import Any @@ -101,6 +104,120 @@ def to_json(self) -> dict[str, Any]: # ---------------------------------------------------------------------------- +RANKING_OPS = ("memory.search", "memory.search_trace", "memory.ask") +_RANKING_SCORES = ("distance", "similarity", "rank_score") +_RANKING_IDENTITY = ( + "memory_key", + "memory_type", + "source", + "namespace", + "agent", + "user_id", + "session_id", + "memory_id", + "version", + "parent_memory_key", +) +RankingSnapshot = dict[str, dict[str, list[dict[str, Any]]]] + + +def _ranking_score(value: object) -> float: + assert isinstance(value, int | float) and not isinstance(value, bool), "missing numeric score" + assert math.isfinite(value), "nonfinite score" + return float(value) + + +def ranked_rows(payload: object, *, namespace: str, positive_key: str) -> list[dict[str, Any]]: + """Keep every returned identity and score, without sorting or truncating rows.""" + assert isinstance(payload, dict), "ranking response is not an object" + rows = payload.get("results") + assert isinstance(rows, list) and rows, "missing or empty ranking results" + out = [] + for row in rows: + assert isinstance(row, dict), "ranking result is not an object" + for name in ("memory_key", "memory_type", "source"): + assert isinstance(row.get(name), str) and row[name], f"missing {name}" + assert row.get("namespace", namespace) in ("", namespace), "wrong result namespace" + identity = {name: row[name] for name in _RANKING_IDENTITY if name in row} + out.append( + { + "identity": identity, + "scores": {name: _ranking_score(row.get(name)) for name in _RANKING_SCORES}, + } + ) + assert out[0]["identity"]["memory_key"] == positive_key, "positive anchor is not first" + return out + + +def capture_rankings( + call: Callable[[str, dict[str, Any]], dict[str, Any]], + requests: dict[str, dict[str, dict[str, Any]]], + positives: dict[str, str], + *, + elapsed: dict[str, dict[str, float]] | None = None, +) -> RankingSnapshot: + """Capture all requested public reads; request errors propagate as failures.""" + assert positives and set(requests) == set(RANKING_OPS), "missing ranking operations" + snapshot: RankingSnapshot = {} + for op in RANKING_OPS: + assert set(requests[op]) == set(positives), "missing ranking queries" + snapshot[op] = {} + for query, positive_key in positives.items(): + body = requests[op][query] + assert body.get("query") == query, "query does not match its request" + namespace = body.get("namespace") + assert isinstance(namespace, str) and namespace and positive_key + started = time.perf_counter() + payload = call(op, body) + duration = time.perf_counter() - started + if elapsed is not None: + elapsed.setdefault(op, {})[query] = duration + if op == "memory.ask": + assert payload.get("source") == "retrieval_only", "ask did not return retrieval" + assert payload.get("synth_calls") == 0, "ask made a synthesis call" + snapshot[op][query] = ranked_rows( + payload, namespace=namespace, positive_key=positive_key + ) + return snapshot + + +def assert_rankings_equal( + before: RankingSnapshot, after: RankingSnapshot, *, queries: list[str] +) -> None: + """Compare each API independently with fixed float tolerances and exact order.""" + assert queries and len(set(queries)) == len(queries), "missing or repeated queries" + assert set(before) == set(after) == set(RANKING_OPS), "missing ranking operations" + for op in RANKING_OPS: + assert set(before[op]) == set(after[op]) == set(queries), "missing ranking queries" + for query in queries: + left, right = before[op][query], after[op][query] + assert left and len(left) == len(right), f"{op}: changed result count" + for a, b in zip(left, right, strict=True): + assert a["identity"] == b["identity"], f"{op}: changed identity or order" + assert set(a["scores"]) == set(b["scores"]) == set(_RANKING_SCORES) + for name in _RANKING_SCORES: + x, y = _ranking_score(a["scores"][name]), _ranking_score(b["scores"][name]) + assert math.isclose(x, y, rel_tol=1e-7, abs_tol=1e-6), ( + f"{op}: changed {name}: {x!r} != {y!r}" + ) + + +def ranking_timings(elapsed: dict[str, dict[str, float]]) -> dict[str, dict[str, Any]]: + """Small-sample timings; nearest-rank p99 is the maximum of three queries.""" + assert set(elapsed) == set(RANKING_OPS), "missing timing operations" + out = {} + for op, calls in elapsed.items(): + values = sorted(_ranking_score(value) for value in calls.values()) + assert values and values[0] >= 0, "missing or negative elapsed time" + out[op] = { + "n": len(values), + "p50_s": statistics.median(values), + "p99_s": values[math.ceil(0.99 * len(values)) - 1], + "calls_s": calls, + } + return out + + def top_keys(search_output: dict[str, Any], k: int) -> list[str]: """Top-k memory keys from a memory.search response, in returned order.""" hits = search_output.get("results") or [] diff --git a/scripts/gates/mixed_version.py b/scripts/gates/mixed_version.py index 6a3ca7bd6a..9f05a39578 100644 --- a/scripts/gates/mixed_version.py +++ b/scripts/gates/mixed_version.py @@ -29,7 +29,7 @@ as in `dev/ci_start_clickhouse.sh`. The gate needs docker; without it, or `git`, it skips with exit 0, or fails with `--require` (CI). - python scripts/gates/mixed_version.py [--base REV] [--keep] [--require] + python scripts/gates/mixed_version.py [--base REV] [--keep] [--require] [--ranking-neutral] """ from __future__ import annotations @@ -49,6 +49,7 @@ import time import uuid from collections.abc import Iterator +from datetime import UTC, datetime from pathlib import Path from typing import Any @@ -58,6 +59,12 @@ if str(ROOT) not in sys.path: # run as a file: this checkout's server, not an installed one sys.path.insert(0, str(ROOT)) +from bench.retrieval_determinism import ( # noqa: E402 + RANKING_OPS, + assert_rankings_equal, + capture_rankings, + ranking_timings, +) from server.core.sql_literal import sql_string # noqa: E402 from server.insights.domain import analysis as boundary # noqa: E402 @@ -207,13 +214,16 @@ def base_tree(rev: str, into: Path) -> Path: class Server: """One Loom process of one build; `restart` keeps its port.""" - def __init__(self, name: str, tree: Path, ch_port: int, logs: Path) -> None: + def __init__( + self, name: str, tree: Path, ch_port: int, logs: Path, *, ranking_neutral: bool = False + ) -> None: self.name, self.tree, self.ch_port = name, tree, ch_port self.port = _free_port() self.url = f"http://127.0.0.1:{self.port}" self.log = logs / f"{name}.log" self.proc: subprocess.Popen[bytes] | None = None self.workers = "off" + self.ranking_neutral = ranking_neutral def env(self) -> dict[str, str]: env = {k: v for k, v in os.environ.items() if not k.startswith("LOOM_")} @@ -238,6 +248,10 @@ def env(self) -> dict[str, str]: LOOM_OTEL_ENABLED="false", LOOM_INSTANCE_LABEL=f"ci:mixed-version-{self.name}", ) + if self.ranking_neutral: + env.update( + LOOM_MEMORY_ASK_RECENCY_WEIGHT="0", LOOM_MEMORY_SEARCH_RECENCY_POOL_WEIGHT="0" + ) if self.workers != "off": env["LOOM_DEPLOYMENT"] = "1" # one shared lease, not one per instance label return env @@ -363,8 +377,10 @@ def get(self, build: str, key: str) -> dict[str, Any]: out = self.call(build, "memory.get", {**self.ident, "memory_key": key}) return dict(out.get("memory") or out) - def run_all(self) -> list[str]: + def run_all(self, *, ranking_sources: tuple[str, str] | None = None) -> list[str]: self.all_ready() + if ranking_sources is not None: + self.ranking_neutrality(*ranking_sources) ids = self.memories_read_across() self.absence_precondition_across() for a, b in (("N", "N-1"), ("N-1", "N")): @@ -378,6 +394,208 @@ def run_all(self) -> list[str]: self.one_leader() return self.failures + def ranking_neutrality(self, base: str, head: str) -> None: + """Compare the real base and head readers over one immutable fake-vector corpus.""" + corpus = { + "orchard": "Amber orchard fruit harvest", + "garden": "Green garden seed watering", + "harbor": "Quiet harbor fishing boats", + "river": "Swift river stone crossing", + "summit": "Snowy summit mountain trail", + "valley": "Sunny valley meadow flowers", + } + positives = {corpus[key]: key for key in ("orchard", "harbor", "summit")} + identity = {**self.ident, "namespace": "ranking"} + token = mint(self.org, "ranking") + for srv in self.builds.values(): + health = httpx.get(f"{srv.url}/v1/health", timeout=10) + assert health.status_code == 200 + assert health.json().get("fake_embedding") is True + assert health.json().get("fake_llm") is True + assert srv.ranking_neutral and srv.workers == "off" + for key, value in corpus.items(): + written = self.builds["N-1"].call( + token, + "memory.set", + { + **identity, + "memory_key": key, + "memory_type": "semantic", + "value": value, + }, + ) + assert written.get("memory_key") == key and written.get("status") == "active" + assert written.get("version", 0) > 0 and written.get("noop") is False + anchor = datetime.now(UTC) + common = { + **identity, + "top_k": len(corpus), + "namespace_scope": "this", + "search_mode": "semantic", + "personalization": "none", + "reinforcement_weight": 0, + "as_of": anchor.isoformat(), + "observation_date": anchor.date().isoformat(), + } + census = {**common, "query": next(iter(positives)), "auto_touch": False} + pending = dict(self.builds) + deadline = time.monotonic() + SETTLE_S + while pending: + for label, srv in list(pending.items()): + rows = srv.call(token, "memory.search_trace", census).get("results") + assert isinstance(rows, list), "missing fixture census" + keys = [row["memory_key"] for row in rows] + assert len(keys) == len(set(keys)) and set(keys) <= set(corpus) + if set(keys) == set(corpus): + pending.pop(label) + assert not pending or time.monotonic() < deadline, "incomplete ranking fixture census" + if pending: + time.sleep(1) + requests = { + op: { + query: { + **common, + "query": query, + **( + { + "allow_synthesis": False, + "reinforce": False, + "precedents": False, + "decompose": "off", + "world_knowledge": "forbid", + "personalize": False, + "knowledge": False, + "llm_cache": False, + "depth": "excerpt", + } + if op == "memory.ask" + else {"auto_touch": False, "recency_pool_weight": 0} + ), + } + for query in positives + } + for op in RANKING_OPS + } + snapshots = {} + elapsed: dict[str, dict[str, dict[str, float]]] = {} + native = {} + for label, srv in self.builds.items(): + since = sql(self.ch_port, "SELECT toString(now64(6))") + elapsed[label] = {} + snapshots[label] = capture_rankings( + lambda op, body, srv=srv: srv.call(token, op, body), + requests, + positives, + elapsed=elapsed[label], + ) + until = sql(self.ch_port, "SELECT toString(now64(6))") + native[label] = self.ranking_native_settings(since, until, required=label == "N") + assert_rankings_equal(snapshots["N-1"], snapshots["N"], queries=list(positives)) + for snapshot in snapshots.values(): + for queries in snapshot.values(): + for rows in queries.values(): + keys = [row["identity"]["memory_key"] for row in rows] + assert len(keys) == len(set(keys)) and set(keys) <= set(corpus) + + def digest(value: Any) -> str: + return hashlib.sha256(json.dumps(value, sort_keys=True).encode()).hexdigest() + + print( + "ranking-neutral " + + json.dumps( + { + "sources": {"base": base, "head": head}, + "admission_sources": { + label: { + "present": (srv.tree / "server/core/vectors/admission.py").is_file(), + "sha256": hashlib.sha256( + (srv.tree / "server/core/vectors/admission.py").read_bytes() + ).hexdigest() + if (srv.tree / "server/core/vectors/admission.py").is_file() + else None, + "catalog_sha256": hashlib.sha256( + (srv.tree / "server/settings/catalog.py").read_bytes() + ).hexdigest(), + } + for label, srv in self.builds.items() + }, + "corpus_sha256": digest(corpus), + "requests_sha256": digest(requests), + "identity": identity, + "profile": { + "embedding_model": "fake-ngram-1024", + "llm_provider": "fake", + "workers": "off", + "ask_recency_weight": 0, + "search_recency_pool_weight": 0, + }, + "native_settings": native, + "admission_credits": ( + "not measured; native settings witness attributed execution only" + ), + "timing_method": ( + "three queries per op/build; p99 nearest rank; no statistical guarantee" + ), + "timings": {label: ranking_timings(calls) for label, calls in elapsed.items()}, + "snapshots": snapshots, + }, + sort_keys=True, + ) + ) + self.check(True, "base and head preserve all three ordered retrieval results and scores") + + def ranking_native_settings( + self, since: str, until: str, *, required: bool + ) -> list[dict[str, Any]]: + """Audit native settings on the compared reads, not source presence alone.""" + names = ("max_rows_to_read", "max_memory_usage", "max_execution_time", "read_overflow_mode") + defaults = { + row["name"]: row["default"] + for row in map( + json.loads, + sql( + self.ch_port, + "SELECT name, default FROM system.settings WHERE name IN " + "('max_rows_to_read','max_memory_usage'," + "'max_execution_time','read_overflow_mode') " + "FORMAT JSONEachRow", + ).splitlines(), + ) + } + assert set(defaults) == set(names), "missing native setting defaults" + sql(self.ch_port, "SYSTEM FLUSH LOGS") + raw = sql( + self.ch_port, + "SELECT splitByChar(' ', log_comment)[1] AS operation, " + "Settings['max_rows_to_read'] AS max_rows_to_read, " + "Settings['max_memory_usage'] AS max_memory_usage, " + "Settings['max_execution_time'] AS max_execution_time, " + "Settings['read_overflow_mode'] AS read_overflow_mode, count() AS queries " + "FROM system.query_log WHERE type = 'QueryFinish' AND is_initial_query " + "AND event_time_microseconds BETWEEN {since:DateTime64(6)} AND {until:DateTime64(6)} " + "AND positionCaseInsensitive(query, 'cosineDistance(') > 0 " + "AND splitByChar(' ', log_comment)[1] IN " + "('memory.search','memory.search_trace','memory.ask') " + "GROUP BY operation, max_rows_to_read, max_memory_usage, max_execution_time, " + "read_overflow_mode ORDER BY operation, max_rows_to_read FORMAT JSONEachRow", + {"since": since, "until": until}, + ) + rows = [json.loads(line) for line in raw.splitlines()] + assert {row["operation"] for row in rows} == set(RANKING_OPS), ( + "missing native read witnesses" + ) + for row in rows: + row["limits"] = {name: row.pop(name) or defaults[name] for name in names} + row["queries"] = int(row["queries"]) + assert row["queries"] > 0 + if required: + limits = row["limits"] + assert float(limits["max_rows_to_read"]) == 1_000_000 + assert limits["read_overflow_mode"] == "throw" + assert 0 < float(limits["max_memory_usage"]) <= 4_000_000_000 + assert 0 < float(limits["max_execution_time"]) <= 30 + return [{"window": {"since": since, "until": until}, **row} for row in rows] + def all_ready(self) -> None: for label, srv in {**self.builds, "N+1": self.next}.items(): ok, failing = srv.ready() @@ -879,6 +1097,11 @@ def main(argv: list[str] | None = None) -> int: ap.add_argument( "--require", action="store_true", help="fail instead of skipping when it cannot run" ) + ap.add_argument( + "--ranking-neutral", + action="store_true", + help="compare retrieval on the current base and head", + ) args = ap.parse_args(argv) if args.port_base: _PORTS.extend(range(args.port_base, args.port_base + 4)) @@ -899,13 +1122,28 @@ def main(argv: list[str] | None = None) -> int: required=args.require, ) base = mb.stdout.strip() + ranking_sources = None + if args.ranking_neutral: + base_sha, head_sha = ( + subprocess.check_output( + ["git", "-C", str(ROOT), "rev-parse", ref + "^{commit}"], text=True + ).strip() + for ref in (base, "HEAD") + ) + ranking_sources = (base_sha, head_sha) work = Path(tempfile.mkdtemp(prefix="loom-mixed-")) failures: list[str] = [] servers: list[Server] = [] try: with clickhouse(args.keep) as ch_port: - old = Server("base", base_tree(base, work / "base"), ch_port, work) - new = Server("head", ROOT, ch_port, work) + old = Server( + "base", + base_tree(base, work / "base"), + ch_port, + work, + ranking_neutral=args.ranking_neutral, + ) + new = Server("head", ROOT, ch_port, work, ranking_neutral=args.ranking_neutral) nxt = Server("next", worktree_copy(work / "next"), ch_port, work) servers += [old, new, nxt] print(f"next carries {add_noop_migration(nxt.tree)}") @@ -922,7 +1160,7 @@ def main(argv: list[str] | None = None) -> int: print("base restarts on head's schema") old.restart() nxt.start() - failures = Scenario(old, new, nxt, ch_port).run_all() + failures = Scenario(old, new, nxt, ch_port).run_all(ranking_sources=ranking_sources) except Exception: for s in servers: print(f"--- {s.name} log tail ---\n{s.tail()}") diff --git a/tests/unit/test_retrieval_determinism.py b/tests/unit/test_retrieval_determinism.py index 1cb2585edb..3a8730250c 100644 --- a/tests/unit/test_retrieval_determinism.py +++ b/tests/unit/test_retrieval_determinism.py @@ -187,3 +187,167 @@ def row(namespace: str) -> SimpleNamespace: n1, n2 = row("n1"), row("n2") assert sorted([n2, n1], key=hit_sort_key) == [n1, n2] + + +def _three_api_snapshot(): + from bench.retrieval_determinism import RANKING_OPS, capture_rankings + + rows = [ + { + "memory_key": key, + "memory_type": "semantic", + "source": "fact", + "namespace": "ns", + "distance": distance, + "similarity": 1 - distance, + "rank_score": 1 - distance, + } + for key, distance in (("harbor", 0.0), ("orchard", 0.75)) + ] + requests = { + op: {"quiet harbor": {"query": "quiet harbor", "namespace": "ns"}} for op in RANKING_OPS + } + return capture_rankings( + lambda _op, _body: {"results": rows, "source": "retrieval_only", "synth_calls": 0}, + requests, + {"quiet harbor": "harbor"}, + ) + + +def test_three_api_snapshot_retains_order_and_all_scores_with_float_tolerance(): + from copy import deepcopy + + from bench.retrieval_determinism import RANKING_OPS, assert_rankings_equal + + before = _three_api_snapshot() + after = deepcopy(before) + for op in RANKING_OPS: + after[op]["quiet harbor"][0]["scores"]["rank_score"] += 1e-8 + assert [row["identity"]["memory_key"] for row in before[op]["quiet harbor"]] == [ + "harbor", + "orchard", + ] + assert set(before[op]["quiet harbor"][0]["scores"]) == { + "distance", + "similarity", + "rank_score", + } + assert_rankings_equal(before, after, queries=["quiet harbor"]) + + +@pytest.mark.parametrize( + "change", + ["order", "score", "identity", "query", "operation", "empty", "missing-score", "nonfinite"], +) +def test_three_api_comparison_refuses_changed_or_incomplete_results(change): + from copy import deepcopy + + from bench.retrieval_determinism import assert_rankings_equal + + before = _three_api_snapshot() + after = deepcopy(before) + rows = after["memory.search"]["quiet harbor"] + if change == "order": + rows.reverse() + elif change == "score": + rows[0]["scores"]["rank_score"] += 0.01 + elif change == "identity": + rows[0]["identity"]["memory_key"] = "another-key" + elif change == "query": + after["memory.search"].pop("quiet harbor") + elif change == "operation": + after.pop("memory.ask") + elif change == "empty": + rows.clear() + elif change == "missing-score": + rows[0]["scores"].pop("distance") + else: + rows[0]["scores"]["distance"] = float("nan") + with pytest.raises(AssertionError): + assert_rankings_equal(before, after, queries=["quiet harbor"]) + + +@pytest.mark.parametrize("invalid", [None, {}, {"results": []}, {"results": [None]}]) +def test_three_api_capture_refuses_missing_or_empty_results(invalid): + from bench.retrieval_determinism import ranked_rows + + with pytest.raises(AssertionError): + ranked_rows(invalid, namespace="ns", positive_key="harbor") + + +@pytest.mark.parametrize( + "field", ["memory_key", "memory_type", "source", "distance", "similarity", "rank_score"] +) +def test_three_api_capture_requires_each_identity_and_score_field(field): + from bench.retrieval_determinism import ranked_rows + + row = { + "memory_key": "harbor", + "memory_type": "semantic", + "source": "fact", + "distance": 0.0, + "similarity": 1.0, + "rank_score": 1.0, + } + row.pop(field) + with pytest.raises(AssertionError): + ranked_rows({"results": [row]}, namespace="ns", positive_key="harbor") + + +@pytest.mark.parametrize("score", [float("nan"), float("inf"), float("-inf"), "1.0", True]) +def test_three_api_capture_refuses_nonfinite_or_nonnumeric_scores(score): + from bench.retrieval_determinism import ranked_rows + + row = { + "memory_key": "harbor", + "memory_type": "semantic", + "source": "fact", + "distance": score, + "similarity": 1.0, + "rank_score": 1.0, + } + with pytest.raises(AssertionError): + ranked_rows({"results": [row]}, namespace="ns", positive_key="harbor") + + +def test_three_api_capture_does_not_omit_a_failed_request(): + from bench.retrieval_determinism import RANKING_OPS, capture_rankings + + requests = { + op: {"quiet harbor": {"query": "quiet harbor", "namespace": "ns"}} for op in RANKING_OPS + } + + def refused(_op, _body): + raise AssertionError("HTTP 429") + + with pytest.raises(AssertionError, match="HTTP 429"): + capture_rankings(refused, requests, {"quiet harbor": "harbor"}) + + +def test_three_api_capture_requires_its_positive_anchor(): + from bench.retrieval_determinism import ranked_rows + + row = { + "memory_key": "another-key", + "memory_type": "semantic", + "source": "fact", + "distance": 0.0, + "similarity": 1.0, + "rank_score": 1.0, + } + with pytest.raises(AssertionError, match="positive anchor"): + ranked_rows({"results": [row]}, namespace="ns", positive_key="harbor") + + +def test_three_api_timings_report_small_sample_median_maximum_and_each_call(): + from bench.retrieval_determinism import RANKING_OPS, ranking_timings + + calls = {op: {"first": 0.03, "second": 0.01, "third": 0.02} for op in RANKING_OPS} + report = ranking_timings(calls) + for op in RANKING_OPS: + assert report[op] == { + "n": 3, + "p50_s": 0.02, + "p99_s": 0.03, + "calls_s": calls[op], + } From 5d052ae6042c4d967af4cc34017c1972b57f08f0 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Wed, 18 Nov 2026 22:39:50 +0000 Subject: [PATCH 17/28] Exercise native vector budgets and paired API timings Keep real refusal, cancellation, retry and parser-retirement evidence in the existing hosted fixtures. Record every attempted call and first-read result, and regenerate the script reach declarations. Signed-off-by: zlareb1 --- bench/scale_load.py | 75 ++ scripts/gates/mixed_version.py | 411 +++++++++- server/operations/reach_manifest.py | 12 +- tests/integration/test_ch_fault_injection.py | 749 ++++++++++++++++-- .../integration/test_vector_native_budget.py | 153 +++- tests/unit/test_mixed_version_gate.py | 156 ++++ 6 files changed, 1488 insertions(+), 68 deletions(-) diff --git a/bench/scale_load.py b/bench/scale_load.py index 2d74bddb5d..9b49b307bb 100644 --- a/bench/scale_load.py +++ b/bench/scale_load.py @@ -415,6 +415,81 @@ def _summary(lat: list[float], errors: dict[str, int]) -> dict[str, Any]: } +def latency_observations(samples: list[dict[str, Any]]) -> dict[str, Any]: + """Keep every attempt beside the successful-call percentiles (small n is explicit).""" + assert samples, "no attempted calls" + for sample in samples: + ms = sample["elapsed_ms"] + assert isinstance(ms, (int, float)) and math.isfinite(ms) and ms >= 0 + errors = Counter(str(s["error"]) for s in samples if s.get("error")) + lat = [float(s["elapsed_ms"]) for s in samples if not s.get("error")] + return {**_summary(lat, dict(errors)), "attempted": len(samples), "samples": samples} + + +def committed_latency_rows( + op: str, result: dict[str, Any], entries: list[dict[str, Any]] +) -> dict[str, dict[str, Any]]: + """A normal-fixture refusal/merge is evidence, never an invented committed row.""" + if op == "memory.set": + assert len(entries) == 1 + assert result.get("noop") is False and result.get("status") == "active" + assert not result.get("duplicate_of") and not result.get("merged_into_key") + rows = [result] + else: + assert op == "memory.bulk_set" + assert _outcomes(op, result) == ["written"] * len(entries) + for key in ( + "merged", + "duplicates", + "conflicted", + "rejected_pinned", + "identity_duplicates", + "skipped_pinned", + "skipped_locked", + "skipped_blacklisted", + "retyped", + ): + assert not result.get(key), f"fixture write: {key}" + rows = result.get("written") + assert isinstance(rows, list) and len(rows) == len(entries) + expected = {e["memory_key"]: e["value"] for e in entries} + assert len(expected) == len(entries) + assert [r.get("memory_key") for r in rows] == [e["memory_key"] for e in entries] + out = {} + for row in rows: + key = row["memory_key"] + assert row.get("conflict_verdict") == "clear", f"{key}: not checked clear" + assert isinstance(row.get("version"), int) and row["version"] > 0 + assert row.get("memory_id"), f"{key}: no committed id" + out[key] = { + "value": expected[key], + "version": row["version"], + "memory_id": row["memory_id"], + } + return out + + +def assert_latency_current(row: dict[str, Any], expected: dict[str, Any]) -> None: + assert row.get("found") is True, "acknowledged row missing from point read" + for field in ("value", "version", "memory_id"): + assert row.get(field) == expected[field], f"point read differs: {field}" + assert row.get("status") == "active" + + +def assert_latency_visible(result: dict[str, Any], key: str, expected: dict[str, Any]) -> None: + rows = result.get("results") + assert isinstance(rows, list), "missing search results" + matches = [r for r in rows if r.get("memory_key") == key] + assert len(matches) == 1, "acknowledged row missing/duplicated on FIRST search" + assert matches[0].get("version") == expected["version"], "stale first search" + assert matches[0].get("status") == "active", "nonactive first search" + + +def assert_latency_worker(pid: int, parent_pid: int, ppid: int, pgid: int) -> None: + assert pid > 0 and parent_pid > 0 and pid != parent_pid + assert ppid == parent_pid and pgid == parent_pid, "health PID not an owned worker" + + def sample_keys(ch: Any, shape: str, n: int) -> list[dict[str, Any]]: """`n` existing heads to read, spread over the load (largest tenant for single).""" where = ( diff --git a/scripts/gates/mixed_version.py b/scripts/gates/mixed_version.py index 9f05a39578..0dc71fe61a 100644 --- a/scripts/gates/mixed_version.py +++ b/scripts/gates/mixed_version.py @@ -49,6 +49,7 @@ import time import uuid from collections.abc import Iterator +from concurrent.futures import ThreadPoolExecutor from datetime import UTC, datetime from pathlib import Path from typing import Any @@ -65,6 +66,15 @@ capture_rankings, ranking_timings, ) +from bench.scale_load import ( # noqa: E402 + _error_code, + _summary, + assert_latency_current, + assert_latency_visible, + assert_latency_worker, + committed_latency_rows, + latency_observations, +) from server.core.sql_literal import sql_string # noqa: E402 from server.insights.domain import analysis as boundary # noqa: E402 @@ -215,8 +225,17 @@ class Server: """One Loom process of one build; `restart` keeps its port.""" def __init__( - self, name: str, tree: Path, ch_port: int, logs: Path, *, ranking_neutral: bool = False + self, + name: str, + tree: Path, + ch_port: int, + logs: Path, + *, + ranking_neutral: bool = False, + web_workers: int = 1, ) -> None: + assert web_workers in (1, 2) + self.web_workers = web_workers self.name, self.tree, self.ch_port = name, tree, ch_port self.port = _free_port() self.url = f"http://127.0.0.1:{self.port}" @@ -248,6 +267,9 @@ def env(self) -> dict[str, str]: LOOM_OTEL_ENABLED="false", LOOM_INSTANCE_LABEL=f"ci:mixed-version-{self.name}", ) + if self.web_workers != 1: + env["LOOM_WEB_WORKERS"] = str(self.web_workers) + env["LOOM_RUNTIME_DIR"] = str(self.log.parent / (self.name + ".runtime")) if self.ranking_neutral: env.update( LOOM_MEMORY_ASK_RECENCY_WEIGHT="0", LOOM_MEMORY_SEARCH_RECENCY_POOL_WEIGHT="0" @@ -257,10 +279,13 @@ def env(self) -> dict[str, str]: return env def start(self) -> None: + if self.web_workers != 1: + (self.log.parent / (self.name + ".runtime")).mkdir(mode=0o700, exist_ok=True) out = self.log.open("ab") self.proc = subprocess.Popen( [sys.executable, "-m", "uvicorn", "server.main:app"] - + ["--host", "127.0.0.1", "--port", str(self.port)], + + ["--host", "127.0.0.1", "--port", str(self.port)] + + (["--workers", str(self.web_workers)] if self.web_workers != 1 else []), cwd=self.tree, env=self.env(), stdout=out, @@ -310,13 +335,34 @@ def restart(self) -> None: def tail(self, n: int = 6000) -> str: return self.log.read_text(errors="replace")[-n:] if self.log.exists() else "" - def call(self, token: str, op: str, body: dict[str, Any]) -> dict[str, Any]: - r = httpx.post( - f"{self.url}/v1/{op}", - json=body, - headers={"Authorization": f"Bearer {token}"}, - timeout=60, - ) + def call( + self, + token: str, + op: str, + body: dict[str, Any], + *, + observation: dict[str, Any] | None = None, + ) -> dict[str, Any]: + started = time.perf_counter() if observation is not None else 0.0 + try: + r = httpx.post( + f"{self.url}/v1/{op}", + json=body, + headers={"Authorization": f"Bearer {token}"}, + timeout=60, + ) + except httpx.HTTPError as exc: + if observation is not None: + observation.update( + elapsed_ms=(time.perf_counter() - started) * 1000, error=type(exc).__name__ + ) + raise + if observation is not None: + observation.update( + elapsed_ms=(time.perf_counter() - started) * 1000, + status=r.status_code, + error="" if r.status_code == 200 else f"{r.status_code}:{_error_code(r)}", + ) if r.status_code != 200: raise AssertionError(f"{self.name} {op}: {r.status_code} {r.text[:400]}") return dict(r.json()) @@ -355,7 +401,17 @@ def mint(org: str, ns: str) -> str: class Scenario: """The checks, each run in both directions between the two builds.""" - def __init__(self, old: Server, new: Server, nxt: Server, ch_port: int) -> None: + def __init__( + self, + old: Server, + new: Server, + nxt: Server, + ch_port: int, + *, + owned_servers: list[Server] | None = None, + owned_logs: Path | None = None, + ) -> None: + self.owned_servers, self.owned_logs = owned_servers, owned_logs self.builds = {"N-1": old, "N": new} self.new = new self.next = nxt @@ -381,6 +437,7 @@ def run_all(self, *, ranking_sources: tuple[str, str] | None = None) -> list[str self.all_ready() if ranking_sources is not None: self.ranking_neutrality(*ranking_sources) + self.latency_acceptance(*ranking_sources) ids = self.memories_read_across() self.absence_precondition_across() for a, b in (("N", "N-1"), ("N-1", "N")): @@ -394,6 +451,337 @@ def run_all(self, *, ranking_sources: tuple[str, str] | None = None) -> list[str self.one_leader() return self.failures + def latency_acceptance(self, base: str, head: str) -> None: + """Small CI timings, two two-worker groups per arm; no production latency claim.""" + from bench.scale_load import _outcomes + + assert self.owned_servers is not None and self.owned_logs is not None + policies = {} + + def value(key: str) -> str: + # Alphabetic codewords avoid token/card/contact shapes without policy overrides. + digest = hashlib.sha256(("latency-fixture/" + key).encode()).hexdigest() + letters = digest.translate(str.maketrans("0123456789abcdef", "abcdefghijklmnop")) + return " ".join(letters[i : i + 16] for i in range(0, 64, 16)) + + corpus = {f"seed/{i:04}": value(f"seed/{i:04}") for i in range(2048)} + light = {f"light/{i:02}": value(f"light/{i:02}") for i in range(16)} + for label, source in (("base", base), ("head", head)): + tree = self.builds["N-1" if label == "base" else "N"].tree + arm = [ + Server( + f"latency-{label}-{i}", + tree, + self.ch_port, + self.owned_logs, + ranking_neutral=True, + web_workers=2, + ) + for i in range(2) + ] + # Registration precedes every start, including a partially failed start. + self.owned_servers.extend(arm) + samples: list[dict[str, Any]] = [] + visibility: dict[str, list[float]] = { + f"{role}/{op}": [] + for role in ("main", "light") + for op in ("memory.set", "memory.bulk_set") + } + proofs = [] + proofs_after = [] + fixtures = [] + complete = False + try: + for srv in arm: + srv.start() + assert srv.ready()[0] + proofs.append(self.latency_workers(srv)) + assert len({w["pid"] for p in proofs for w in p["workers"]}) == 4 + for role, contents in (("main", corpus), ("light", light)): + org = f"{self.org}-{label}-{role}" + ident = { + "org": org, + "namespace": "latency", + "agent": "gate", + "user_id": "-", + "session_id": "-", + } + token = mint(org, "latency") + expected = {} + entries = [ + {"memory_key": k, "value": v, "memory_type": "semantic"} + for k, v in contents.items() + ] + for offset in range(0, len(entries), 8): + seed_started = time.monotonic() + batch = entries[offset : offset + 8] + result = arm[0].call(token, "memory.bulk_set", {**ident, "entries": batch}) + expected.update(committed_latency_rows("memory.bulk_set", result, batch)) + # Four requests/sec is below the ordinary namespace write rate. + time.sleep(max(0, 0.25 - (time.monotonic() - seed_started))) + fixture = (role, ident, token, expected) + fixtures.append(fixture) + for srv in arm: + self.latency_census(srv, ident, token, expected) + + def exercise( + fixture: tuple[Any, ...], + *, + owned_arm: list[Server] = arm, + arm_samples: list[dict[str, Any]] = samples, + arm_visibility: dict[str, list[float]] = visibility, + ) -> None: + role, ident, token, expected = fixture + writer, reader = owned_arm + + def call( + target: Server, op: str, body: dict[str, Any], phase: str, check: Any = None + ) -> dict[str, Any]: + sample: dict[str, Any] = { + "role": role, + "op": op, + "phase": phase, + "server": target.name, + } + sample["request_sha256"] = hashlib.sha256( + json.dumps(body, sort_keys=True).encode() + ).hexdigest() + arm_samples.append(sample) + try: + result = target.call(token, op, {**ident, **body}, observation=sample) + if op in ("memory.set", "memory.bulk_set"): + sample["outcomes"] = _outcomes(op, result) + committed_latency_rows( + op, result, [body] if op == "memory.set" else body["entries"] + ) + if check is not None: + check(result) + return result + except (AssertionError, httpx.HTTPError, ValueError, TypeError) as exc: + sample["error"] = sample.get("error") or type(exc).__name__ + raise + + def search(key: str, row: dict[str, Any], phase: str) -> None: + call( + reader, + "memory.search", + { + "query": row["value"], + "top_k": 5, + "namespace_scope": "this", + "search_mode": "semantic", + "auto_touch": False, + "personalization": "none", + "reinforcement_weight": 0, + "recency_pool_weight": 0, + }, + phase, + lambda found: assert_latency_visible(found, key, row), + ) + + for i in range(8 if role == "main" else 4): + key = next(iter(expected)) + call( + reader, + "memory.get", + {"memory_key": key}, + "measured", + lambda current, row=expected[key]: assert_latency_current(current, row), + ) + for op, count in (("memory.set", 1), ("memory.bulk_set", 2)): + entries = [ + { + "memory_key": f"{op}/{i}/{j}", + "value": value(f"{role}/{op}/{i}/{j}"), + "memory_type": "semantic", + } + for j in range(count) + ] + body = entries[0] if count == 1 else {"entries": entries} + result = call(writer, op, body, "measured") + ack = time.perf_counter() + committed = committed_latency_rows(op, result, entries) + expected.update(committed) + # No settle loop: the FIRST post-ack search must be current. + for new_key, row in committed.items(): + search(new_key, row, "first_after_ack") + arm_visibility[f"{role}/{op}"].append( + (time.perf_counter() - ack) * 1000 + ) + for new_key, row in committed.items(): + call( + reader, + "memory.get", + {"memory_key": new_key}, + "first_point_after_ack", + lambda current, row=row: assert_latency_current(current, row), + ) + search(key, expected[key], "measured") + + with ThreadPoolExecutor(max_workers=2) as pool: + futures = [pool.submit(exercise, fixture) for fixture in fixtures] + for future in futures: + future.result() + for _role, ident, token, expected in fixtures: + for srv in arm: + self.latency_census(srv, ident, token, expected) + proofs_after = [self.latency_workers(srv) for srv in arm] + assert {(w["pid"], w["start_ticks"]) for p in proofs for w in p["workers"]} == { + (w["pid"], w["start_ticks"]) for p in proofs_after for w in p["workers"] + }, "web worker restarted during measurement" + policies[label] = [w["ratelimit"] for p in proofs_after for w in p["workers"]] + assert all(p == policies[label][0] for p in policies[label]), ( + "unequal worker policy" + ) + if label == "head": + assert policies["base"] == policies["head"], "different arm rate/process shares" + complete = True + finally: + # Cleanup precedes formatting; even a malformed receipt cannot strand an arm. + try: + arm[0].stop() + finally: + arm[1].stop() + print( + "latency-acceptance " + + json.dumps( + { + "source": source, + "arm": label, + "authority": "clickhouse", + "complete": complete, + "providers": "fake-ngram-1024/fake", + "process_groups": proofs, + "process_groups_after": proofs_after, + "main_seed_rows": len(corpus), + "light_seed_rows": len(light), + "fixtures": [ + {"role": role, "identity": ident, "committed_head_count": len(rows)} + for role, ident, _token, rows in fixtures + ], + "corpus_sha256": hashlib.sha256( + json.dumps( + {"main": corpus, "light": light}, sort_keys=True + ).encode() + ).hexdigest(), + "summary": { + role: { + op: latency_observations(selected) + for op in ( + "memory.get", + "memory.set", + "memory.bulk_set", + "memory.search", + ) + if ( + selected := [ + s + for s in samples + if s["role"] == role + and s["op"] == op + and s["phase"] == "measured" + ] + ) + } + for role in ("main", "light") + }, + "attempts": samples, + "normal_load_errors": { + "attempted": len(samples), + "errors": { + error: sum(s.get("error") == error for s in samples) + for error in sorted( + {s["error"] for s in samples if s.get("error")} + ) + }, + "lag_errors": { + code: sum(s.get("error") == "503:" + code for s in samples) + for code in ("PROJECTION_BEHIND", "PUBLICATION_PENDING") + }, + "lag_error_rate": ( + sum( + s.get("error") + in ("503:PROJECTION_BEHIND", "503:PUBLICATION_PENDING") + for s in samples + ) + / len(samples) + if samples + else None + ), + }, + "ack_to_first_search_ms": { + op: _summary(ms, {}) for op, ms in visibility.items() + }, + "visibility_definition": ( + "ack to all first serial searches; bulk=two rows; no retry" + ), + "small_n_only": True, + }, + sort_keys=True, + ), + flush=True, + ) + + def latency_workers(self, srv: Server) -> dict[str, Any]: + assert srv.proc is not None and srv.web_workers == 2 and srv.workers == "off" + seen = {} + deadline = time.monotonic() + SETTLE_S + while len(seen) < 2 and time.monotonic() < deadline: + response = httpx.get( + f"{srv.url}/v1/health", headers={"Connection": "close"}, timeout=10 + ) + assert response.status_code == 200 + health = response.json() + assert health.get("fake_embedding") is True and health.get("fake_llm") is True + assert health.get("storage_authority") == "clickhouse" + assert health.get("instance_label") == f"ci:mixed-version-{srv.name}" + pid = int(health["instance"].rsplit(":", 1)[1]) + stat = Path(f"/proc/{pid}/stat").read_text().rsplit(")", 1)[1].split() + assert_latency_worker(pid, srv.proc.pid, int(stat[1]), os.getpgid(pid)) + assert health.get("participant_id") + policy = health["ratelimit"] + assert policy.get("web_workers") == 2 and not policy.get("error") + seen[pid] = { + "pid": pid, + "start_ticks": stat[19], + "participant_id": health["participant_id"], + "instance": health["instance"], + "ratelimit": { + k: policy[k] + for k in ( + "backend", + "divisor", + "replica_count", + "replica_count_floor", + "web_workers", + "class_fractions", + ) + }, + } + if len(seen) < 2: + time.sleep(0.01) + assert len(seen) == 2, "did not observe two actual owned web workers" + return { + "parent_pid": srv.proc.pid, + "server": srv.name, + "workers": list(seen.values()), + "tree": str(srv.tree), + "command": srv.proc.args, + } + + def latency_census( + self, srv: Server, ident: dict[str, Any], token: str, expected: dict[str, Any] + ) -> None: + count = srv.call(token, "memory.list", {**ident, "count_only": True}) + assert count.get("total") == len(expected), "fixture has extra/missing heads" + keys = list(expected) + for offset in range(0, len(keys), 64): + batch = keys[offset : offset + 64] + got = srv.call(token, "memory.bulk_get", {**ident, "memory_keys": batch}) + assert not got.get("missing") and set(got.get("results", {})) == set(batch) + for key, row in got["results"].items(): + assert_latency_current(row, expected[key]) + def ranking_neutrality(self, base: str, head: str) -> None: """Compare the real base and head readers over one immutable fake-vector corpus.""" corpus = { @@ -1160,7 +1548,8 @@ def main(argv: list[str] | None = None) -> int: print("base restarts on head's schema") old.restart() nxt.start() - failures = Scenario(old, new, nxt, ch_port).run_all(ranking_sources=ranking_sources) + scenario = Scenario(old, new, nxt, ch_port, owned_servers=servers, owned_logs=work) + failures = scenario.run_all(ranking_sources=ranking_sources) except Exception: for s in servers: print(f"--- {s.name} log tail ---\n{s.tail()}") diff --git a/server/operations/reach_manifest.py b/server/operations/reach_manifest.py index d1eafc7121..55cf68f3e6 100644 --- a/server/operations/reach_manifest.py +++ b/server/operations/reach_manifest.py @@ -363,12 +363,16 @@ "memory.agents_list": ("dashboard",), "memory.aggregate": ("sdk_method",), "memory.apply": ("sdk_method",), - "memory.ask": ("sdk_method",), + "memory.ask": ( + "sdk_method", + "sdk_example", + ), "memory.assign_category": ("sdk_method",), "memory.auto_retirements": ("dashboard",), "memory.backlinks": ("dashboard",), "memory.bulk_get": ( "sdk_method", + "sdk_example", "dashboard", ), "memory.bulk_set": ( @@ -440,6 +444,7 @@ "memory.link": ("sdk_method",), "memory.list": ( "sdk_method", + "sdk_example", "plugin_hook", "dashboard", ), @@ -503,7 +508,10 @@ "plugin_hook", "dashboard", ), - "memory.search_trace": ("sdk_method",), + "memory.search_trace": ( + "sdk_method", + "sdk_example", + ), "memory.search_trace_log": ( "sdk_method", "dashboard", diff --git a/tests/integration/test_ch_fault_injection.py b/tests/integration/test_ch_fault_injection.py index eef673783c..57d989d1dd 100644 --- a/tests/integration/test_ch_fault_injection.py +++ b/tests/integration/test_ch_fault_injection.py @@ -23,10 +23,18 @@ from __future__ import annotations import asyncio +import ipaddress +import json import re +import ssl +import sys +import threading import time import uuid -from datetime import UTC, datetime +from concurrent.futures import ThreadPoolExecutor +from datetime import UTC, datetime, timedelta +from pathlib import Path +from types import SimpleNamespace from typing import Any import pytest @@ -43,84 +51,155 @@ class _FaultProxy: moment the first response byte arrives (once, then back to `pass`); `stall` relays the request and never relays the response.""" - def __init__(self, upstream_host: str, upstream_port: int) -> None: + def __init__( + self, upstream_host: str, upstream_port: int, *, tls: ssl.SSLContext | None = None + ) -> None: self.upstream = (upstream_host, upstream_port) self.mode = "pass" self.drops = 0 + self.drop_remaining = 1 self.stalled = 0 self.port = 0 self._server: asyncio.AbstractServer | None = None self._stalls: list[asyncio.Event] = [] + self._tls = tls + self.tls_connections = 0 + self._handlers: set[asyncio.Task[None]] = set() + self._writers: set[asyncio.StreamWriter] = set() + self._stopping = False + self._errors: list[BaseException] = [] + self.parser_headers = asyncio.Event() + self.parser_release = asyncio.Event() + self.parser_marker = b"" + self.parser_handler: asyncio.Task[None] | None = None async def start(self) -> None: - self._server = await asyncio.start_server(self._handle, "127.0.0.1", 0) + self._server = await asyncio.start_server(self._accept, "127.0.0.1", 0, ssl=self._tls) self.port = self._server.sockets[0].getsockname()[1] + def _accept(self, reader: asyncio.StreamReader, writer: asyncio.StreamWriter) -> None: + task = asyncio.create_task(self._handle(reader, writer)) + self._handlers.add(task) + task.add_done_callback(self._finished) + + def _finished(self, task: asyncio.Task[None]) -> None: + self._handlers.discard(task) + if task.cancelled(): + self._errors.append(asyncio.CancelledError("relay handler was cancelled")) + elif error := task.exception(): + self._errors.append(error) + async def stop(self) -> None: - for ev in self._stalls: - ev.set() + self._stopping = True if self._server is not None: self._server.close() await self._server.wait_closed() + for ev in self._stalls: + ev.set() + for writer in tuple(self._writers): + writer.close() + if self._handlers: + await asyncio.wait_for(asyncio.gather(*tuple(self._handlers)), timeout=5) + assert not self._handlers and not self._writers, "relay ownership did not drain" + assert not self._errors, self._errors async def _handle(self, cr: asyncio.StreamReader, cw: asyncio.StreamWriter) -> None: + owner = asyncio.current_task() + parser_target = False + writers, children = [cw], [] + self._writers.add(cw) + self.tls_connections += int(cw.get_extra_info("ssl_object") is not None) try: - ur, uw = await asyncio.open_connection(*self.upstream) - except OSError: - cw.close() - return - - async def c2u() -> None: + if self._stopping: + return try: - while True: - data = await cr.read(65536) - if not data: - break - uw.write(data) - await uw.drain() - except (ConnectionError, asyncio.CancelledError): - pass - finally: + ur, uw = await asyncio.wait_for(asyncio.open_connection(*self.upstream), 5) + except OSError: + return + writers.append(uw) + self._writers.add(uw) + if self._stopping: + return + + async def c2u() -> None: + nonlocal parser_target + tail = b"" try: - uw.write_eof() - except Exception: # noqa: BLE001 — peer may be gone + while data := await cr.read(65536): + if self.parser_marker: + scan = tail + data + parser_target = parser_target or self.parser_marker in scan + tail = scan[-len(self.parser_marker) :] + uw.write(data) + await uw.drain() + except (ConnectionError, asyncio.CancelledError): pass + finally: + try: + uw.write_eof() + except Exception: # noqa: BLE001 — peer may be gone + pass - async def u2c() -> None: - try: - while True: - data = await ur.read(65536) - if not data: - break - if self.mode == "drop_response": - self.mode = "pass" - self.drops += 1 - cw.transport.abort() # RST: the client sees a peer close - return - if self.mode == "stall": - self.stalled += 1 - ev = asyncio.Event() - self._stalls.append(ev) - await ev.wait() # hold the response until told otherwise - return - cw.write(data) - await cw.drain() - except (ConnectionError, asyncio.CancelledError): - pass - - t1 = asyncio.create_task(c2u()) - t2 = asyncio.create_task(u2c()) - try: - await asyncio.wait({t1, t2}, return_when=asyncio.FIRST_COMPLETED) - finally: - for t in (t1, t2): - t.cancel() - for w in (cw, uw): + async def u2c() -> None: try: - w.close() - except Exception: # noqa: BLE001 + while data := await ur.read(65536): + if self.mode == "parser-body" and parser_target: + # Real headers start the installed parser; real body bytes stay withheld. + pending = bytearray(data) + while b"\r\n\r\n" not in pending: + assert len(pending) < 65536, ( + "response headers exceeded finite bound" + ) + more = await ur.read(4096) + assert more, "upstream closed before HTTP headers" + pending.extend(more) + split = pending.index(b"\r\n\r\n") + 4 + self.parser_handler = owner + cw.write(pending[:split]) + await cw.drain() + self.parser_headers.set() + await self.parser_release.wait() + cw.write(pending[split:]) + await cw.drain() + self.mode = "pass" + continue + if self.mode == "drop_response": + self.drop_remaining -= 1 + if self.drop_remaining == 0: + self.mode = "pass" + self.drop_remaining = 1 + self.drops += 1 + cw.transport.abort() + return + if self.mode == "stall": + self.stalled += 1 + ev = asyncio.Event() + self._stalls.append(ev) + await ev.wait() + return + cw.write(data) + await cw.drain() + except (ConnectionError, asyncio.CancelledError): pass + children = [asyncio.create_task(c2u()), asyncio.create_task(u2c())] + await asyncio.wait(children, return_when=asyncio.FIRST_COMPLETED) + finally: + for child in children: + child.cancel() + outcomes = await asyncio.gather(*children, return_exceptions=True) + for writer in writers: + writer.close() + await asyncio.gather( + *(writer.wait_closed() for writer in writers), return_exceptions=True + ) + self._writers.difference_update(writers) + for result in outcomes: + if isinstance(result, BaseException) and not isinstance( + result, (asyncio.CancelledError, OSError) + ): + raise result + @pytest.fixture async def proxy() -> Any: @@ -257,3 +336,565 @@ async def test_a_hung_statement_releases_its_slot_at_the_class_deadline( proxy.mode = "pass" # The client is intact: the next statement runs on a fresh connection. assert await through_proxy.query_all("SELECT 3 AS x") == [{"x": 3}] + + +def _vector_tls(tmp_path: Path) -> tuple[ssl.SSLContext, Path]: + """A fixture-owned certificate trusted by the actual driver's default context.""" + from cryptography import x509 + from cryptography.hazmat.primitives import hashes, serialization + from cryptography.hazmat.primitives.asymmetric import ec + from cryptography.x509.oid import NameOID + + key = ec.generate_private_key(ec.SECP256R1()) + name = x509.Name([x509.NameAttribute(NameOID.COMMON_NAME, "localhost")]) + now = datetime.now(UTC) + cert = ( + x509.CertificateBuilder() + .subject_name(name) + .issuer_name(name) + .public_key(key.public_key()) + .serial_number(x509.random_serial_number()) + .not_valid_before(now - timedelta(minutes=1)) + .not_valid_after(now + timedelta(hours=1)) + .add_extension( + x509.SubjectAlternativeName( + [x509.DNSName("localhost"), x509.IPAddress(ipaddress.ip_address("127.0.0.1"))] + ), + critical=False, + ) + .add_extension(x509.BasicConstraints(ca=True, path_length=0), critical=True) + .sign(key, hashes.SHA256()) + ) + cert_file, key_file = tmp_path / "relay.crt", tmp_path / "relay.key" + cert_file.write_bytes(cert.public_bytes(serialization.Encoding.PEM)) + key_file.write_bytes( + key.private_bytes( + serialization.Encoding.PEM, + serialization.PrivateFormat.PKCS8, + serialization.NoEncryption(), + ) + ) + key_file.chmod(0o600) + context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + context.minimum_version = ssl.TLSVersion.TLSv1_2 + context.load_cert_chain(cert_file, key_file) + return context, cert_file + + +@pytest.fixture(params=[False, True], ids=["http", "verified-https"]) +async def vector_native(request, monkeypatch, tmp_path): + from server.core import ch, request_work + from server.core.vectors import admission + from server.settings import settings + from tests.integration._resources import unique_org + + for name, value in { + "ch_pool_maxsize": 4, + "ch_pool_wait_ms": 500, + "ch_timeout_read_s": 3.0, + "ch_caller_shaped_max_execution_s": 3.0, + "vector_scan_max_concurrent_per_org": 2, + "vector_scan_max_waiters_per_org": 4, + "vector_scan_max_rows_to_read": 1_000_000, + }.items(): + monkeypatch.setattr(settings, name, value) + gate = admission._Admission() + monkeypatch.setattr(admission, "_admission", gate) + direct, client = ch.CHClient(), None + relay = None + try: + with admission.vector_work("", (), 0), request_work.bind(request_work.RequestWork()): + await direct.connect() + tls, cert_file = _vector_tls(tmp_path) if request.param else (None, None) + if cert_file is not None: + monkeypatch.setenv("SSL_CERT_FILE", str(cert_file)) + relay = _FaultProxy(settings.ch_host, int(settings.ch_port), tls=tls) + await relay.start() + monkeypatch.setattr(settings, "ch_host", "127.0.0.1") + monkeypatch.setattr(settings, "ch_port", relay.port) + monkeypatch.setattr(settings, "ch_secure", request.param) + client = ch.CHClient() + await client.connect() + assert await client.query_one("SELECT 1 AS ready") == {"ready": 1} + if request.param: + context = client.client._backend.ssl_context + assert context.verify_mode == ssl.CERT_REQUIRED and context.check_hostname + assert relay.tls_connections > 0 + yield SimpleNamespace( + client=client, + direct=direct, + relay=relay, + gate=gate, + hot=unique_org("vector-hot"), + light=unique_org("vector-light"), + ) + finally: + with admission.vector_work("", (), 0): + try: + if client is not None: + await client.disconnect() + finally: + try: + if relay is not None: + await relay.stop() + finally: + await direct.disconnect() + assert gate.population == 0 and gate.orgs == {} + + +async def _vector_call(client, org, request_id, sql, *, weight=1): + from server.core import ch, request_work + from server.core.vectors import admission + + with ( + request_work.bind(request_work.RequestWork()), + ch.query_tag("itest.vector-native " + request_id), + admission.vector_work(org, "native", weight), + ): + return await client.query_all(sql) + + +async def _vector_running(direct, request_id): + from server.core import ch + from server.core.vectors import admission + + with admission.vector_work("", (), 0), ch.query_tag("itest.vector-observer"): + rows = await direct.query_all( + "SELECT query_id, elapsed FROM system.processes " + "WHERE startsWith(query_id, {prefix:String}) AND query_id != queryID() LIMIT 16", + {"prefix": request_id + ":"}, + ) + assert len(rows) < 16, "owned running-query census reached its finite bound" + return rows + + +async def _vector_remote_end(direct, request_id, initially_seen): + """Positive native terminal events for every observed start, not one empty sample.""" + from server.core import ch + from server.core.vectors import admission + + with admission.vector_work("", (), 0), ch.query_tag("itest.vector-observer"): + defaults = { + row["name"]: row["default"] + for row in await direct.query_all( + "SELECT name, default FROM system.settings WHERE name IN " + "('max_rows_to_read','max_memory_usage','max_execution_time','read_overflow_mode')" + ) + } + deadline = time.monotonic() + 12 + peak, samples = len(initially_seen), 0 + seen = set(initially_seen) + while True: + active = await _vector_running(direct, request_id) + seen.update(row["query_id"] for row in active) + peak, samples = max(peak, len(active)), samples + 1 + await direct.execute("SYSTEM FLUSH LOGS") + events = await direct.query_all( + "SELECT query_id, toString(type) AS event, exception_code, " + "toUnixTimestamp64Micro(event_time_microseconds) AS at_us, " + "Settings['max_rows_to_read'] AS rows_cap, " + "Settings['max_memory_usage'] AS memory_cap, " + "Settings['max_execution_time'] AS time_cap, " + "Settings['read_overflow_mode'] AS overflow " + "FROM system.query_log WHERE startsWith(query_id, {prefix:String}) " + "AND is_initial_query ORDER BY event_time_microseconds, event LIMIT 64", + {"prefix": request_id + ":"}, + ) + assert len(events) < 64, "owned attempt evidence exceeded its finite census" + starts = [row for row in events if row["event"] == "QueryStart"] + terminal = [ + row + for row in events + if row["event"] + in ("QueryFinish", "ExceptionWhileProcessing", "ExceptionBeforeStart") + ] + seen.update(row["query_id"] for row in events) + ended = {row["query_id"] for row in terminal} + for row in starts: + for field, name in { + "rows_cap": "max_rows_to_read", + "memory_cap": "max_memory_usage", + "time_cap": "max_execution_time", + "overflow": "read_overflow_mode", + }.items(): + row[field] = row[field] or defaults[name] + assert float(row["rows_cap"]) == 1_000_000 and row["overflow"] == "throw" + assert 0 < float(row["memory_cap"]) <= 4_000_000_000 + assert 0 < float(row["time_cap"]) <= 3 + if starts and not active and seen <= ended and len(terminal) >= len(starts): + print( + "vector-native-lifetime " + + json.dumps( + { + "request_id": request_id, + "peak_observed_remote": peak, + "samples": samples, + "events": events, + }, + sort_keys=True, + ) + ) + return events + assert time.monotonic() < deadline, { + "active": active, + "seen": sorted(seen), + "events": events, + } + await asyncio.sleep(0.05) + + +async def _vector_cleanup(direct, request_id, failure): + """Stop only exact owned queries, preserving a failed case over a cleanup error.""" + from server.core import ch + from server.core.vectors import admission + + try: + with admission.vector_work("", (), 0), ch.query_tag("itest.vector-cleanup"): + for row in await _vector_running(direct, request_id): + query_id = row["query_id"] + assert query_id.startswith(request_id + ":") + await direct.execute( + "KILL QUERY WHERE query_id = {owned:String} SYNC", {"owned": query_id} + ) + assert not await _vector_running(direct, request_id) + except Exception as error: # Keep the first failed assertion as the result. + if failure is None: + raise + failure.add_note("owned vector cleanup also failed: " + type(error).__name__) + + +def _vector_drained(state): + from server.core import ch + + assert state.gate.population == 0 and state.gate.orgs == {} + assert state.client._slots._value == 4 + assert state.client._inflight == {} and state.client._retired == {} + assert ch.pool_reading()["in_use"] == 0 and ch.pool_reading()["waiters"] == 0 + + +@pytest.mark.parametrize("outcome", ["cancel", "deadline", "stalled-response"]) +async def test_native_vector_failure_drains_local_work_and_observes_remote_end( + vector_native, monkeypatch, outcome +): + from server.core.errors import VectorAdmissionError + from server.settings import settings + + state = vector_native + request_id = "vector-held-" + uuid.uuid4().hex + # The server executes a finite ~2 s block; local cancellation is not a remote ack. + sql = ( + "SELECT sum(number) AS total, " + "sum(cosineDistance([toFloat32(number + 1), 1.0], [1.0, 0.0])) AS score " + "FROM numbers(100) WHERE sleepEachRow(0.02) = 0" + ) + if outcome == "deadline": + monkeypatch.setattr(settings, "ch_timeout_read_s", 1.0) + caller = asyncio.create_task(_vector_call(state.client, state.hot, request_id, sql, weight=2)) + queued = None + try: + deadline = time.monotonic() + 2 + while not (active := await _vector_running(state.direct, request_id)): + assert not caller.done(), "native held query ended before the observation" + assert time.monotonic() < deadline, "native held query never appeared" + await asyncio.sleep(0.01) + assert len(active) == 1 + assert state.gate.population == 2 and state.gate.orgs[state.hot].held == 2 + queued = asyncio.create_task( + _vector_call( + state.client, + state.hot, + "vector-queued-" + uuid.uuid4().hex, + "SELECT cosineDistance([1.0, 0.0], [1.0, 0.0]) AS score", + ) + ) + await asyncio.sleep(0) + assert state.gate.orgs[state.hot].waiting == 1 and state.gate.population == 3 + queued.cancel() + with pytest.raises(asyncio.CancelledError): + await queued + assert state.gate.orgs[state.hot].waiting == 0 and state.gate.population == 2 + light_id = "vector-light-" + uuid.uuid4().hex + light = await _vector_call( + state.client, + state.light, + light_id, + "SELECT cosineDistance([1.0, 0.0], [1.0, 0.0]) AS score, 7 AS complete", + ) + assert light == [{"score": 0.0, "complete": 7}] + overlap = await _vector_running(state.direct, request_id) + assert overlap and not caller.done(), ( + "the light org must finish while hot native work remains" + ) + assert state.gate.population == 2 and state.gate.orgs[state.hot].held == 2 + if outcome == "stalled-response": + state.relay.mode = "stall" + if outcome == "cancel": + caller.cancel() + with pytest.raises(asyncio.CancelledError): + await caller + else: + with pytest.raises(VectorAdmissionError) as caught: + await caller + assert caught.value.extra["reason"] == "deadline" + if outcome == "stalled-response": + assert state.relay.stalled == 1, "the attributed response was not stalled" + state.relay.mode = "pass" + _vector_drained(state) + assert ( + await _vector_call( + state.client, + state.light, + "vector-reused-" + uuid.uuid4().hex, + "SELECT cosineDistance([1.0, 0.0], [1.0, 0.0]) AS score, 7 AS complete", + ) + == light + ) + after_local = await _vector_running(state.direct, request_id) + print( + "vector-native-competing-progress " + + json.dumps( + { + "outcome": outcome, + "hot_query_ids_at_light_completion": [row["query_id"] for row in overlap], + "remote_after_local_failure": after_local, + "light_complete": light, + }, + sort_keys=True, + ) + ) + events = await _vector_remote_end( + state.direct, request_id, [row["query_id"] for row in active] + ) + assert events and all(row["query_id"].startswith(request_id + ":") for row in events) + await _vector_remote_end(state.direct, light_id, []) + _vector_drained(state) + finally: + state.relay.mode = "pass" + for task in (caller, queued): + if task is not None and not task.done(): + task.cancel() + await asyncio.gather( + *(task for task in (caller, queued) if task is not None), return_exceptions=True + ) + await _vector_cleanup(state.direct, request_id, sys.exception()) + + +@pytest.mark.parametrize("retry_layer", ["driver", "core"]) +async def test_native_vector_remote_close_correlates_installed_retries(vector_native, retry_layer): + state = vector_native + request_id = "vector-retry-" + uuid.uuid4().hex + sql = "SELECT cosineDistance([1.0, 0.0], [1.0, 0.0]) AS score, 7 AS complete" + control = await _vector_call(state.client, state.hot, "vector-control-" + uuid.uuid4().hex, sql) + generation = state.client.generation + hidden_attempts = max(2, state.client.client.query_retries + 1) + drops = 1 if retry_layer == "driver" else hidden_attempts + try: + state.relay.drop_remaining = drops + state.relay.mode = "drop_response" + result = await _vector_call(state.client, state.hot, request_id, sql) + assert result == control == [{"score": 0.0, "complete": 7}] + assert state.relay.drops == drops + assert state.client.generation == generation + int(retry_layer == "core") + events = await _vector_remote_end(state.direct, request_id, []) + starts = [row for row in events if row["event"] == "QueryStart"] + finishes = [row for row in events if row["event"] == "QueryFinish"] + assert len(starts) == len(finishes) == drops + 1 + identities = {row["query_id"] for row in starts} + assert len(identities) == (1 if retry_layer == "driver" else 2) + if retry_layer == "core": + counts = sorted(sum(row["query_id"] == qid for row in starts) for qid in identities) + assert counts == [1, hidden_attempts] + assert all(row["exception_code"] == 0 for row in finishes) + print( + "vector-native-retry-policy " + + json.dumps( + { + "layer": retry_layer, + "query_retries": state.client.client.query_retries, + "dropped_responses": drops, + "native_executions": len(starts), + "native_identities": sorted(identities), + "generation_change": state.client.generation - generation, + }, + sort_keys=True, + ) + ) + _vector_drained(state) + finally: + state.relay.mode = "pass" + await _vector_cleanup(state.direct, request_id, sys.exception()) + + +def _native_parser_stack(thread_id, source): + """Observe the real executor parser waiting on this response's normal byte queue.""" + frame = sys._current_frames().get(thread_id) + found = {"parser": False, "queue_get": False, "condition_wait": False} + while frame is not None: + name, filename = frame.f_code.co_name, frame.f_code.co_filename + if name == "parse_streaming" and frame.f_locals.get("streaming_source") is source: + found["parser"] = True + receiver = frame.f_locals.get("self") + if ( + name == "get" + and filename.endswith("/clickhouse_connect/driver/asyncqueue.py") + and getattr(receiver, "_p", None) is source.queue + ): + found["queue_get"] = True + if name == "wait" and receiver is source.queue._sync_not_empty: + found["condition_wait"] = True + frame = frame.f_back + return found + + +async def test_native_parser_executor_retires_after_caller_cancellation(vector_native, monkeypatch): + """Cancel a real blocked parser, then prove its concurrent executor work actually ended.""" + state = vector_native + request_id = "vector-parser-" + uuid.uuid4().hex + marker = "/* " + request_id + " */" + query = ( + "SELECT number, cosineDistance([toFloat32(number + 1), 1.0], [1.0, 0.0]) AS score " + "FROM numbers(3) " + ) + control = await _vector_call( + state.client, state.hot, "vector-parser-control-" + uuid.uuid4().hex, query + ) + assert len(control) == 3 and [r["number"] for r in control] == [0, 1, 2] + loop = asyncio.get_running_loop() + executor = loop._default_executor + assert isinstance(executor, ThreadPoolExecutor), "actual driver parser executor unavailable" + submit = executor.submit + calls = [] + + def observed_submit(function, *args, **kwargs): + code = getattr(function, "__code__", None) + if code is None or code.co_name != "parse_streaming": + return submit(function, *args, **kwargs) + closure = dict( + zip(code.co_freevars, (c.cell_contents for c in function.__closure__), strict=True) + ) + context = closure.get("context") + if marker not in str(getattr(context, "query", "")): + return submit(function, *args, **kwargs) + assert not calls, "unexpected repeated parser submission for the held native response" + record = { + "source": closure["streaming_source"], + "entered": threading.Event(), + "retired": threading.Event(), + "thread_id": None, + } + calls.append(record) + + def delegate(): + record["thread_id"] = threading.get_ident() + record["entered"].set() + try: + return function(*args, **kwargs) + finally: + record["retired"].set() + + # Observe the actual concurrent Future; do not replace the driver parser/executor. + record["future"] = submit(delegate) + return record["future"] + + monkeypatch.setattr(executor, "submit", observed_submit) + state.relay.parser_marker = marker.encode() + state.relay.mode = "parser-body" + caller = asyncio.create_task( + _vector_call(state.client, state.hot, request_id, query + marker, weight=2) + ) + try: + deadline = time.monotonic() + 2 + while True: + assert not caller.done(), "caller ended before an actual parser read was observed" + if calls and calls[0]["entered"].is_set(): + record = calls[0] + blocked = _native_parser_stack(record["thread_id"], record["source"]) + if all(blocked.values()): + break + assert time.monotonic() < deadline, "actual executor parser never blocked on real bytes" + await asyncio.sleep(0.01) + assert state.relay.parser_headers.is_set() and not state.relay.parser_release.is_set() + assert record["future"].running() and not record["retired"].is_set() + assert state.gate.population == 2 and state.gate.orgs[state.hot].held == 2 + caller.cancel() + with pytest.raises(asyncio.CancelledError): + await caller + deadline = time.monotonic() + 2 + while not record["future"].done() or not record["retired"].is_set(): + assert time.monotonic() < deadline, "canceled await left actual executor work running" + await asyncio.sleep(0.01) + assert not record["future"].cancelled(), "canceled Future is not executor-retirement proof" + assert not _native_parser_stack(record["thread_id"], record["source"])["parser"] + source = record["source"] + assert source.queue._shutdown and source._producer_task.done() + assert source.response.closed and source._lease_released + # Cancellation, not releasing bytes, must have retired the real thread's work. + assert not state.relay.parser_release.is_set() + handler = state.relay.parser_handler + assert handler is not None + deadline = time.monotonic() + 2 + while not handler.done() or handler in state.relay._handlers: + assert time.monotonic() < deadline, ( + "owned parser relay did not close after cancellation" + ) + await asyncio.sleep(0.01) + assert handler.exception() is None + _vector_drained(state) + events = await _vector_remote_end(state.direct, request_id, []) + starts = [r for r in events if r["event"] == "QueryStart"] + ends = [r for r in events if r["event"] == "QueryFinish"] + assert len(starts) == len(ends) == 1 and ends[0]["exception_code"] == 0 + state.relay.mode = "pass" + state.relay.parser_release.set() + reused_id = "vector-parser-reused-" + uuid.uuid4().hex + assert await _vector_call(state.client, state.light, reused_id, query) == control + await _vector_remote_end(state.direct, reused_id, []) + _vector_drained(state) + error = record["future"].exception() + print( + "vector-native-parser-retirement " + + json.dumps( + { + "request_id": request_id, + "native_query_ids": [r["query_id"] for r in starts], + "executor": type(executor).__name__, + "thread_id": record["thread_id"], + "observed_real_blocked_stack": blocked, + "concurrent_future_done": record["future"].done(), + "concurrent_future_cancelled": record["future"].cancelled(), + "delegate_retired": record["retired"].is_set(), + "delegate_error": type(error).__name__ if error else None, + "response_producer_done": source._producer_task.done(), + "response_lease_released": source._lease_released, + "held_relay_done": handler.done(), + "complete_reuse": control, + }, + sort_keys=True, + ) + ) + finally: + failure = sys.exception() + monkeypatch.setattr(executor, "submit", submit) + state.relay.mode = "pass" + state.relay.parser_release.set() + if not caller.done(): + caller.cancel() + await asyncio.gather(caller, return_exceptions=True) + try: + if calls and (pending_future := calls[0].get("future")) is not None: + deadline = time.monotonic() + 2 + while not pending_future.done(): + if time.monotonic() >= deadline: + if failure is not None: + failure.add_note( + "owned parser work did not retire during finite cleanup" + ) + break + raise AssertionError( + "owned parser work did not retire during finite cleanup" + ) + await asyncio.sleep(0.01) + if pending_future.done() and not pending_future.cancelled(): + pending_future.exception() + finally: + await _vector_cleanup(state.direct, request_id, failure or sys.exception()) diff --git a/tests/integration/test_vector_native_budget.py b/tests/integration/test_vector_native_budget.py index 7aff71037d..90c7b2fcac 100644 --- a/tests/integration/test_vector_native_budget.py +++ b/tests/integration/test_vector_native_budget.py @@ -2,6 +2,12 @@ from __future__ import annotations +import random +import uuid +from collections.abc import Iterator +from typing import Any + +import httpx import pytest from clickhouse_connect.driver.exceptions import DatabaseError @@ -10,7 +16,9 @@ from server.core.errors import VectorAdmissionError from server.core.vectors import admission from server.settings import settings -from tests.integration._resources import unique_org +from tests.integration._eventually import wait_until +from tests.integration._resources import memory_rows, minted_identity, unique_org +from tests.integration._spawn import loom_server pytestmark = pytest.mark.usefixtures("ch_target_required") @@ -62,3 +70,146 @@ async def test_native_row_ceiling_refuses_the_read_and_releases_credits( finally: await client.disconnect() assert admission.current_work() == work_before + + +_BUDGET_WORDS = ( + "amber anchor apple arrow autumn bamboo barley beach berry birch bridge bronze " + "cabin canal candle canyon carbon cedar chalk cherry cipher citrus cliff cloud " + "cobalt comet copper coral cotton crane crystal desert dune eagle echo ember " + "engine falcon fern field flint forest fossil garden garnet glacier granite " + "harbor hazel helium heron hollow honey horizon indigo island ivory jade jasper " + "juniper kernel kettle lagoon lantern laurel lemon linen lotus magnet maple " + "marble meadow meteor mint mirror mosaic nectar nickel north oasis ocean olive " + "onyx orbit orchid oyster pebble pepper pine planet plaza pollen prism quartz " + "quill radar raven reef ridge river rocket saffron sage salmon sapphire shadow " + "signal silver slate solar spruce summit sunset thistle thunder timber topaz " + "tulip tundra valley velvet violet walnut willow winter zephyr" +).split() + + +def _budget_value(index: int) -> str: + # Ordinary words keep the real sensitivity and conflict checks in force. + return " ".join(random.Random(index).sample(_BUDGET_WORDS, 12)) + + +@pytest.fixture +def native_api_budget( + control_plane_url: str, mint_test_token: Any, ch_client: Any +) -> Iterator[tuple[httpx.Client, httpx.Client, dict[str, str], Any]]: + principal = minted_identity(mint_test_token, "vector-api-budget") + identity = { + "org": principal["org"], + "namespace": principal["namespace"], + "agent": "native-budget", + "user_id": "native-user", + "session_id": "native-session", + } + headers = {"Authorization": f"Bearer {principal['token']}"} + with httpx.Client(base_url=control_plane_url, headers=headers, timeout=30) as ordinary: + for index in range(32): + response = ordinary.post( + "/v1/memory.set", + json={ + **identity, + "memory_key": f"budget-{index:02d}", + "memory_type": "semantic", + "value": _budget_value(index), + }, + ) + assert response.status_code == 200, response.text + body = response.json() + assert body["memory_key"] == f"budget-{index:02d}" + assert body["status"] == "active" and body["version"] > 0 and not body["noop"] + + def all_seed_embeddings() -> bool: + response = ordinary.post( + "/v1/memory.search_trace", + json={**identity, "query": _budget_value(0), "top_k": 64, "auto_touch": False}, + ) + assert response.status_code == 200, response.text + keys = [row["memory_key"] for row in response.json()["results"]] + assert len(keys) == len(set(keys)) + return set(keys) == {f"budget-{index:02d}" for index in range(32)} + + # This establishes the seed precondition, not a write-visibility timing result. + wait_until(all_seed_embeddings, description="all native budget seed embeddings") + with ( + loom_server( + "vector-api-budget", + { + "LOOM_AUTH_REQUIRED": "true", + "LOOM_JWT_SECRET": settings.jwt_secret, + "LOOM_BACKGROUND_WORKERS": "off", + "LOOM_VECTOR_SCAN_MAX_ROWS_TO_READ": "16", + }, + ) as base, + httpx.Client(base_url=base, headers=headers, timeout=30) as capped, + ): + scalar = capped.post("/v1/memory.get", json={**identity, "memory_key": "budget-00"}) + assert scalar.status_code == 200 and scalar.json()["found"] is True + assert scalar.json()["value"] == _budget_value(0) + yield ordinary, capped, identity, ch_client + + +@pytest.mark.parametrize("operation", ["search", "set", "update", "bulk_set"]) +def test_native_api_row_refusal_preserves_committed_heads( + native_api_budget: tuple[httpx.Client, httpx.Client, dict[str, str], Any], + operation: str, +) -> None: + ordinary, capped, identity, store = native_api_budget + before = memory_rows(store, identity["org"]) + assert len(before) == 32 + head = ordinary.post("/v1/memory.get", json={**identity, "memory_key": "budget-00"}) + assert head.status_code == 200 and head.json()["found"] is True + version_before = head.json()["version"] + assert version_before > 0 + key = "budget-00" if operation == "update" else "refused-new" + if operation == "search": + op = "memory.search" + payload: dict[str, Any] = { + **identity, + "query": _budget_value(0), + "top_k": 8, + "auto_touch": False, + } + elif operation == "update": + op = "memory.update" + payload = {**identity, "memory_key": key, "value": _budget_value(100)} + else: + entry = {"memory_key": key, "memory_type": "semantic", "value": _budget_value(100)} + op = "memory.bulk_set" if operation == "bulk_set" else "memory.set" + payload = ( + {**identity, "entries": [entry]} + if operation == "bulk_set" + else { + **identity, + **entry, + } + ) + request_id = uuid.uuid4().hex + response = capped.post(f"/v1/{op}", json=payload, headers={"X-Request-Id": request_id}) + assert response.status_code == 429, response.text + assert response.json()["detail"]["code"] == "VECTOR_CAPACITY_REFUSED" + assert response.headers["Retry-After"] == "1" + assert response.headers["X-Request-Id"] == request_id + # A retryable response must be caused by the real rows ceiling, not a queue refusal. + store.command("SYSTEM FLUSH LOGS") + witnesses = store.query( + "SELECT exception_code, Settings['max_rows_to_read'] FROM system.query_log " + "WHERE log_comment = {tag:String} AND is_initial_query " + "AND type IN ('ExceptionBeforeStart', 'ExceptionWhileProcessing') LIMIT 65", + parameters={"tag": f"{op} {request_id}"}, + ).result_rows + assert 0 < len(witnesses) <= 64 + assert any(int(code) == 158 and limit == "16" for code, limit in witnesses) + assert memory_rows(store, identity["org"]) == before + found = ordinary.post("/v1/memory.get", json={**identity, "memory_key": key}) + assert found.status_code == 200 + assert found.json()["found"] is (operation == "update") + if operation == "update": + assert found.json()["value"] == _budget_value(0) + # The capped process still handles a complete scalar read after the refused vector work. + scalar = capped.post("/v1/memory.get", json={**identity, "memory_key": "budget-00"}) + assert scalar.status_code == 200 and scalar.json()["found"] is True + assert scalar.json()["value"] == _budget_value(0) + assert scalar.json()["version"] == version_before diff --git a/tests/unit/test_mixed_version_gate.py b/tests/unit/test_mixed_version_gate.py index 75de898ddb..e2690a63eb 100644 --- a/tests/unit/test_mixed_version_gate.py +++ b/tests/unit/test_mixed_version_gate.py @@ -70,3 +70,159 @@ def run(cmd, **kwargs): assert port == 18123 assert calls[-1] == ["docker", "rm", "-f", "test-mixed-isolated"] assert calls[0][calls[0].index("--name") + 1] == "test-mixed-isolated" + + +def test_latency_summary_keeps_refusals_and_small_sample_percentiles() -> None: + from bench.scale_load import latency_observations + + samples = [ + {"elapsed_ms": 2.0, "error": ""}, + {"elapsed_ms": 9.0, "error": ""}, + {"elapsed_ms": 3.0, "error": "503:PROJECTION_BEHIND"}, + ] + out = latency_observations(samples) + assert out["attempted"] == 3 and out["n"] == 2 + assert out["p50_ms"] == 2 and out["p99_ms"] == 9 + assert out["errors"] == {"503:PROJECTION_BEHIND": 1} + assert out["samples"] == samples + + +@pytest.mark.parametrize("ms", [float("nan"), float("inf"), -1]) +def test_latency_summary_refuses_invalid_timings(ms) -> None: + from bench.scale_load import latency_observations + + with pytest.raises(AssertionError): + latency_observations([{"elapsed_ms": ms}]) + + +def _latency_written(): + return { + "memory_key": "k", + "version": 2, + "memory_id": "id", + "status": "active", + "noop": False, + "conflict_verdict": "clear", + } + + +@pytest.mark.parametrize( + "delta", + [ + {"noop": True}, + {"version": 0}, + {"duplicate_of": "other"}, + {"merged_into_key": "other"}, + {"conflict_verdict": ""}, + ], +) +def test_latency_fixture_never_treats_every_200_as_a_commit(delta) -> None: + from bench.scale_load import committed_latency_rows + + with pytest.raises(AssertionError): + committed_latency_rows( + "memory.set", {**_latency_written(), **delta}, [{"memory_key": "k", "value": "v"}] + ) + + +def test_latency_bulk_checks_complete_outcomes_and_point_endstate() -> None: + from bench.scale_load import assert_latency_current, committed_latency_rows + + result = {"written": [_latency_written()]} + expected = committed_latency_rows( + "memory.bulk_set", result, [{"memory_key": "k", "value": "v"}] + ) + assert_latency_current({"found": True, "status": "active", **expected["k"]}, expected["k"]) + with pytest.raises(AssertionError): + committed_latency_rows( + "memory.bulk_set", {**result, "unchecked": 1}, [{"memory_key": "k", "value": "v"}] + ) + with pytest.raises(AssertionError): + assert_latency_current( + {"found": True, "status": "active", **expected["k"], "value": "old"}, expected["k"] + ) + + +@pytest.mark.parametrize( + "rows", + [ + [], + [{"memory_key": "k", "version": 1, "status": "active"}], + [{"memory_key": "k", "version": 2, "status": "rejected"}], + [{"memory_key": "k", "version": 2, "status": "active"}] * 2, + ], +) +def test_latency_first_search_cannot_poll_away_a_missing_stale_or_rejected_row(rows) -> None: + from bench.scale_load import assert_latency_visible + + with pytest.raises(AssertionError): + assert_latency_visible({"results": rows}, "k", {"version": 2}) + + +def test_latency_first_search_and_real_worker_identity_positive_controls() -> None: + from bench.scale_load import assert_latency_visible, assert_latency_worker + + assert_latency_visible( + {"results": [{"memory_key": "k", "version": 2, "status": "active"}]}, "k", {"version": 2} + ) + assert_latency_worker(102, 100, 100, 100) + for args in ((100, 100, 100, 100), (102, 100, 999, 100), (102, 100, 100, 999)): + with pytest.raises(AssertionError): + assert_latency_worker(*args) + + +def test_latency_partial_start_registers_and_stops_both_owned_groups( + monkeypatch: pytest.MonkeyPatch, tmp_path +) -> None: + from types import SimpleNamespace + + owned = [] + stopped = [] + + class RefusedStart: + def __init__(self, name, *args, **kwargs): + self.name = name + + def start(self): + assert len(owned) == 2 + raise RuntimeError("start refused") + + def stop(self): + stopped.append(self.name) + + monkeypatch.setattr(mixed_version, "Server", RefusedStart) + scenario = mixed_version.Scenario.__new__(mixed_version.Scenario) + scenario.builds = {label: SimpleNamespace(tree=tmp_path) for label in ("N-1", "N")} + scenario.ch_port = 18123 + scenario.owned_logs = tmp_path + scenario.owned_servers = owned + with pytest.raises(RuntimeError, match="start refused"): + scenario.latency_acceptance("base", "head") + assert len(owned) == 2 and stopped == ["latency-base-0", "latency-base-1"] + + +def test_latency_transport_retains_lag_refusal_without_retry( + monkeypatch: pytest.MonkeyPatch, tmp_path +) -> None: + from types import SimpleNamespace + + calls = [] + + def post(url, **kwargs): + calls.append((url, kwargs)) + return SimpleNamespace( + status_code=503, + text="projection behind", + json=lambda: {"detail": {"code": "PROJECTION_BEHIND"}}, + ) + + monkeypatch.setattr(mixed_version.httpx, "post", post) + monkeypatch.setattr(mixed_version.time, "perf_counter", lambda: 1.0) + srv = mixed_version.Server("latency-test", tmp_path, 18123, tmp_path) + observation = {} + with pytest.raises(AssertionError, match="503"): + srv.call("token", "memory.search", {}, observation=observation) + assert len(calls) == 1 + assert calls[0][1]["headers"] == {"Authorization": "Bearer token"} + assert calls[0][1]["timeout"] == 60 + assert observation == {"elapsed_ms": 0.0, "status": 503, "error": "503:PROJECTION_BEHIND"} From c0577749ff88531f877b2d231d03a2a595e97079 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Wed, 18 Nov 2026 23:32:45 +0000 Subject: [PATCH 18/28] Retain refusal request IDs and verify native process restarts Signed-off-by: zlareb1 --- server/main.py | 7 + server/transport/http.py | 1 + tests/integration/test_ch_fault_injection.py | 324 +++++++++++++++++++ tests/unit/test_http_error_request_id.py | 176 ++++++++++ 4 files changed, 508 insertions(+) create mode 100644 tests/unit/test_http_error_request_id.py diff --git a/server/main.py b/server/main.py index e41cd570de..c57078e2bd 100644 --- a/server/main.py +++ b/server/main.py @@ -39,6 +39,7 @@ from server.settings import settings from server.transport import http as http_transport from server.transport import mcp as mcp_transport +from server.transport._common import request_id_from from server.transport.request_scope import HTTPApplication from server.wiring import wire @@ -436,6 +437,12 @@ async def _web_gate(request: Request, call_next: Any) -> Response: async def _response_headers(request: Request, call_next: Any) -> Response: """Headers every answer leaves with.""" response: Response = await call_next(request) + # An exception replaces FastAPI's temporary response; retain its bound request id. + response.headers["X-Request-Id"] = ( + getattr(request.state, "loom_request_id", None) + or response.headers.get("X-Request-Id") + or request_id_from(request.headers.get("X-Request-Id")) + ) path = request.url.path # A year-long immutable cache for content-hashed /assets/* names: the hash is the # version, so a load does not re-validate them. diff --git a/server/transport/http.py b/server/transport/http.py index 007b5f43ce..5b3fc8d581 100644 --- a/server/transport/http.py +++ b/server/transport/http.py @@ -230,6 +230,7 @@ def _begin_request(op: Operation, request: Request, response: Response) -> str: """The request id, the schema header and the per-request context vars — before the body is read. Returns the request id.""" request_id = request_id_from(request.headers.get("X-Request-Id")) + request.state.loom_request_id = request_id response.headers["X-Request-Id"] = request_id response.headers["X-Schema-Version"] = SCHEMA_VERSION bind_request(op.name, request_id) diff --git a/tests/integration/test_ch_fault_injection.py b/tests/integration/test_ch_fault_injection.py index 57d989d1dd..c4b209b14e 100644 --- a/tests/integration/test_ch_fault_injection.py +++ b/tests/integration/test_ch_fault_injection.py @@ -23,9 +23,12 @@ from __future__ import annotations import asyncio +import hashlib import ipaddress import json +import os import re +import signal import ssl import sys import threading @@ -898,3 +901,324 @@ def delegate(): pending_future.exception() finally: await _vector_cleanup(state.direct, request_id, failure or sys.exception()) + + +# One finite native client, not a serving web worker or a reusable command server. +_VECTOR_RESTART_CHILD = """ +import asyncio, hashlib, json, os, ssl, sys, time +from pathlib import Path +from types import SimpleNamespace +from server.core import ch +from server.core.vectors import admission +from server.settings import settings +from tests.integration.test_ch_fault_injection import ( + _vector_call, _vector_drained, _vector_running, +) + +def note(kind, **fields): + print(json.dumps({"kind": kind, **fields}, sort_keys=True), flush=True) + +async def main(payload): + for name, value in payload["settings"].items(): + setattr(settings, name, value) + client = ch.CHClient() + caller = None + try: + await client.connect() + assert await client.query_one("SELECT 1 AS ready") == {"ready": 1} + verified = False + if settings.ch_secure: + context = client.client._backend.ssl_context + assert context.verify_mode == ssl.CERT_REQUIRED and context.check_hostname + verified = True + stat = Path("/proc/self/stat").read_text().rsplit(")", 1)[1].split() + note("ready", pid=os.getpid(), uid=os.getuid(), parent=os.getppid(), + start_ticks=stat[19], settings=payload["settings"], verified_tls=verified, + query_retries=client.client.query_retries, + core_sha256=hashlib.sha256(Path(ch.__file__).read_bytes()).hexdigest()) + deadline = time.monotonic() + 10 + while not Path(payload["start_file"]).exists(): + assert time.monotonic() < deadline, "own start barrier was not released" + await asyncio.sleep(0.01) + request_id = payload["request_id"] + caller = asyncio.create_task(_vector_call( + client, payload["hot"], request_id, payload["query"], weight=2 + )) + deadline = time.monotonic() + 2 + while not (active := await _vector_running(client, request_id)): + assert not caller.done(), "held native read ended before observation" + assert time.monotonic() < deadline, "held native read never started" + await asyncio.sleep(0.01) + gate = admission._admission + assert len(active) == 1 and gate.population == 2 and gate.orgs[payload["hot"]].held == 2 + note("held", query_ids=[r["query_id"] for r in active], population=gate.population, + held=gate.orgs[payload["hot"]].held, pool_in_use=ch.pool_reading()["in_use"]) + light = await _vector_call( + client, payload["light"], request_id + "-light", + "SELECT cosineDistance([1.0, 0.0], [1.0, 0.0]) AS score, 7 AS complete", + ) + assert light == [{"score": 0.0, "complete": 7}] + active = await _vector_running(client, request_id) + assert active and not caller.done() and gate.orgs[payload["hot"]].held == 2 + note("light", result=light, hot_query_ids=[r["query_id"] for r in active]) + result = await caller + assert len(result) == 1 and result[0]["total"] == 6 + _vector_drained(SimpleNamespace(client=client, gate=gate)) + note("drained", result=result, population=gate.population, orgs=gate.orgs, + pool=ch.pool_reading(), generation=client.generation, + inflight=client._inflight, retired=client._retired) + finally: + if caller is not None and not caller.done(): + caller.cancel() + if caller is not None: + await asyncio.gather(caller, return_exceptions=True) + await client.disconnect() + +asyncio.run(main(json.loads(sys.argv[1]))) +""" + + +def _native_child_identity(process): + stat = Path(f"/proc/{process.pid}/stat").read_text().rsplit(")", 1)[1].split() + identity = { + "pid": process.pid, + "uid": Path(f"/proc/{process.pid}").stat().st_uid, + "parent": int(stat[1]), + "start_ticks": stat[19], + } + assert identity["uid"] == os.getuid() and identity["parent"] == os.getpid() + return identity + + +async def _native_child_note(process, expected): + line = await asyncio.wait_for(process.stdout.readline(), timeout=30) + assert line and len(line) < 16384, "own child exited or exceeded its finite receipt bound" + note = json.loads(line) + assert note.get("kind") == expected, note + return note + + +async def test_native_process_death_restart_keeps_local_caps_and_observes_remote_work( + vector_native, tmp_path +): + from server.core import ch + from server.settings import settings + from tests.integration._redact import redact + from tests.integration._resources import ch_target + + ch_target() # The child inherits the explicitly configured CI store's credentials. + state = vector_native + names = ( + "ch_host", + "ch_port", + "ch_secure", + "ch_pool_maxsize", + "ch_pool_wait_ms", + "ch_timeout_read_s", + "ch_caller_shaped_max_execution_s", + "ch_caller_shaped_max_memory_bytes", + "vector_scan_max_concurrent_per_org", + "vector_scan_max_waiters_per_org", + "vector_scan_max_rows_to_read", + ) + configuration = {name: getattr(settings, name) for name in names} + assert configuration["ch_pool_maxsize"] == 4 + assert configuration["vector_scan_max_concurrent_per_org"] == 2 + query = ( + "SELECT sum(number) AS total, " + "sum(cosineDistance([toFloat32(number + 1), 1.0], [1.0, 0.0])) AS score " + "FROM numbers(4) WHERE sleepEachRow(0.4) = 0" + ) + owned = [] + requests = [] + parent_reuse_id = "vector-process-parent-reuse-" + uuid.uuid4().hex + requests.append(parent_reuse_id) + root = (await asyncio.to_thread(Path(__file__).resolve)).parents[2] + expected_core = hashlib.sha256( + await asyncio.to_thread(Path(ch.__file__).read_bytes) + ).hexdigest() + + async def start(label, barrier): + request_id = "vector-process-" + uuid.uuid4().hex + requests.extend((request_id, request_id + "-light")) + payload = { + "request_id": request_id, + "settings": configuration, + "query": query, + "hot": state.hot, + "light": state.light, + "start_file": str(barrier), + } + log_path = tmp_path / (label + ".stderr") + log = log_path.open("wb") + try: + process = await asyncio.create_subprocess_exec( + sys.executable, + "-c", + _VECTOR_RESTART_CHILD, + json.dumps(payload), + cwd=root, + env=os.environ.copy(), + stdout=asyncio.subprocess.PIPE, + stderr=log, + ) + except BaseException: # A failed/canceled spawn must close its private log descriptor. + log.close() + raise + record = { + "process": process, + "log": log, + "log_path": log_path, + "request_id": request_id, + "identity": None, + "pidfd": None, + } + owned.append(record) # Register before any identity/readiness failure can unwind. + record["identity"] = _native_child_identity(process) + record["pidfd"] = os.pidfd_open(process.pid) + assert _native_child_identity(process) == record["identity"] + ready = await _native_child_note(process, "ready") + assert all(ready[key] == value for key, value in record["identity"].items()) + assert ready["settings"] == configuration and ready["core_sha256"] == expected_core + assert ready["verified_tls"] == bool(settings.ch_secure) + record["ready"] = ready + return record + + async def complete(record): + process = record["process"] + light = await _native_child_note(process, "light") + assert light["result"] == [{"score": 0.0, "complete": 7}] and light["hot_query_ids"] + drained = await _native_child_note(process, "drained") + assert drained["population"] == 0 and drained["orgs"] == {} + assert drained["inflight"] == {} and drained["retired"] == {} + assert drained["pool"]["in_use"] == 0 and drained["pool"]["waiters"] == 0 + assert await asyncio.wait_for(process.wait(), timeout=5) == 0 + record["light"], record["drained"] = light, drained + return drained["result"] + + try: + barrier = tmp_path / "initial.start" + first, peer = await start("first", barrier), await start("peer", barrier) + assert first["identity"]["pid"] != peer["identity"]["pid"] + assert first["ready"]["query_retries"] == peer["ready"]["query_retries"] + barrier.touch() + first_held = await _native_child_note(first["process"], "held") + peer_held = await _native_child_note(peer["process"], "held") + for record, held in ((first, first_held), (peer, peer_held)): + assert held["held"] == held["population"] == 2 + assert held["pool_in_use"] >= 1 and len(held["query_ids"]) == 1 + active = await _vector_running(state.direct, record["request_id"]) + assert {row["query_id"] for row in active} == set(held["query_ids"]) + assert first_held["held"] + peer_held["held"] == 4 # Two local C2 gates, no fleet lock. + immediately_before_stop = await _vector_running(state.direct, first["request_id"]) + assert {r["query_id"] for r in immediately_before_stop} == set(first_held["query_ids"]) + assert first["process"].returncode is None + assert _native_child_identity(first["process"]) == first["identity"] + signal.pidfd_send_signal(first["pidfd"], signal.SIGKILL) + assert await asyncio.wait_for(first["process"].wait(), timeout=5) == -signal.SIGKILL + after_reap = await _vector_running(state.direct, first["request_id"]) + # Restart promptly; waiting for old native retirement here would hide overlap. + restart_barrier = tmp_path / "restart.start" + replacement = await start("replacement", restart_barrier) + assert replacement["ready"]["query_retries"] == peer["ready"]["query_retries"] + restart_barrier.touch() + restarted_held = await _native_child_note(replacement["process"], "held") + assert restarted_held["held"] == restarted_held["population"] == 2 + peer_result = await complete(peer) + restarted_result = await complete(replacement) + assert restarted_result == peer_result and len(peer_result) == 1 + old_events = await _vector_remote_end( + state.direct, first["request_id"], first_held["query_ids"] + ) + new_events = await _vector_remote_end( + state.direct, replacement["request_id"], restarted_held["query_ids"] + ) + peer_events = await _vector_remote_end( + state.direct, peer["request_id"], peer_held["query_ids"] + ) + for record in (peer, replacement): + await _vector_remote_end(state.direct, record["request_id"] + "-light", []) + starts = [r for r in old_events if r["event"] == "QueryStart"] + ends = [r for r in old_events if r["event"] != "QueryStart"] + new_starts = [r for r in new_events if r["event"] == "QueryStart"] + new_ends = [r for r in new_events if r["event"] != "QueryStart"] + peer_starts = [r for r in peer_events if r["event"] == "QueryStart"] + peer_ends = [r for r in peer_events if r["event"] != "QueryStart"] + assert len(starts) == len(ends) == len(new_starts) == len(new_ends) == 1 + assert len(peer_starts) == len(peer_ends) == 1 + initial_overlap_us = min(ends[0]["at_us"], peer_ends[0]["at_us"]) - max( + starts[0]["at_us"], peer_starts[0]["at_us"] + ) + assert initial_overlap_us > 0, "two owned native clients did not actually overlap" + overlap_us = max( + 0, + min(ends[0]["at_us"], new_ends[0]["at_us"]) + - max(starts[0]["at_us"], new_starts[0]["at_us"]), + ) + _vector_drained(state) + assert await _vector_call( + state.client, + state.light, + parent_reuse_id, + "SELECT cosineDistance([1.0, 0.0], [1.0, 0.0]) AS score, 7 AS complete", + ) == [{"score": 0.0, "complete": 7}] + await _vector_remote_end(state.direct, parent_reuse_id, []) + _vector_drained(state) + print( + "vector-native-process-restart " + + json.dumps( + { + "local_capacity_per_child": 2, + "simultaneous_local_held_sum": 4, + "first_identity": first["identity"], + "peer_identity": peer["identity"], + "replacement_identity": replacement["identity"], + "actual_core_sha256": expected_core, + "configuration": configuration, + "killed_child_returncode": first["process"].returncode, + "old_native_immediately_before_stop": immediately_before_stop, + "old_native_after_reap": after_reap, + "old_native_terminal": ends, + "replacement_native_start": new_starts, + "initial_two_process_native_overlap_us": initial_overlap_us, + "old_native_overlap_after_restart_us": overlap_us, + "peer_light": peer["light"], + "replacement_light": replacement["light"], + "complete_result": restarted_result, + "all_owned_processes_reaped": True, + "client_processes_are_not_web_workers": True, + }, + sort_keys=True, + ) + ) + finally: + failure = sys.exception() + cleanup_errors = [] + for record in reversed(owned): + process, descriptor = record["process"], record["pidfd"] + try: + if process.returncode is None and descriptor is not None: + try: + assert _native_child_identity(process) == record["identity"] + signal.pidfd_send_signal(descriptor, signal.SIGKILL) + except (ProcessLookupError, FileNotFoundError): + pass + await asyncio.wait_for(process.wait(), timeout=5 if descriptor is not None else 20) + except Exception as error: # noqa: BLE001 — attempt every owned child/native cleanup + cleanup_errors.append(type(error).__name__) + finally: + if descriptor is not None: + os.close(descriptor) + record["log"].close() + if failure is not None: + failure.add_note(redact(record["log_path"].read_text(errors="replace")[-4000:])) + for request_id in requests: + try: + await _vector_cleanup(state.direct, request_id, failure) + except Exception as error: # noqa: BLE001 — remaining exact owned queries still need cleanup + cleanup_errors.append(type(error).__name__) + if cleanup_errors: + if failure is not None: + failure.add_note("own process cleanup: " + ",".join(cleanup_errors)) + else: + raise AssertionError("own process cleanup: " + ",".join(cleanup_errors)) diff --git a/tests/unit/test_http_error_request_id.py b/tests/unit/test_http_error_request_id.py new file mode 100644 index 0000000000..1e5049b1b0 --- /dev/null +++ b/tests/unit/test_http_error_request_id.py @@ -0,0 +1,176 @@ +"""The actual HTTP response keeps the id bound to logs and native query ids.""" + +from __future__ import annotations + +from typing import Any +from uuid import UUID + +import pytest +from fastapi import FastAPI, HTTPException +from fastapi.responses import JSONResponse +from fastapi.testclient import TestClient +from pydantic import BaseModel + +from server.core.ch import current_query_tag +from server.core.ch_request_cost import request_query_id +from server.core.errors import VectorAdmissionError, error_help_link, register_exception_handler +from server.core.logging_config import request_id_var +from server.main import _response_headers +from server.observability import capture_consent +from server.operations.base import Operation, Role +from server.transport import http + + +class _Input(BaseModel): + org: str = "-" + namespace: str = "-" + + +class _Output(BaseModel): + ok: bool = True + + +@pytest.mark.parametrize("caller_id", ["caller-123._", "invalid id", None]) +@pytest.mark.parametrize( + ("failure", "status", "code", "retry_after"), + [ + ("vector", 429, "VECTOR_CAPACITY_REFUSED", "1"), + ("http", 409, "CONFLICT", None), + ("internal", 500, "INTERNAL_ERROR", None), + ("success", 200, None, None), + ], +) +def test_response_keeps_the_bound_request_id( + monkeypatch: pytest.MonkeyPatch, + caller_id: str | None, + failure: str, + status: int, + code: str | None, + retry_after: str | None, +) -> None: + bound: dict[str, str] = {} + + async def handler(_: _Input) -> _Output: + bound["id"] = request_id_var.get() + bound["tag"] = current_query_tag() + query_id = request_query_id(bound["tag"]) + assert query_id is not None + bound["query_id"] = query_id + if failure == "vector": + raise VectorAdmissionError("row budget exhausted", reason="rows") + if failure == "http": + raise HTTPException(409, detail={"code": "CONFLICT", "message": "conflict"}) + if failure == "internal": + raise RuntimeError("private failure") + return _Output() + + async def no_post_handler_rows(*args: Any, **kwargs: Any) -> None: + pass + + async def capture_off(org: str, namespace: str) -> str: + return "off" + + op = Operation( + name="probe.request_id", + input_model=_Input, + output_model=_Output, + handler=handler, + idempotent=False, + required_role=Role.VIEWER, + ) + monkeypatch.setattr(http, "_emit_event", lambda *a, **k: None) + monkeypatch.setattr(http, "_record_failure", lambda *a, **k: None) + monkeypatch.setattr(http, "post_handler_rows", no_post_handler_rows) + monkeypatch.setattr(capture_consent, "resolve", capture_off) + monkeypatch.setattr(http.settings, "auth_required", False) + monkeypatch.setattr(type(http.settings), "auth_actually_required", lambda self: False) + app = FastAPI() + app.middleware("http")(_response_headers) + register_exception_handler(app) + app.add_api_route(op.http_path, http._make_endpoint(op), methods=["POST"]) + headers = {"X-Request-Id": caller_id} if caller_id is not None else {} + with TestClient(app, raise_server_exceptions=False) as client: + response = client.post(op.http_path, json={}, headers=headers) + + assert response.status_code == status + response_id = response.headers["X-Request-Id"] + assert response_id == bound["id"] + if caller_id == "caller-123._": + assert response_id == caller_id + else: + assert str(UUID(response_id)) == response_id + assert response_id != caller_id + assert bound["tag"] == f"{op.name} {response_id}" + assert bound["query_id"].startswith(f"{response_id}:1:") + assert response.headers.get("Retry-After") == retry_after + if failure == "success": + assert response.json()["ok"] is True + return + detail = response.json()["detail"] + assert detail["code"] == code + assert response.headers["Link"] == error_help_link(str(code)) + if failure == "vector": + assert detail["reason"] == "rows" + assert detail["retry_after_s"] == 1 + assert detail["retryable"] is True + if failure == "internal": + assert detail["request_id"] == response_id + assert detail["message"] == f"internal error; request_id={response_id}" + assert detail["retryable"] is False + assert "private failure" not in response.text + + +@pytest.mark.parametrize("caller_id", [None, "invalid id"]) +@pytest.mark.parametrize("status", [200, 409]) +def test_me_response_keeps_its_own_bound_id( + monkeypatch: pytest.MonkeyPatch, caller_id: str | None, status: int +) -> None: + bound: dict[str, str] = {} + ledger: list[dict[str, Any]] = [] + monkeypatch.setattr(http, "emit_agent_event", lambda **kw: ledger.append(kw)) + app = FastAPI() + register_exception_handler(app) + + @app.get("/v1/me/probe") + async def probe() -> dict[str, bool]: + bound["id"] = request_id_var.get() + bound["tag"] = current_query_tag() + query_id = request_query_id(bound["tag"]) + assert query_id is not None + bound["query_id"] = query_id + if status == 409: + raise HTTPException(409, detail={"code": "CONFLICT", "message": "conflict"}) + return {"ok": True} + + # The production order: measured /me wraps the common response middleware. + app.middleware("http")(_response_headers) + app.middleware("http")(http.me_route_measured) + headers = {"X-Request-Id": caller_id} if caller_id is not None else {} + with TestClient(app) as client: + response = client.get("/v1/me/probe", headers=headers) + assert response.status_code == status + request_id = response.headers["X-Request-Id"] + assert str(UUID(request_id)) == request_id + assert request_id == bound["id"] == ledger[0]["request_id"] + assert bound["tag"] == f"me.probe {request_id}" + assert bound["query_id"].startswith(f"{request_id}:1:") + assert ledger[0]["success"] is (status == 200) + if status == 409: + assert response.json()["detail"]["code"] == "CONFLICT" + assert response.headers["Link"] == error_help_link("CONFLICT") + else: + assert response.json() == {"ok": True} + + +def test_existing_response_request_id_is_preserved() -> None: + app = FastAPI() + app.middleware("http")(_response_headers) + + @app.get("/probe") + async def probe() -> JSONResponse: + return JSONResponse({"ok": True}, headers={"X-Request-Id": "route-bound"}) + + with TestClient(app) as client: + response = client.get("/probe") + assert response.status_code == 200 + assert response.headers["X-Request-Id"] == "route-bound" From 39552490aff3180f8a339ed59332eb5b39abc589 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Thu, 19 Nov 2026 00:12:50 +0000 Subject: [PATCH 19/28] Correlate mixed-build request timings with native work Record client setup, response phases and bounded request timing evidence so slower samples can be diagnosed. Keep cold requests, workload assertions and the original timers. Signed-off-by: zlareb1 --- scripts/gates/mixed_version.py | 117 +++++++++++++- tests/unit/test_mixed_version_gate.py | 210 +++++++++++++++++++++++++- 2 files changed, 318 insertions(+), 9 deletions(-) diff --git a/scripts/gates/mixed_version.py b/scripts/gates/mixed_version.py index 0dc71fe61a..483c2c7709 100644 --- a/scripts/gates/mixed_version.py +++ b/scripts/gates/mixed_version.py @@ -344,17 +344,47 @@ def call( observation: dict[str, Any] | None = None, ) -> dict[str, Any]: started = time.perf_counter() if observation is not None else 0.0 + if observation is not None: + observation["started_unix_ns"] = time.time_ns() + observation["started_monotonic_s"] = started try: - r = httpx.post( - f"{self.url}/v1/{op}", - json=body, - headers={"Authorization": f"Bearer {token}"}, - timeout=60, - ) + if observation is None: + r = httpx.post( + f"{self.url}/v1/{op}", + json=body, + headers={"Authorization": f"Bearer {token}"}, + timeout=60, + ) + else: + connection: dict[str, float] = {} + observation["connection_events_ms"] = connection + + def trace(name: str, _info: dict[str, Any]) -> None: + if name in ( + "connection.connect_tcp.started", + "connection.connect_tcp.complete", + "connection.connect_tcp.failed", + "http11.send_request_body.complete", + "http11.receive_response_headers.complete", + ): + connection[name] = (time.perf_counter() - started) * 1000 + + setup = time.perf_counter() + # Match httpx.post: a fresh default client, closed inside elapsed_ms. + with httpx.Client(timeout=60) as client: + observation["client_setup_ms"] = (time.perf_counter() - setup) * 1000 + r = client.post( + f"{self.url}/v1/{op}", + json=body, + headers={"Authorization": f"Bearer {token}"}, + extensions={"trace": trace}, + ) except httpx.HTTPError as exc: if observation is not None: observation.update( - elapsed_ms=(time.perf_counter() - started) * 1000, error=type(exc).__name__ + elapsed_ms=(time.perf_counter() - started) * 1000, + error=type(exc).__name__, + finished_unix_ns=time.time_ns(), ) raise if observation is not None: @@ -362,7 +392,20 @@ def call( elapsed_ms=(time.perf_counter() - started) * 1000, status=r.status_code, error="" if r.status_code == 200 else f"{r.status_code}:{_error_code(r)}", + finished_unix_ns=time.time_ns(), ) + try: + observation["httpx_elapsed_ms"] = r.elapsed.total_seconds() * 1000 + request_id = r.headers.get("X-Request-Id", "") + if re.fullmatch(r"[A-Za-z0-9._-]{1,64}", request_id): + observation["request_id"] = request_id + for name, duration in re.findall( + r"(?:^|,\s*)(app|ch);dur=([0-9]+(?:\.[0-9]+)?)", + r.headers.get("Server-Timing", ""), + ): + observation[f"server_{name}_ms"] = float(duration) + except Exception as error: # noqa: BLE001 — timing metadata cannot change the response + observation["timing_error"] = type(error).__name__ if r.status_code != 200: raise AssertionError(f"{self.name} {op}: {r.status_code} {r.text[:400]}") return dict(r.json()) @@ -642,6 +685,7 @@ def search(key: str, row: dict[str, Any], phase: str) -> None: arm[0].stop() finally: arm[1].stop() + timing_evidence = self.latency_timing_evidence(samples, fixtures) print( "latency-acceptance " + json.dumps( @@ -686,6 +730,8 @@ def search(key: str, row: dict[str, Any], phase: str) -> None: for role in ("main", "light") }, "attempts": samples, + "timing_evidence": timing_evidence, + "ack_to_first_search_samples_ms": visibility, "normal_load_errors": { "attempted": len(samples), "errors": { @@ -722,6 +768,63 @@ def search(key: str, row: dict[str, Any], phase: str) -> None: flush=True, ) + def latency_timing_evidence( + self, samples: list[dict[str, Any]], fixtures: list[Any] + ) -> dict[str, Any]: + """Read bounded timing witnesses after the measured server groups stop.""" + timing_evidence: dict[str, Any] = {} + try: + ids = list(dict.fromkeys(s["request_id"] for s in samples if s.get("request_id"))) + timing_evidence["request_count"] = len(ids) + if len(ids) > 120: + timing_evidence["error"] = "request_limit" + elif ids: + params = { + "ids": "[" + ", ".join(map(sql_string, ids)) + "]", + "orgs": "[" + ", ".join(sql_string(f[1]["org"]) for f in fixtures) + "]", + } + bounds = { + "max_execution_time": 5, + "max_rows_to_read": 1_000_000, + "max_memory_usage": 64_000_000, + "max_result_rows": 121, + "result_overflow_mode": "throw", + } + sql(self.ch_port, "SYSTEM FLUSH LOGS", settings=bounds) + statements = { + "requests": ( + "SELECT request_id, operation, toString(ts) AS emitted_at, " + "latency_ms, instance, instance_label, build_commit, " + "args['out_ch_elapsed_ms'] AS ch_elapsed_ms, " + "args['out_ch_statements'] AS ch_statements " + "FROM loom.agent_events WHERE kind = 'request' " + "AND org IN {orgs:Array(String)} AND namespace = 'latency' " + "AND request_id IN {ids:Array(String)} " + "ORDER BY request_id LIMIT 121 FORMAT JSONEachRow" + ), + "native": ( + "SELECT splitByChar(' ', log_comment)[2] AS request_id, " + "count() AS statements, " + "toString(min(query_start_time_microseconds)) AS first_start, " + "toString(max(event_time_microseconds)) AS last_end, " + "sum(query_duration_ms) AS summed_ms, max(query_duration_ms) AS max_ms, " + "argMax(query_id, query_duration_ms) AS slowest_query_id, " + "sum(read_rows) AS read_rows, sum(read_bytes) AS read_bytes, " + "max(memory_usage) AS max_memory_bytes " + "FROM system.query_log WHERE is_initial_query " + "AND type IN ('QueryFinish', 'ExceptionBeforeStart', " + "'ExceptionWhileProcessing') " + "AND request_id IN {ids:Array(String)} " + "GROUP BY request_id ORDER BY request_id LIMIT 121 FORMAT JSONEachRow" + ), + } + for name, statement in statements.items(): + raw = sql(self.ch_port, statement, params, bounds) + timing_evidence[name] = [json.loads(line) for line in raw.splitlines()] + except Exception as error: # noqa: BLE001 — evidence cannot replace a workload failure + timing_evidence["error"] = type(error).__name__ + return timing_evidence + def latency_workers(self, srv: Server) -> dict[str, Any]: assert srv.proc is not None and srv.web_workers == 2 and srv.workers == "off" seen = {} diff --git a/tests/unit/test_mixed_version_gate.py b/tests/unit/test_mixed_version_gate.py index e2690a63eb..bb35e875ce 100644 --- a/tests/unit/test_mixed_version_gate.py +++ b/tests/unit/test_mixed_version_gate.py @@ -204,6 +204,7 @@ def stop(self): def test_latency_transport_retains_lag_refusal_without_retry( monkeypatch: pytest.MonkeyPatch, tmp_path ) -> None: + from datetime import timedelta from types import SimpleNamespace calls = [] @@ -214,9 +215,26 @@ def post(url, **kwargs): status_code=503, text="projection behind", json=lambda: {"detail": {"code": "PROJECTION_BEHIND"}}, + elapsed=timedelta(0), + headers={"X-Request-Id": "fixture-refusal"}, ) - monkeypatch.setattr(mixed_version.httpx, "post", post) + class Client: + def __init__(self, *, timeout): + self.timeout = timeout + + def __enter__(self): + return self + + def __exit__(self, *_args): + pass + + def post(self, url, *, extensions, **kwargs): + return post(url, timeout=self.timeout, **kwargs) + + monkeypatch.setattr(mixed_version.httpx, "Client", Client) + monkeypatch.setattr(mixed_version, "_free_port", lambda: 18080) + monkeypatch.setattr(mixed_version.time, "time_ns", lambda: 0) monkeypatch.setattr(mixed_version.time, "perf_counter", lambda: 1.0) srv = mixed_version.Server("latency-test", tmp_path, 18123, tmp_path) observation = {} @@ -225,4 +243,192 @@ def post(url, **kwargs): assert len(calls) == 1 assert calls[0][1]["headers"] == {"Authorization": "Bearer token"} assert calls[0][1]["timeout"] == 60 - assert observation == {"elapsed_ms": 0.0, "status": 503, "error": "503:PROJECTION_BEHIND"} + original = {key: observation[key] for key in ("elapsed_ms", "status", "error")} + assert original == {"elapsed_ms": 0.0, "status": 503, "error": "503:PROJECTION_BEHIND"} + assert {key: value for key, value in observation.items() if key not in original} == { + "started_unix_ns": 0, + "started_monotonic_s": 1.0, + "connection_events_ms": {}, + "client_setup_ms": 0.0, + "finished_unix_ns": 0, + "httpx_elapsed_ms": 0.0, + "request_id": "fixture-refusal", + } + + +@pytest.fixture +def latency_transport(monkeypatch, tmp_path): + from datetime import timedelta + from types import SimpleNamespace + + from tests.unit.clock_fake import ManualClock + + clock = ManualClock(t=10.0) + body = {"results": [{"memory_key": "k", "value": "complete private body"}]} + response = SimpleNamespace( + status_code=200, + text="complete private body", + json=lambda: body, + elapsed=timedelta(milliseconds=15), + headers={ + "X-Request-Id": "from-response", + "Server-Timing": 'app;dur=6, ch;dur=9;desc="concurrent statements"', + "Authorization": "private header", + }, + ) + state = SimpleNamespace( + clock=clock, body=body, response=response, failure=None, clients=[], calls=[], closed=[] + ) + + class Client: + def __init__(self, **options): + state.clients.append((self, options)) + clock.advance(0.002) + + def __enter__(self): + return self + + def __exit__(self, *_args): + clock.advance(0.011) + state.closed.append(self) + + def post(self, url, **kwargs): + state.calls.append((url, kwargs)) + trace = kwargs["extensions"]["trace"] + info = {"private": "private trace credentials/body/SQL/address"} + trace("private.unrecognized", info) + clock.advance(0.003) + trace("connection.connect_tcp.started", info) + clock.advance(0.005) + trace("connection.connect_tcp.complete", info) + trace("http11.send_request_body.complete", info) + clock.advance(0.007) + if state.failure is not None: + raise state.failure + trace("http11.receive_response_headers.complete", info) + return state.response + + monkeypatch.setattr(mixed_version.httpx, "Client", Client) + monkeypatch.setattr(mixed_version.time, "perf_counter", clock.monotonic) + monkeypatch.setattr(mixed_version.time, "time_ns", lambda: round(clock.t * 1e9)) + monkeypatch.setattr(mixed_version, "_free_port", lambda: 18080) + state.server = mixed_version.Server("latency-test", tmp_path, 18123, tmp_path) + return state + + +def test_observed_calls_keep_cold_clients_complete_results_and_closed_timing(latency_transport): + import json + + state = latency_transport + request = {"query": "complete request"} + samples = [{}, {}] + for sample in samples: + assert ( + state.server.call("fixture-token", "memory.search", request, observation=sample) + == state.body + ) + assert sample["elapsed_ms"] == pytest.approx(28) + assert sample["client_setup_ms"] == pytest.approx(2) + assert sample["httpx_elapsed_ms"] == 15 + assert sample["request_id"] == "from-response" + assert (sample["status"], sample["error"]) == (200, "") + assert (sample["server_app_ms"], sample["server_ch_ms"]) == (6, 9) + assert sample["finished_unix_ns"] - sample["started_unix_ns"] == 28_000_000 + assert sample["connection_events_ms"] == pytest.approx( + { + "connection.connect_tcp.started": 5, + "connection.connect_tcp.complete": 10, + "http11.send_request_body.complete": 10, + "http11.receive_response_headers.complete": 17, + } + ) + assert set(sample) == { + "elapsed_ms", + "client_setup_ms", + "httpx_elapsed_ms", + "request_id", + "status", + "error", + "server_app_ms", + "server_ch_ms", + "started_unix_ns", + "started_monotonic_s", + "finished_unix_ns", + "connection_events_ms", + } + assert "private" not in json.dumps(sample) + assert len(state.clients) == len(state.closed) == len(state.calls) == 2 + assert state.clients[0][0] is not state.clients[1][0] + assert state.closed == [client for client, _options in state.clients] + assert [options for _client, options in state.clients] == [{"timeout": 60}] * 2 + for url, kwargs in state.calls: + assert url == state.server.url + "/v1/memory.search" + assert kwargs["json"] is request + assert kwargs["headers"] == {"Authorization": "Bearer fixture-token"} + assert set(kwargs) == {"json", "headers", "extensions"} + + +def test_observed_transport_exception_keeps_identity_and_includes_client_close(latency_transport): + state = latency_transport + state.failure = mixed_version.httpx.ReadError("fixture transport error") + sample = {} + with pytest.raises(mixed_version.httpx.ReadError) as caught: + state.server.call("fixture-token", "memory.search", {}, observation=sample) + assert caught.value is state.failure + assert sample["elapsed_ms"] == pytest.approx(28) + assert sample["error"] == "ReadError" + assert len(state.calls) == len(state.clients) == len(state.closed) == 1 + assert state.closed[0] is state.clients[0][0] + + +@pytest.mark.parametrize("refused", [False, True], ids=["complete", "same-exception"]) +def test_unobserved_call_keeps_the_original_post_path(monkeypatch, latency_transport, refused): + state = latency_transport + failure = mixed_version.httpx.ReadError("original transport error") + calls = [] + + def post(url, **kwargs): + calls.append((url, kwargs)) + if refused: + raise failure + return state.response + + monkeypatch.setattr(mixed_version.httpx, "post", post) + request = {"memory_key": "k"} + if refused: + with pytest.raises(mixed_version.httpx.ReadError) as caught: + state.server.call("fixture-token", "memory.get", request) + assert caught.value is failure + else: + assert state.server.call("fixture-token", "memory.get", request) == state.body + assert state.clients == state.closed == [] + assert calls == [ + ( + state.server.url + "/v1/memory.get", + { + "json": request, + "headers": {"Authorization": "Bearer fixture-token"}, + "timeout": 60, + }, + ) + ] + + +@pytest.mark.parametrize("broken", [False, True], ids=["invalid-values", "unreadable-headers"]) +def test_malformed_timing_metadata_cannot_replace_the_complete_response(latency_transport, broken): + state = latency_transport + state.response.headers = ( + None + if broken + else { + "X-Request-Id": "invalid request id", + "Server-Timing": "app;dur=invalid, ch;dur=-1, private;dur=10", + } + ) + sample = {} + assert state.server.call("fixture-token", "memory.search", {}, observation=sample) == state.body + assert (sample["status"], sample["error"]) == (200, "") + assert sample["elapsed_ms"] == pytest.approx(28) + assert not {"request_id", "server_app_ms", "server_ch_ms"} & sample.keys() + assert sample.get("timing_error") == ("AttributeError" if broken else None) + assert len(state.closed) == len(state.calls) == 1 From 025c36de6019509d0323a87c421787bd972905e7 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Thu, 19 Nov 2026 00:36:24 +0000 Subject: [PATCH 20/28] Throw when vector reads exhaust native execution time Override inherited timeout break mode for attributed vector work so a shorter native deadline cannot return partial success. Retain scalar settings and tighter execution limits. Signed-off-by: zlareb1 --- server/core/vectors/admission.py | 1 + tests/unit/test_vector_read_admission.py | 56 ++++++++++++++++++++++++ 2 files changed, 57 insertions(+) diff --git a/server/core/vectors/admission.py b/server/core/vectors/admission.py index 933864e8ea..a4291be11c 100644 --- a/server/core/vectors/admission.py +++ b/server/core/vectors/admission.py @@ -233,6 +233,7 @@ def native_settings(out: dict[str, Any], *, timeout: float) -> dict[str, Any]: bound = min(previous, cap) if math.isfinite(previous) and previous > 0 else cap out[key] = int(bound) if bound.is_integer() else bound out["read_overflow_mode"] = "throw" + out["timeout_overflow_mode"] = "throw" return out diff --git a/tests/unit/test_vector_read_admission.py b/tests/unit/test_vector_read_admission.py index 91148452f5..1ed9087e13 100644 --- a/tests/unit/test_vector_read_admission.py +++ b/tests/unit/test_vector_read_admission.py @@ -444,3 +444,59 @@ def test_vector_concrete_error_hierarchy_preserves_wire_contracts(kind: str) -> "category": category, "retryable": retryable, } + + +@pytest.mark.parametrize( + ("source", "vector", "configured", "expected"), + [ + ("class", True, "break", "throw"), + ("statement", True, "break", "throw"), + ("class", True, "throw", "throw"), + ("class", False, "break", "break"), + ("statement", False, "break", "break"), + ], +) +@pytest.mark.asyncio +async def test_vector_native_timeout_refuses_partial_results( + monkeypatch: pytest.MonkeyPatch, source: str, vector: bool, configured: str, expected: str +) -> None: + import contextlib + import json + + from server.core.traffic_context import traffic_class_var + from server.core.vectors.admission import vector_work + + ch, driver = client(monkeypatch) + driver.release.set() + monkeypatch.setattr(settings, "ch_timeout_read_s", 0.0) + caps = {"max_execution_time": 0.05, "timeout_overflow_mode": configured} + monkeypatch.setattr(settings, "ch_settings_by_traffic_class", json.dumps({"interactive": caps})) + token = traffic_class_var.set(("interactive", "test") if source == "class" else ("", "")) + scope = vector_work("one-org", ["private"]) if vector else contextlib.nullcontext() + try: + with scope: + if source == "statement": + + async def raw_query(sql: str, **kwargs: Any) -> bytes: + driver.started += 1 + driver.settings.append(kwargs["settings"]) + return b"body" + + monkeypatch.setattr(driver, "raw_query", raw_query, raising=False) + assert ( + await ch.raw_bytes("SELECT number FROM numbers(10)", settings=caps) == b"body" + ) + else: + assert await ch.query_all("SELECT number FROM numbers(10)") == [] + finally: + traffic_class_var.reset(token) + assert driver.started == 1 + assert driver.active == 0 + native = driver.settings[0] + assert native["max_execution_time"] == 0.05 + assert native["timeout_overflow_mode"] == expected + if vector: + assert native["max_rows_to_read"] == settings.vector_scan_max_rows_to_read + assert native["read_overflow_mode"] == "throw" + else: + assert "max_rows_to_read" not in native From 0448043eeaeb8a2df7c0b29ab51ae16888379091 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Thu, 19 Nov 2026 01:26:30 +0000 Subject: [PATCH 21/28] Vector reads reject incomplete native results Native output, sorting, grouping, membership and join limits must refuse the whole attributed read. Preserve scalar profiles and configured limits, and propagate native set and result budget errors through optional readers. Signed-off-by: zlareb1 --- server/core/vectors/admission.py | 7 + .../integration/test_vector_native_budget.py | 153 ++++++++++++++++++ tests/unit/test_vector_read_admission.py | 131 +++++++++++++++ 3 files changed, 291 insertions(+) diff --git a/server/core/vectors/admission.py b/server/core/vectors/admission.py index a4291be11c..c6d331488c 100644 --- a/server/core/vectors/admission.py +++ b/server/core/vectors/admission.py @@ -234,6 +234,11 @@ def native_settings(out: dict[str, Any], *, timeout: float) -> dict[str, Any]: out[key] = int(bound) if bound.is_integer() else bound out["read_overflow_mode"] = "throw" out["timeout_overflow_mode"] = "throw" + out["result_overflow_mode"] = "throw" + out["sort_overflow_mode"] = "throw" + out["group_by_overflow_mode"] = "throw" + out["set_overflow_mode"] = "throw" + out["join_overflow_mode"] = "throw" return out @@ -246,6 +251,8 @@ def raise_native(error: Exception) -> None: reasons: dict[int, VectorCapacityReason] = { 158: "rows", 307: "rows", + 191: "rows", + 396: "rows", 241: "memory", 159: "deadline", } diff --git a/tests/integration/test_vector_native_budget.py b/tests/integration/test_vector_native_budget.py index 90c7b2fcac..e063008aa0 100644 --- a/tests/integration/test_vector_native_budget.py +++ b/tests/integration/test_vector_native_budget.py @@ -213,3 +213,156 @@ def test_native_api_row_refusal_preserves_committed_heads( assert scalar.status_code == 200 and scalar.json()["found"] is True assert scalar.json()["value"] == _budget_value(0) assert scalar.json()["version"] == version_before + + +_NATIVE_RANKED_ROWS = ( + "SELECT number AS n, " + "cosineDistance([toFloat64(number + 1), 1.0], [1.0, 0.0]) AS distance " + "FROM numbers(64) ORDER BY distance, n" +) + + +@pytest.mark.parametrize( + ("sql", "profile", "complete_count", "native_code"), + [ + pytest.param( + _NATIVE_RANKED_ROWS, + {"max_result_rows": 8, "result_overflow_mode": "break"}, + 64, + 396, + id="result-rows", + ), + pytest.param( + _NATIVE_RANKED_ROWS, + {"max_result_bytes": 128, "result_overflow_mode": "break"}, + 64, + 396, + id="result-bytes", + ), + pytest.param( + _NATIVE_RANKED_ROWS, + {"max_rows_to_sort": 8, "sort_overflow_mode": "break"}, + 64, + 396, + id="sort", + ), + pytest.param( + "SELECT number % 32 AS n, count() AS copies, " + "min(cosineDistance([toFloat64(number + 1), 1.0], [1.0, 0.0])) AS distance " + "FROM numbers(64) GROUP BY n ORDER BY distance, n", + {"max_rows_to_group_by": 8, "group_by_overflow_mode": "any"}, + 32, + 158, + id="group-by-any", + ), + pytest.param( + "SELECT number AS n, " + "cosineDistance([toFloat64(number + 1), 1.0], [1.0, 0.0]) AS distance " + "FROM numbers(32) WHERE number IN (SELECT number FROM numbers(32)) " + "ORDER BY distance, n", + {"max_rows_in_set": 8, "set_overflow_mode": "break"}, + 32, + 191, + id="in-set", + ), + pytest.param( + "SELECT l.n AS n, r.n + 1 AS matched, " + "cosineDistance([toFloat64(l.n + 1), 1.0], [1.0, 0.0]) AS distance " + "FROM (SELECT number AS n FROM numbers(32)) AS l " + "ALL INNER JOIN (SELECT number AS n FROM numbers(32)) AS r ON l.n = r.n " + "ORDER BY distance, n SETTINGS join_algorithm = 'hash'", + {"max_rows_in_join": 8, "join_overflow_mode": "break"}, + 32, + 191, + id="hash-join", + ), + ], +) +async def test_native_overflow_refuses_partial_vector_results_and_allows_complete_reuse( + monkeypatch: pytest.MonkeyPatch, + sql: str, + profile: dict[str, Any], + complete_count: int, + native_code: int, +) -> None: + import json + + from server.core.traffic_context import traffic_class_var + + monkeypatch.setattr(settings, "vector_scan_max_concurrent_per_org", 2) + monkeypatch.setattr(settings, "vector_scan_max_waiters_per_org", 2) + monkeypatch.setattr(settings, "ch_pool_maxsize", 2) + monkeypatch.setattr(settings, "ch_pool_wait_ms", 500) + common = {"max_block_size": 1, "max_threads": 1, "use_query_cache": 0} + monkeypatch.setattr( + settings, "ch_settings_by_traffic_class", json.dumps({"interactive": common}) + ) + org = unique_org("vector-native-overflow") + client = ch.CHClient() + gate = admission._admission + population_before = gate.population + pool_before = ch.pool_reading() + work_before = admission.current_work() + traffic_token = traffic_class_var.set(("", "")) + + def assert_drained() -> None: + assert gate.population == population_before + assert org not in gate.orgs + assert ch.pool_reading() == pool_before + assert client._inflight == {} + assert client._retired == {} + + try: + with ch.query_tag("itest.vector-native-overflow"), admission.vector_work("", (), 0): + await client.connect() + traffic_class_var.set(("interactive", "test")) + complete = await client.query_all(sql) + assert len(complete) == complete_count + assert [row["n"] for row in complete] == list(reversed(range(complete_count))) + assert_drained() + with admission.vector_work(org, "native-overflow", weight=2): + assert await client.query_all(sql) == complete + assert_drained() + + capped = {**common, **profile} + monkeypatch.setattr( + settings, "ch_settings_by_traffic_class", json.dumps({"interactive": capped}) + ) + scalar_settings = ch.statement_settings(sql) + assert all(scalar_settings[key] == value for key, value in capped.items()) + partial = await client.query_all(sql) + assert len(partial) < len(complete) + assert_drained() + + with admission.vector_work(org, "native-overflow", weight=2): + vector_settings = ch.statement_settings(sql) + result: object = None + with pytest.raises(VectorAdmissionError) as caught: + result = await client.query_all(sql) + assert result is None + assert type(caught.value) is VectorAdmissionError + assert caught.value.code == "VECTOR_CAPACITY_REFUSED" + assert caught.value.status_code == 429 + assert caught.value.extra == {"reason": "rows", "retry_after_s": 1} + assert caught.value.headers() == {"Retry-After": "1"} + assert isinstance(caught.value.__cause__, DatabaseError) + assert error_code(caught.value.__cause__) == native_code + for key, value in capped.items(): + expected = "throw" if key.endswith("_overflow_mode") else value + assert vector_settings[key] == expected + assert_drained() + + monkeypatch.setattr( + settings, "ch_settings_by_traffic_class", json.dumps({"interactive": common}) + ) + with admission.vector_work(org, "native-overflow", weight=2): + assert await client.query_all(sql) == complete + assert admission.current_work() is None + assert_drained() + finally: + try: + await client.disconnect() + finally: + traffic_class_var.reset(traffic_token) + assert admission.current_work() == work_before + assert_drained() diff --git a/tests/unit/test_vector_read_admission.py b/tests/unit/test_vector_read_admission.py index 1ed9087e13..abd8759708 100644 --- a/tests/unit/test_vector_read_admission.py +++ b/tests/unit/test_vector_read_admission.py @@ -500,3 +500,134 @@ async def raw_query(sql: str, **kwargs: Any) -> bytes: assert native["read_overflow_mode"] == "throw" else: assert "max_rows_to_read" not in native + + +@pytest.mark.parametrize( + ("mode", "configured", "limits"), + [ + ("result_overflow_mode", "break", {"max_result_rows": 1, "max_result_bytes": 64}), + ("sort_overflow_mode", "break", {"max_rows_to_sort": 1, "max_bytes_to_sort": 64}), + ("group_by_overflow_mode", "any", {"max_rows_to_group_by": 1}), + ("set_overflow_mode", "break", {"max_rows_in_set": 1, "max_bytes_in_set": 64}), + ("join_overflow_mode", "break", {"max_rows_in_join": 1, "max_bytes_in_join": 64}), + ], +) +@pytest.mark.parametrize( + ("source", "vector", "already_throws"), + [ + ("class", True, False), + ("statement", True, False), + ("class", True, True), + ("class", False, False), + ("statement", False, False), + ], +) +@pytest.mark.asyncio +async def test_vector_native_partial_modes_throw_without_relaxing_limits( + monkeypatch: pytest.MonkeyPatch, + mode: str, + configured: str, + limits: dict[str, int], + source: str, + vector: bool, + already_throws: bool, +) -> None: + import contextlib + import json + + from server.core.traffic_context import traffic_class_var + from server.core.vectors.admission import vector_work + + ch, driver = client(monkeypatch) + driver.release.set() + configured = "throw" if already_throws else configured + caps = {**limits, mode: configured} + monkeypatch.setattr(settings, "ch_settings_by_traffic_class", json.dumps({"interactive": caps})) + token = traffic_class_var.set(("interactive", "test") if source == "class" else ("", "")) + try: + with vector_work("one-org", ["private"]) if vector else contextlib.nullcontext(): + if source == "statement": + + async def raw_query(sql: str, **kwargs: Any) -> bytes: + driver.started += 1 + driver.settings.append(kwargs["settings"]) + return b"body" + + monkeypatch.setattr(driver, "raw_query", raw_query, raising=False) + assert ( + await ch.raw_bytes("SELECT number FROM numbers(10)", settings=caps) == b"body" + ) + else: + assert await ch.query_all("SELECT number FROM numbers(10)") == [] + finally: + traffic_class_var.reset(token) + assert driver.started == 1 and driver.active == 0 + native = driver.settings[0] + assert native[mode] == ("throw" if vector else configured) + assert {key: native[key] for key in limits} == limits + if vector: + assert native["read_overflow_mode"] == native["timeout_overflow_mode"] == "throw" + else: + assert "max_rows_to_read" not in native + + +@pytest.mark.parametrize("code", [191, 396]) +@pytest.mark.asyncio +async def test_entity_boost_native_set_or_result_limit_is_a_complete_failure( + monkeypatch: pytest.MonkeyPatch, code: int +) -> None: + from server.core.errors import VectorCapacityError + + ch, driver = client(monkeypatch) + driver.release.set() + driver.failure = NativeBudgetError(code) + monkeypatch.setattr(ann_entity_boosts, "get_client", lambda: ch) + with pytest.raises(VectorCapacityError) as caught: + await ann_entity_boosts._ann_entity_boosts( + org="one-org", + namespaces=["private"], + agent="a", + user_id="u", + query_entities=["entity"], + embedder=Embedder(), + ) + assert caught.value.__cause__ is driver.failure + assert caught.value.extra["reason"] == "rows" + assert caught.value.extra["retry_after_s"] == 1 + assert driver.started == 1 and driver.active == 0 + + +@pytest.mark.parametrize("code", [191, 396]) +@pytest.mark.asyncio +async def test_scalar_set_or_result_limit_retains_original_native_error( + monkeypatch: pytest.MonkeyPatch, code: int +) -> None: + ch, driver = client(monkeypatch) + driver.release.set() + driver.failure = NativeBudgetError(code) + with pytest.raises(NativeBudgetError) as caught: + await ch.query_all("SELECT 1") + assert caught.value is driver.failure + assert driver.started == 1 and driver.active == 0 + + +@pytest.mark.asyncio +async def test_entity_boost_unrelated_native_error_retains_optional_fallback( + monkeypatch: pytest.MonkeyPatch, +) -> None: + ch, driver = client(monkeypatch) + driver.release.set() + driver.failure = NativeBudgetError(60) + monkeypatch.setattr(ann_entity_boosts, "get_client", lambda: ch) + assert ( + await ann_entity_boosts._ann_entity_boosts( + org="one-org", + namespaces=["private"], + agent="a", + user_id="u", + query_entities=["entity"], + embedder=Embedder(), + ) + == {} + ) + assert driver.started == 1 and driver.active == 0 From 0af6ffe6ae0b4d9096b3ace932671d97234df728 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Thu, 19 Nov 2026 01:28:50 +0000 Subject: [PATCH 22/28] Record request stages in mixed-build timing checks Attribute delays before and inside handlers with bounded numeric metadata from both source arms. Preserve requests, security decisions, native limits and closed-client timings. Signed-off-by: zlareb1 --- scripts/gates/_latency_observer.py | 181 +++++++++++++++ scripts/gates/mixed_version.py | 20 +- tests/unit/test_mixed_version_gate.py | 311 ++++++++++++++++++++++++++ 3 files changed, 511 insertions(+), 1 deletion(-) create mode 100644 scripts/gates/_latency_observer.py diff --git a/scripts/gates/_latency_observer.py b/scripts/gates/_latency_observer.py new file mode 100644 index 0000000000..a5aa61fd9d --- /dev/null +++ b/scripts/gates/_latency_observer.py @@ -0,0 +1,181 @@ +"""Numeric HTTP-stage observations for the mixed-version latency fixture only.""" + +from __future__ import annotations + +import hashlib +import inspect +import json +import math +import os +import time +from contextvars import ContextVar +from functools import wraps +from importlib import import_module +from importlib.util import find_spec +from pathlib import Path +from typing import Any + +HEADER = "X-Loom-Fixture-Timing" +_STAGES = ( + ("_begin_request", "bind"), + ("_extract_auth", "auth"), + ("_read_body", "body"), + ("_authorize", "authorize"), + ("_bind_context_and_policy", "policy"), + ("_rate_limit", "rate"), + ("_idempotency_gate", "idempotency"), + ("_start_telemetry", "telemetry"), + ("_run_handler", "handler"), + ("_finish_success", "finish"), +) +_PATHS = {"http_sha256": "server/transport/http.py", "main_sha256": "server/main.py"} +_FIELDS = { + "pid", + "asgi_unix_ns", + "asgi_monotonic_s", + "body_delivered_ms", + "response_start_ms", + "endpoint_start_ms", + *(name + suffix for _, name in _STAGES for suffix in ("_start_ms", "_duration_ms")), +} +_current: ContextVar[dict[str, Any] | None] = ContextVar("mixed_latency_stages", default=None) + + +def source_hashes(tree: Path) -> dict[str, str]: + return { + key: hashlib.sha256((tree / path).read_bytes()).hexdigest() for key, path in _PATHS.items() + } + + +def _timed(name: str, function: Any) -> Any: + def finish(record: dict[str, Any], started: float) -> None: + record[name + "_start_ms"] = (started - record["asgi_monotonic_s"]) * 1000 + record[name + "_duration_ms"] = (time.perf_counter() - started) * 1000 + if name == "bind": + record["endpoint_start_ms"] = record[name + "_start_ms"] + + if inspect.iscoroutinefunction(function): + + @wraps(function) + async def asynchronous(*args: Any, **kwargs: Any) -> Any: + record = _current.get() + if record is None: + return await function(*args, **kwargs) + started = time.perf_counter() + try: + return await function(*args, **kwargs) + finally: + finish(record, started) + + return asynchronous + + @wraps(function) + def synchronous(*args: Any, **kwargs: Any) -> Any: + record = _current.get() + if record is None: + return function(*args, **kwargs) + started = time.perf_counter() + try: + return function(*args, **kwargs) + finally: + finish(record, started) + + return synchronous + + +class _ObservedApp: + def __init__(self, app: Any, sources: dict[str, str]) -> None: + self.app, self.sources = app, sources + + async def __call__(self, scope: Any, receive: Any, send: Any) -> Any: + if scope["type"] != "http": + return await self.app(scope, receive, send) + record = { + **self.sources, + "pid": os.getpid(), + "asgi_unix_ns": time.time_ns(), + "asgi_monotonic_s": time.perf_counter(), + } + token = _current.set(record) + + async def received() -> Any: + message = await receive() + if message["type"] == "http.request" and not message.get("more_body", False): + record["body_delivered_ms"] = ( + time.perf_counter() - record["asgi_monotonic_s"] + ) * 1000 + return message + + async def sent(message: Any) -> None: + if message["type"] == "http.response.start": + record["response_start_ms"] = ( + time.perf_counter() - record["asgi_monotonic_s"] + ) * 1000 + try: + encoded = json.dumps(record, allow_nan=False, separators=(",", ":")).encode( + "ascii" + ) + if len(encoded) <= 4096: + message = { + **message, + "headers": [ + *message.get("headers", []), + (HEADER.lower().encode(), encoded), + ], + } + except Exception: # noqa: BLE001 — timing cannot replace a response + pass + await send(message) + + try: + return await self.app(scope, received, sent) + finally: + _current.reset(token) + + +def create_app() -> _ObservedApp: + tree = Path.cwd().resolve() + # The helper directory may be on sys.path; server must still come from this arm. + for module, relative in ( + ("server", "server/__init__.py"), + ("server.transport.http", _PATHS["http_sha256"]), + ("server.main", _PATHS["main_sha256"]), + ): + spec = find_spec(module) + if spec is None or spec.origin is None or Path(spec.origin).resolve() != tree / relative: + raise RuntimeError("latency observer resolved a server outside its arm") + http, main = import_module("server.transport.http"), import_module("server.main") + if ( + Path(http.__file__).resolve() != tree / _PATHS["http_sha256"] + or Path(main.__file__).resolve() != tree / _PATHS["main_sha256"] + ): + raise RuntimeError("latency observer imported a server outside its arm") + for attribute, name in _STAGES: + setattr(http, attribute, _timed(name, getattr(http, attribute))) + return _ObservedApp(main.app, source_hashes(tree)) + + +def parse_metadata(raw: str, expected: dict[str, str]) -> dict[str, Any]: + if len(raw) > 4096: + raise ValueError("latency stage metadata exceeded its bound") + record = json.loads(raw) + if not isinstance(record, dict) or not record.keys() <= _FIELDS | _PATHS.keys(): + raise ValueError("latency stage metadata has unexpected fields") + if any(record.get(key) != expected[key] for key in _PATHS): + raise ValueError("latency stage metadata has the wrong source") + for key, value in record.items(): + if key in _PATHS: + continue + if ( + isinstance(value, bool) + or not isinstance(value, int | float) + or not math.isfinite(value) + or value < 0 + ): + raise ValueError("latency stage metadata has an invalid number") + required = ("pid", "asgi_unix_ns", "asgi_monotonic_s", "response_start_ms") + if not all(key in record for key in required): + raise ValueError("latency stage metadata is incomplete") + if not isinstance(record["pid"], int) or record["pid"] <= 0: + raise ValueError("latency stage metadata has no worker") + return record diff --git a/scripts/gates/mixed_version.py b/scripts/gates/mixed_version.py index 483c2c7709..1aa207426b 100644 --- a/scripts/gates/mixed_version.py +++ b/scripts/gates/mixed_version.py @@ -75,6 +75,7 @@ committed_latency_rows, latency_observations, ) +from scripts.gates._latency_observer import HEADER, parse_metadata, source_hashes # noqa: E402 from server.core.sql_literal import sql_string # noqa: E402 from server.insights.domain import analysis as boundary # noqa: E402 @@ -233,9 +234,13 @@ def __init__( *, ranking_neutral: bool = False, web_workers: int = 1, + stage_observer: bool = False, ) -> None: assert web_workers in (1, 2) self.web_workers = web_workers + assert not stage_observer or web_workers == 2 + self.stage_observer = stage_observer + self.stage_sources = source_hashes(tree) if stage_observer else {} self.name, self.tree, self.ch_port = name, tree, ch_port self.port = _free_port() self.url = f"http://127.0.0.1:{self.port}" @@ -281,9 +286,17 @@ def env(self) -> dict[str, str]: def start(self) -> None: if self.web_workers != 1: (self.log.parent / (self.name + ".runtime")).mkdir(mode=0o700, exist_ok=True) + target = ["server.main:app"] + if self.stage_observer: + target = [ + "_latency_observer:create_app", + "--factory", + "--app-dir", + str(ROOT / "scripts" / "gates"), + ] out = self.log.open("ab") self.proc = subprocess.Popen( - [sys.executable, "-m", "uvicorn", "server.main:app"] + [sys.executable, "-m", "uvicorn", *target] + ["--host", "127.0.0.1", "--port", str(self.port)] + (["--workers", str(self.web_workers)] if self.web_workers != 1 else []), cwd=self.tree, @@ -404,6 +417,10 @@ def trace(name: str, _info: dict[str, Any]) -> None: r.headers.get("Server-Timing", ""), ): observation[f"server_{name}_ms"] = float(duration) + if self.stage_observer: + observation["server_stages"] = parse_metadata( + r.headers.get(HEADER, ""), self.stage_sources + ) except Exception as error: # noqa: BLE001 — timing metadata cannot change the response observation["timing_error"] = type(error).__name__ if r.status_code != 200: @@ -519,6 +536,7 @@ def value(key: str) -> str: self.owned_logs, ranking_neutral=True, web_workers=2, + stage_observer=True, ) for i in range(2) ] diff --git a/tests/unit/test_mixed_version_gate.py b/tests/unit/test_mixed_version_gate.py index bb35e875ce..e00775ab35 100644 --- a/tests/unit/test_mixed_version_gate.py +++ b/tests/unit/test_mixed_version_gate.py @@ -432,3 +432,314 @@ def test_malformed_timing_metadata_cannot_replace_the_complete_response(latency_ assert not {"request_id", "server_app_ms", "server_ch_ms"} & sample.keys() assert sample.get("timing_error") == ("AttributeError" if broken else None) assert len(state.closed) == len(state.calls) == 1 + + +@pytest.mark.parametrize("asynchronous", [False, True]) +@pytest.mark.parametrize("failure", ["none", "error", "cancel"]) +async def test_stage_wrappers_delegate_exact_arguments_results_and_errors(asynchronous, failure): + import asyncio + + from scripts.gates import _latency_observer as observer + + marker, answer = object(), object() + error = ( + asyncio.CancelledError("owned cancellation") + if failure == "cancel" + else RuntimeError("private") + ) + calls = [] + + def target(*args, **kwargs): + calls.append((args, kwargs)) + if failure != "none": + raise error + return answer + + async def async_target(*args, **kwargs): + return target(*args, **kwargs) + + record = {"asgi_monotonic_s": observer.time.perf_counter()} + token = observer._current.set(record) + try: + wrapped = observer._timed("auth", async_target if asynchronous else target) + if failure != "none": + with pytest.raises(type(error)) as caught: + if asynchronous: + await wrapped(marker, key=marker) + else: + wrapped(marker, key=marker) + assert caught.value is error + else: + result = ( + await wrapped(marker, key=marker) if asynchronous else wrapped(marker, key=marker) + ) + assert result is answer + assert calls == [((marker,), {"key": marker})] + assert record["auth_duration_ms"] >= 0 + assert record["auth_start_ms"] >= 0 + assert len(record) == 3 + finally: + observer._current.reset(token) + + +@pytest.mark.parametrize("cancelled", [False, True]) +async def test_stage_asgi_delegates_body_errors_and_cancellation_without_leaking_context(cancelled): + import asyncio + + from scripts.gates import _latency_observer as observer + + error = asyncio.CancelledError("owned cancellation") if cancelled else RuntimeError("private") + original = {"type": "http.request", "body": b"private body", "more_body": False} + receives, sends = [], [] + + async def receive(): + receives.append(True) + return original + + async def send(message): + sends.append(message) + + async def app(scope, receive, send): + assert await receive() is original + raise error + + wrapped = observer._ObservedApp(app, {}) + with pytest.raises(type(error)) as caught: + await wrapped({"type": "http"}, receive, send) + assert caught.value is error + assert receives == [True] and sends == [] + assert observer._current.get() is None + + +async def test_stage_asgi_metadata_is_bounded_private_and_does_not_change_response(monkeypatch): + import json + + from scripts.gates import _latency_observer as observer + from tests.unit.clock_fake import ManualClock + + clock = ManualClock(t=10) + monkeypatch.setattr(observer.time, "perf_counter", clock.monotonic) + monkeypatch.setattr(observer.time, "time_ns", lambda: 100) + sources = {"main_sha256": "a" * 64, "http_sha256": "b" * 64} + request = {"type": "http.request", "body": b"private argument", "more_body": False} + body = {"type": "http.response.body", "body": b"complete private response"} + headers = [(b"set-cookie", b"private cookie")] + messages = [] + + async def receive(): + return request + + async def send(message): + messages.append(message) + + async def handler(value): + assert value is request + clock.advance(0.003) + return body + + async def app(scope, receive, send): + assert await receive() is request + result = await observer._timed("handler", handler)(request) + await send({"type": "http.response.start", "status": 200, "headers": headers}) + await send(result) + + await observer._ObservedApp(app, sources)({"type": "http"}, receive, send) + assert messages[0]["status"] == 200 + assert messages[0]["headers"][:-1] == headers + name, raw = messages[0]["headers"][-1] + assert name == observer.HEADER.lower().encode() and len(raw) <= 4096 + metadata = observer.parse_metadata(raw.decode(), sources) + assert metadata["handler_duration_ms"] == pytest.approx(3) + assert metadata["body_delivered_ms"] == 0 + assert metadata["pid"] > 0 + assert "private" not in json.dumps(metadata) + assert messages[1] is body and observer._current.get() is None + + +@pytest.mark.parametrize("change", ["unknown", "nonfinite", "source", "oversize"]) +def test_stage_metadata_refuses_untrusted_fields_numbers_and_sources(change): + import json + + from scripts.gates import _latency_observer as observer + + sources = {"main_sha256": "a" * 64, "http_sha256": "b" * 64} + record = { + **sources, + "pid": 123, + "asgi_unix_ns": 1, + "asgi_monotonic_s": 1, + "response_start_ms": 1, + } + if change == "unknown": + record["private_argument"] = "private" + elif change == "nonfinite": + record["auth_duration_ms"] = float("nan") + elif change == "source": + record["main_sha256"] = "c" * 64 + raw = json.dumps(record) if change != "oversize" else " " * 4097 + with pytest.raises(ValueError): + observer.parse_metadata(raw, sources) + + +@pytest.mark.parametrize("wrong_origin", [False, True]) +def test_stage_factory_checks_arm_origins_before_importing_server( + monkeypatch, tmp_path, wrong_origin +): + from types import SimpleNamespace + + from scripts.gates import _latency_observer as observer + + paths = { + "server": "server/__init__.py", + "server.transport.http": "server/transport/http.py", + "server.main": "server/main.py", + } + for relative in paths.values(): + path = tmp_path / relative + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text("# This arm's source.\n") + http = SimpleNamespace(__file__=str(tmp_path / paths["server.transport.http"])) + for attribute, _name in observer._STAGES: + setattr(http, attribute, lambda *args, **kwargs: None) + app = object() + main = SimpleNamespace(__file__=str(tmp_path / paths["server.main"]), app=app) + imports, specs = [], [] + + def find_spec(name): + specs.append(name) + path = tmp_path / paths[name] + if wrong_origin and name == "server": + path = tmp_path / "wrong" / "__init__.py" + return SimpleNamespace(origin=str(path)) + + def import_module(name): + assert specs == list(paths) + imports.append(name) + return http if name == "server.transport.http" else main + + monkeypatch.chdir(tmp_path) + monkeypatch.setattr(observer, "find_spec", find_spec) + monkeypatch.setattr(observer, "import_module", import_module) + if wrong_origin: + with pytest.raises(RuntimeError, match="resolved a server outside"): + observer.create_app() + assert imports == [] + else: + wrapped = observer.create_app() + assert wrapped.app is app and wrapped.sources == observer.source_hashes(tmp_path) + assert imports == ["server.transport.http", "server.main"] + + +@pytest.mark.parametrize("observed", [False, True]) +def test_stage_profile_keeps_exact_arm_pythonpath_and_ordinary_launch( + monkeypatch, tmp_path, observed +): + from types import SimpleNamespace + + tree = tmp_path / "arm" + for relative in ("server/main.py", "server/transport/http.py"): + path = tree / relative + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text("# This arm's source.\n") + calls = [] + monkeypatch.setattr(mixed_version, "_free_port", lambda: 18080) + monkeypatch.setattr( + mixed_version.subprocess, + "Popen", + lambda command, **kw: calls.append((command, kw)) or SimpleNamespace(poll=lambda: None), + ) + monkeypatch.setattr( + mixed_version.httpx, "get", lambda *a, **kw: SimpleNamespace(status_code=200) + ) + server = mixed_version.Server( + "latency-test", tree, 18123, tmp_path, web_workers=2, stage_observer=observed + ) + server.start() + command, options = calls[0] + assert options["cwd"] == tree and options["env"]["PYTHONPATH"] == str(tree) + assert command[:3] == [mixed_version.sys.executable, "-m", "uvicorn"] + target = ( + [ + "_latency_observer:create_app", + "--factory", + "--app-dir", + str(mixed_version.ROOT / "scripts" / "gates"), + ] + if observed + else ["server.main:app"] + ) + assert command[3:] == target + ["--host", "127.0.0.1", "--port", "18080", "--workers", "2"] + options["stdout"].close() + + +async def test_stage_wrapper_delegates_non_http_scopes_without_observation(): + from scripts.gates import _latency_observer as observer + + scope, receive, send, answer = {"type": "lifespan"}, object(), object(), object() + + async def app(actual_scope, actual_receive, actual_send): + assert (actual_scope, actual_receive, actual_send) == (scope, receive, send) + assert observer._current.get() is None + return answer + + assert await observer._ObservedApp(app, {})(scope, receive, send) is answer + + +async def test_stage_encoding_failure_keeps_original_response(monkeypatch): + from scripts.gates import _latency_observer as observer + + start = {"type": "http.response.start", "status": 200, "headers": []} + body = {"type": "http.response.body", "body": b"complete response"} + sent = [] + + async def app(scope, receive, send): + await send(start) + await send(body) + + async def send(message): + sent.append(message) + + def refused(*args, **kwargs): + raise ValueError("encoding unavailable") + + monkeypatch.setattr(observer.json, "dumps", refused) + await observer._ObservedApp(app, {})({"type": "http"}, None, send) + assert sent[0] is start and sent[1] is body + assert observer._current.get() is None + + +@pytest.mark.parametrize("malformed", [False, True]) +def test_stage_client_metadata_does_not_change_closed_timing_or_complete_result( + latency_transport, malformed +): + import json + + from scripts.gates import _latency_observer as observer + + state = latency_transport + sources = {"main_sha256": "a" * 64, "http_sha256": "b" * 64} + state.server.stage_observer = True + state.server.stage_sources = sources + state.response.headers[observer.HEADER] = ( + "invalid" + if malformed + else json.dumps( + { + **sources, + "pid": 123, + "asgi_unix_ns": 1, + "asgi_monotonic_s": 1, + "response_start_ms": 1, + } + ) + ) + sample = {} + assert state.server.call("fixture-token", "memory.search", {}, observation=sample) == state.body + assert sample["elapsed_ms"] == pytest.approx(28) + assert sample["request_id"] == "from-response" + assert (sample["status"], sample["error"]) == (200, "") + assert len(state.calls) == len(state.clients) == len(state.closed) == 1 + if malformed: + assert "server_stages" not in sample and sample["timing_error"] == "JSONDecodeError" + else: + assert sample["server_stages"]["pid"] == 123 From 927dec2f296250cd781ad812be5e1d0f39c6d411 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Thu, 19 Nov 2026 02:13:12 +0000 Subject: [PATCH 23/28] Distinct-limited vector reads refuse partial identities Preserve inherited DISTINCT row and byte limits while refusing incomplete candidate sets before ranking. Retain scalar profiles, native causes and complete reuse after refusal. State the ordinary-failure boundary on cache misses. Signed-off-by: zlareb1 --- server/code/app/answers_cache.py | 5 +- server/core/vectors/admission.py | 1 + .../integration/test_vector_native_budget.py | 24 ++++ tests/unit/test_vector_read_admission.py | 103 ++++++++++++++++++ 4 files changed, 131 insertions(+), 2 deletions(-) diff --git a/server/code/app/answers_cache.py b/server/code/app/answers_cache.py index f021e1a196..24667b4070 100644 --- a/server/code/app/answers_cache.py +++ b/server/code/app/answers_cache.py @@ -98,7 +98,8 @@ class _Lookup: async def _serve_from_cache(ctx: _AskCtx) -> _Lookup: """Own namespace's semantic entry, else the exact question's cached - abstain, else the release branch's trunk entry. Fails open to a miss.""" + abstain, else the release branch's trunk entry. Ordinary read failures return + a miss; capacity refusals propagate.""" async def _read(call: Awaitable[Any]) -> Any: return await call @@ -112,7 +113,7 @@ async def _read(call: Awaitable[Any]) -> Any: ) except VectorCapacityError: raise - except Exception as e: # noqa: BLE001 — a cache read never sinks the ask + except Exception as e: # noqa: BLE001 — ordinary cache failures must not stop the ask logger.debug("code.ask cache read failed: %s", e) return _Lookup("miss") if row is not None: diff --git a/server/core/vectors/admission.py b/server/core/vectors/admission.py index c6d331488c..e4d18279bc 100644 --- a/server/core/vectors/admission.py +++ b/server/core/vectors/admission.py @@ -237,6 +237,7 @@ def native_settings(out: dict[str, Any], *, timeout: float) -> dict[str, Any]: out["result_overflow_mode"] = "throw" out["sort_overflow_mode"] = "throw" out["group_by_overflow_mode"] = "throw" + out["distinct_overflow_mode"] = "throw" out["set_overflow_mode"] = "throw" out["join_overflow_mode"] = "throw" return out diff --git a/tests/integration/test_vector_native_budget.py b/tests/integration/test_vector_native_budget.py index e063008aa0..b36d688c2b 100644 --- a/tests/integration/test_vector_native_budget.py +++ b/tests/integration/test_vector_native_budget.py @@ -222,9 +222,33 @@ def test_native_api_row_refusal_preserves_committed_heads( ) +_NATIVE_DISTINCT_IDENTITY_ROWS = ( + "SELECT memory_key AS n, cosineDistance(embedding, [1.0, 0.0]) AS distance " + "FROM (SELECT number AS memory_key, number % 2 AS writer, " + "[toFloat64(number + 1), 1.0] AS embedding FROM numbers(64)) AS candidates " + "WHERE (memory_key, writer) IN (" + "SELECT DISTINCT number, number % 2 FROM numbers(64)) " + "ORDER BY distance, n LIMIT 64" +) + + @pytest.mark.parametrize( ("sql", "profile", "complete_count", "native_code"), [ + pytest.param( + _NATIVE_DISTINCT_IDENTITY_ROWS, + {"max_rows_in_distinct": 8, "distinct_overflow_mode": "break"}, + 64, + 191, + id="distinct-identity-rows", + ), + pytest.param( + _NATIVE_DISTINCT_IDENTITY_ROWS, + {"max_bytes_in_distinct": 128, "distinct_overflow_mode": "break"}, + 64, + 191, + id="distinct-identity-bytes", + ), pytest.param( _NATIVE_RANKED_ROWS, {"max_result_rows": 8, "result_overflow_mode": "break"}, diff --git a/tests/unit/test_vector_read_admission.py b/tests/unit/test_vector_read_admission.py index abd8759708..df4bde767d 100644 --- a/tests/unit/test_vector_read_admission.py +++ b/tests/unit/test_vector_read_admission.py @@ -508,6 +508,8 @@ async def raw_query(sql: str, **kwargs: Any) -> bytes: ("result_overflow_mode", "break", {"max_result_rows": 1, "max_result_bytes": 64}), ("sort_overflow_mode", "break", {"max_rows_to_sort": 1, "max_bytes_to_sort": 64}), ("group_by_overflow_mode", "any", {"max_rows_to_group_by": 1}), + ("distinct_overflow_mode", "break", {"max_rows_in_distinct": 1}), + ("distinct_overflow_mode", "break", {"max_bytes_in_distinct": 64}), ("set_overflow_mode", "break", {"max_rows_in_set": 1, "max_bytes_in_set": 64}), ("join_overflow_mode", "break", {"max_rows_in_join": 1, "max_bytes_in_join": 64}), ], @@ -631,3 +633,104 @@ async def test_entity_boost_unrelated_native_error_retains_optional_fallback( == {} ) assert driver.started == 1 and driver.active == 0 + + +@pytest.mark.parametrize("limit", ["max_rows_in_distinct", "max_bytes_in_distinct"]) +@pytest.mark.asyncio +async def test_identifier_distinct_refusal_preserves_complete_results_and_reuses_capacity( + monkeypatch: pytest.MonkeyPatch, limit: str +) -> None: + import json + + from server.core.errors import VectorAdmissionError + from server.core.traffic_context import traffic_class_var + from server.core.vectors import admission + from server.memory.store.lookups import find_by_identifier + + gate = admission._Admission() + monkeypatch.setattr(admission, "_admission", gate) + ch, driver = client(monkeypatch, pool=2) + complete = [ + { + "memory_key": "closest", + "memory_type": "semantic", + "content_excerpt": "Issue context.\nThe complete closest passage remains.", + "version": 2, + }, + { + "memory_key": "other", + "memory_type": "semantic", + "content_excerpt": "Issue context in another complete passage.", + "version": 1, + }, + ] + failure = NativeBudgetError(191) + + async def native(sql: str, **kwargs: Any) -> Any: + driver.started += 1 + driver.settings.append(dict(kwargs["settings"])) + assert "SELECT DISTINCT" in sql and "cosineDistance" in sql + assert "org = {org:String}" in sql and "namespace = {ns:String}" in sql + assert kwargs["parameters"]["org"] == "one-org" + assert kwargs["parameters"]["ns"] == "private" + assert admission.current_work() == admission.VectorWork("one-org", ("private",), 1) + assert gate.population == 1 and gate.orgs["one-org"].held == 1 + selected = complete + settings_for_query = kwargs["settings"] + if limit in settings_for_query: + assert settings_for_query[limit] == (1 if limit == "max_rows_in_distinct" else 64) + if settings_for_query["distinct_overflow_mode"] == "break": + selected = complete[1:] + else: + raise failure + names = list(selected[0]) + return SimpleNamespace( + result_rows=[tuple(row[name] for name in names) for row in selected], + column_names=names, + summary={}, + ) + + monkeypatch.setattr(driver, "query", native) + + async def read() -> list[dict[str, Any]]: + return await find_by_identifier( + org="one-org", + namespace="private", + identifiers=["Issue"], + query_embedding=[1.0, 0.0], + ch=ch, + ) + + def drained() -> None: + assert gate.population == 0 and gate.orgs == {} + assert ch._slots is not None and ch._slots._value == 2 + assert ch._inflight == {} and ch._retired == {} + assert admission.current_work() is None + + token = traffic_class_var.set(("interactive", "test")) + try: + monkeypatch.setattr(settings, "ch_settings_by_traffic_class", "{}") + assert await read() == complete + drained() + cap = { + limit: 1 if limit == "max_rows_in_distinct" else 64, + "distinct_overflow_mode": "break", + } + monkeypatch.setattr( + settings, "ch_settings_by_traffic_class", json.dumps({"interactive": cap}) + ) + result: object = None + with pytest.raises(VectorAdmissionError) as caught: + result = await read() + assert result is None + assert caught.value.__cause__ is failure + assert caught.value.extra == {"reason": "rows", "retry_after_s": 1} + assert caught.value.code == "VECTOR_CAPACITY_REFUSED" and caught.value.status_code == 429 + assert caught.value.headers() == {"Retry-After": "1"} + drained() + monkeypatch.setattr(settings, "ch_settings_by_traffic_class", "{}") + assert await read() == complete + drained() + assert driver.started == 3 + finally: + traffic_class_var.reset(token) From cae53a7bdd2e8e2c513afd5a543257e2c69f45c1 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Thu, 19 Nov 2026 02:39:32 +0000 Subject: [PATCH 24/28] Mixed-version timing captures bounded native statement costs Retain numeric statement shapes and costs per observed request so timing analysis can separate repeated native work from more expensive queries. Keep the original witnesses, workload and resource bounds. Signed-off-by: zlareb1 --- scripts/gates/mixed_version.py | 38 ++++++++++++- tests/unit/test_mixed_version_gate.py | 77 +++++++++++++++++++++++++++ 2 files changed, 114 insertions(+), 1 deletion(-) diff --git a/scripts/gates/mixed_version.py b/scripts/gates/mixed_version.py index 1aa207426b..dbcdf0805a 100644 --- a/scripts/gates/mixed_version.py +++ b/scripts/gates/mixed_version.py @@ -835,10 +835,46 @@ def latency_timing_evidence( "AND request_id IN {ids:Array(String)} " "GROUP BY request_id ORDER BY request_id LIMIT 121 FORMAT JSONEachRow" ), + "statement_costs": ( + "SELECT splitByChar(' ', log_comment)[2] AS request_id, " + "count() AS total_statements, toUInt8(total_statements > 64) AS overflow, " + "arraySort(s -> (s.1, s.2, s.3), groupArray(64)((" + "toUnixTimestamp64Micro(query_start_time_microseconds), " + "toUnixTimestamp64Micro(event_time_microseconds), " + "if(startsWith(query_id, concat(request_id, ':')), " + "toUInt64OrZero(splitByChar(':', query_id)[2]), 0), " + "normalized_query_hash, query_duration_ms, read_rows, read_bytes, " + "memory_usage, toUInt8(type), exception_code, " + "toFloat64OrNull(Settings['max_execution_time']), " + "toUInt64OrNull(Settings['max_memory_usage']), " + "toUInt64OrNull(Settings['max_rows_to_read']), " + "if(mapContains(Settings, 'read_overflow_mode'), " + "toUInt8(Settings['read_overflow_mode'] = 'throw'), NULL)" + "))) AS statements " + "FROM system.query_log WHERE is_initial_query " + "AND type IN ('QueryFinish', 'ExceptionBeforeStart', " + "'ExceptionWhileProcessing') " + "AND request_id IN {ids:Array(String)} " + "GROUP BY request_id ORDER BY request_id LIMIT 121 FORMAT JSONEachRow" + ), } for name, statement in statements.items(): - raw = sql(self.ch_port, statement, params, bounds) + limits = bounds + if name == "statement_costs": + limits = { + **bounds, + "max_result_bytes": 2_097_152, + "output_format_json_quote_64bit_integers": 0, + } + raw = sql(self.ch_port, statement, params, limits) + if name == "statement_costs" and len(raw.encode()) > 2_097_152: + timing_evidence["statement_cost_error"] = "output_limit" + continue timing_evidence[name] = [json.loads(line) for line in raw.splitlines()] + if name == "statement_costs" and any( + row["overflow"] for row in timing_evidence[name] + ): + timing_evidence["statement_cost_error"] = "statement_limit" except Exception as error: # noqa: BLE001 — evidence cannot replace a workload failure timing_evidence["error"] = type(error).__name__ return timing_evidence diff --git a/tests/unit/test_mixed_version_gate.py b/tests/unit/test_mixed_version_gate.py index e00775ab35..0c8b03b535 100644 --- a/tests/unit/test_mixed_version_gate.py +++ b/tests/unit/test_mixed_version_gate.py @@ -743,3 +743,80 @@ def test_stage_client_metadata_does_not_change_closed_timing_or_complete_result( assert "server_stages" not in sample and sample["timing_error"] == "JSONDecodeError" else: assert sample["server_stages"]["pid"] == 123 + + +@pytest.mark.parametrize( + "mode", ["complete", "overflow", "parse-failure", "output-limit", "backend-failure"] +) +def test_latency_statement_costs_keep_original_witnesses_and_bound_diagnostics(monkeypatch, mode): + import json + + scenario = mixed_version.Scenario.__new__(mixed_version.Scenario) + scenario.ch_port = 18123 + request_id = "00000000-0000-0000-0000-000000000001" + samples = [{"request_id": request_id, "error": "503:PROJECTION_BEHIND"}] + requests = [{"request_id": request_id, "latency_ms": 20}] + native = [{"request_id": request_id, "statements": 1}] + cost = [1000, 2000, 1, 18446744073709551615, 1, 2, 3, 4, 2, 0, 30, 4000000000, 1000000, 1] + row = { + "request_id": request_id, + "total_statements": 65 if mode == "overflow" else 1, + "overflow": int(mode == "overflow"), + "statements": [cost] * (64 if mode == "overflow" else 1), + } + calls = [] + + def sql(port, query, params=None, settings=None): + assert port == 18123 + calls.append((query, params, settings)) + if query == "SYSTEM FLUSH LOGS": + return "" + if "FROM loom.agent_events" in query: + return json.dumps(requests[0]) + if "groupArray(64)" not in query: + return json.dumps(native[0]) + if mode == "backend-failure": + raise RuntimeError("private backend detail") + if mode == "parse-failure": + return "{" + if mode == "output-limit": + return " " * 2_097_153 + return json.dumps(row) + + monkeypatch.setattr(mixed_version, "sql", sql) + out = scenario.latency_timing_evidence(samples, [("main", {"org": "fixture"})]) + assert out["requests"] == requests and out["native"] == native + assert samples == [{"request_id": request_id, "error": "503:PROJECTION_BEHIND"}] + bounds = { + "max_execution_time": 5, + "max_rows_to_read": 1_000_000, + "max_memory_usage": 64_000_000, + "max_result_rows": 121, + "result_overflow_mode": "throw", + } + assert len(calls) == 4 and all(call[2] == bounds for call in calls[:3]) + query, params, limits = calls[-1] + assert params == calls[1][1] == calls[2][1] + assert limits == { + **bounds, + "max_result_bytes": 2_097_152, + "output_format_json_quote_64bit_integers": 0, + } + assert "request_id IN {ids:Array(String)}" in query + assert "GROUP BY request_id ORDER BY request_id LIMIT 121" in query + assert "normalized_query_hash" in query and "count() AS total_statements" in query + assert not any(name in query for name in ("SELECT *", "query,", "address", "exception,")) + if mode in ("complete", "overflow"): + assert out["statement_costs"] == [row] + assert out.get("statement_cost_error") == ( + "statement_limit" if mode == "overflow" else None + ) + else: + assert "statement_costs" not in out + if mode == "output-limit": + assert out["statement_cost_error"] == "output_limit" + else: + assert out["error"] == ( + "JSONDecodeError" if mode == "parse-failure" else "RuntimeError" + ) + assert "private backend detail" not in json.dumps(out) From f9f266e2f298c746d9442188398622ea1720450c Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Thu, 19 Nov 2026 04:01:49 +0000 Subject: [PATCH 25/28] Fixture timing records response publication stages Measure the existing request-tail settle and publication intervals before response headers. Keep exact source provenance and unchanged delegation so late response costs can be investigated without changing production behavior. Signed-off-by: zlareb1 --- scripts/gates/_latency_observer.py | 21 +- tests/unit/test_mixed_version_gate.py | 296 +++++++++++++++++++++++++- 2 files changed, 309 insertions(+), 8 deletions(-) diff --git a/scripts/gates/_latency_observer.py b/scripts/gates/_latency_observer.py index a5aa61fd9d..fec30582d5 100644 --- a/scripts/gates/_latency_observer.py +++ b/scripts/gates/_latency_observer.py @@ -28,7 +28,12 @@ ("_run_handler", "handler"), ("_finish_success", "finish"), ) -_PATHS = {"http_sha256": "server/transport/http.py", "main_sha256": "server/main.py"} +_TAIL_STAGES = (("settle", "tail_settle"), ("_publication_barrier", "tail_publication")) +_PATHS = { + "http_sha256": "server/transport/http.py", + "main_sha256": "server/main.py", + "request_tail_sha256": "server/core/request_tail.py", +} _FIELDS = { "pid", "asgi_unix_ns", @@ -36,7 +41,11 @@ "body_delivered_ms", "response_start_ms", "endpoint_start_ms", - *(name + suffix for _, name in _STAGES for suffix in ("_start_ms", "_duration_ms")), + *( + name + suffix + for _, name in (*_STAGES, *_TAIL_STAGES) + for suffix in ("_start_ms", "_duration_ms") + ), } _current: ContextVar[dict[str, Any] | None] = ContextVar("mixed_latency_stages", default=None) @@ -49,6 +58,7 @@ def source_hashes(tree: Path) -> dict[str, str]: def _timed(name: str, function: Any) -> Any: def finish(record: dict[str, Any], started: float) -> None: + # The last completed call wins; an outer call finishes after its nested calls. record[name + "_start_ms"] = (started - record["asgi_monotonic_s"]) * 1000 record[name + "_duration_ms"] = (time.perf_counter() - started) * 1000 if name == "bind": @@ -108,6 +118,8 @@ async def received() -> Any: async def sent(message: Any) -> None: if message["type"] == "http.response.start": + # This snapshot includes completed pre-header tail calls only. Later + # body publication checks and final cleanup cannot update this header. record["response_start_ms"] = ( time.perf_counter() - record["asgi_monotonic_s"] ) * 1000 @@ -140,18 +152,23 @@ def create_app() -> _ObservedApp: ("server", "server/__init__.py"), ("server.transport.http", _PATHS["http_sha256"]), ("server.main", _PATHS["main_sha256"]), + ("server.core.request_tail", _PATHS["request_tail_sha256"]), ): spec = find_spec(module) if spec is None or spec.origin is None or Path(spec.origin).resolve() != tree / relative: raise RuntimeError("latency observer resolved a server outside its arm") http, main = import_module("server.transport.http"), import_module("server.main") + request_tail = import_module("server.core.request_tail") if ( Path(http.__file__).resolve() != tree / _PATHS["http_sha256"] or Path(main.__file__).resolve() != tree / _PATHS["main_sha256"] + or Path(request_tail.__file__).resolve() != tree / _PATHS["request_tail_sha256"] ): raise RuntimeError("latency observer imported a server outside its arm") for attribute, name in _STAGES: setattr(http, attribute, _timed(name, getattr(http, attribute))) + for attribute, name in _TAIL_STAGES: + setattr(request_tail, attribute, _timed(name, getattr(request_tail, attribute))) return _ObservedApp(main.app, source_hashes(tree)) diff --git a/tests/unit/test_mixed_version_gate.py b/tests/unit/test_mixed_version_gate.py index 0c8b03b535..9a13b280b6 100644 --- a/tests/unit/test_mixed_version_gate.py +++ b/tests/unit/test_mixed_version_gate.py @@ -520,7 +520,11 @@ async def test_stage_asgi_metadata_is_bounded_private_and_does_not_change_respon clock = ManualClock(t=10) monkeypatch.setattr(observer.time, "perf_counter", clock.monotonic) monkeypatch.setattr(observer.time, "time_ns", lambda: 100) - sources = {"main_sha256": "a" * 64, "http_sha256": "b" * 64} + sources = { + "main_sha256": "a" * 64, + "http_sha256": "b" * 64, + "request_tail_sha256": "c" * 64, + } request = {"type": "http.request", "body": b"private argument", "more_body": False} body = {"type": "http.response.body", "body": b"complete private response"} headers = [(b"set-cookie", b"private cookie")] @@ -562,7 +566,11 @@ def test_stage_metadata_refuses_untrusted_fields_numbers_and_sources(change): from scripts.gates import _latency_observer as observer - sources = {"main_sha256": "a" * 64, "http_sha256": "b" * 64} + sources = { + "main_sha256": "a" * 64, + "http_sha256": "b" * 64, + "request_tail_sha256": "c" * 64, + } record = { **sources, "pid": 123, @@ -593,6 +601,7 @@ def test_stage_factory_checks_arm_origins_before_importing_server( "server": "server/__init__.py", "server.transport.http": "server/transport/http.py", "server.main": "server/main.py", + "server.core.request_tail": "server/core/request_tail.py", } for relative in paths.values(): path = tmp_path / relative @@ -603,6 +612,11 @@ def test_stage_factory_checks_arm_origins_before_importing_server( setattr(http, attribute, lambda *args, **kwargs: None) app = object() main = SimpleNamespace(__file__=str(tmp_path / paths["server.main"]), app=app) + request_tail = SimpleNamespace( + __file__=str(tmp_path / paths["server.core.request_tail"]), + settle=lambda *args, **kwargs: None, + _publication_barrier=lambda *args, **kwargs: None, + ) imports, specs = [], [] def find_spec(name): @@ -615,7 +629,11 @@ def find_spec(name): def import_module(name): assert specs == list(paths) imports.append(name) - return http if name == "server.transport.http" else main + return { + "server.transport.http": http, + "server.main": main, + "server.core.request_tail": request_tail, + }[name] monkeypatch.chdir(tmp_path) monkeypatch.setattr(observer, "find_spec", find_spec) @@ -627,7 +645,7 @@ def import_module(name): else: wrapped = observer.create_app() assert wrapped.app is app and wrapped.sources == observer.source_hashes(tmp_path) - assert imports == ["server.transport.http", "server.main"] + assert imports == ["server.transport.http", "server.main", "server.core.request_tail"] @pytest.mark.parametrize("observed", [False, True]) @@ -637,7 +655,11 @@ def test_stage_profile_keeps_exact_arm_pythonpath_and_ordinary_launch( from types import SimpleNamespace tree = tmp_path / "arm" - for relative in ("server/main.py", "server/transport/http.py"): + for relative in ( + "server/main.py", + "server/transport/http.py", + "server/core/request_tail.py", + ): path = tree / relative path.parent.mkdir(parents=True, exist_ok=True) path.write_text("# This arm's source.\n") @@ -717,7 +739,11 @@ def test_stage_client_metadata_does_not_change_closed_timing_or_complete_result( from scripts.gates import _latency_observer as observer state = latency_transport - sources = {"main_sha256": "a" * 64, "http_sha256": "b" * 64} + sources = { + "main_sha256": "a" * 64, + "http_sha256": "b" * 64, + "request_tail_sha256": "c" * 64, + } state.server.stage_observer = True state.server.stage_sources = sources state.response.headers[observer.HEADER] = ( @@ -820,3 +846,261 @@ def sql(port, query, params=None, settings=None): "JSONDecodeError" if mode == "parse-failure" else "RuntimeError" ) assert "private backend detail" not in json.dumps(out) + + +@pytest.fixture +def tail_observer_arm(monkeypatch, tmp_path): + from types import SimpleNamespace + + from scripts.gates import _latency_observer as observer + from tests.unit.clock_fake import ManualClock + + clock = ManualClock(t=10) + monkeypatch.setattr(observer.time, "perf_counter", clock.monotonic) + monkeypatch.setattr(observer.time, "time_ns", lambda: 100) + paths = { + "server": "server/__init__.py", + "server.transport.http": "server/transport/http.py", + "server.main": "server/main.py", + "server.core.request_tail": "server/core/request_tail.py", + } + for relative in paths.values(): + path = tmp_path / relative + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text("# This arm's source.\n") + http = SimpleNamespace(__file__=str(tmp_path / paths["server.transport.http"])) + for attribute, _name in observer._STAGES: + setattr(http, attribute, lambda *args, **kwargs: None) + main = SimpleNamespace(__file__=str(tmp_path / paths["server.main"]), app=None) + tail = SimpleNamespace(__file__=str(tmp_path / paths["server.core.request_tail"])) + modules = { + "server.transport.http": http, + "server.main": main, + "server.core.request_tail": tail, + } + monkeypatch.chdir(tmp_path) + monkeypatch.setattr( + observer, "find_spec", lambda name: SimpleNamespace(origin=str(tmp_path / paths[name])) + ) + monkeypatch.setattr(observer, "import_module", lambda name: modules[name]) + + def install(app, settle, publication): + main.app, tail.settle, tail._publication_barrier = app, settle, publication + return observer.create_app() + + return SimpleNamespace( + observer=observer, clock=clock, http=http, tail=tail, install=install, tree=tmp_path + ) + + +async def test_tail_stage_header_locates_post_finish_work_without_changing_delegation( + tail_observer_arm, +): + arm = tail_observer_arm + marker, answer = object(), object() + calls, sent = [], [] + + async def settle(value, *, selected): + calls.append(("settle", value, selected)) + arm.clock.advance(0.109384679) + return answer + + async def publication(value, *, selected): + calls.append(("publication", value, selected)) + arm.clock.advance(0.007) + return answer + + async def finished(): + arm.clock.advance(0.020) + return answer + + async def app(scope, receive, send): + assert await arm.observer._timed("finish", finished)() is answer + assert await arm.tail.settle(marker, selected=marker) is answer + assert await arm.tail._publication_barrier(marker, selected=marker) is answer + await send({"type": "http.response.start", "status": 200, "headers": []}) + return answer + + async def send(message): + sent.append(message) + + wrapped = arm.install(app, settle, publication) + assert await wrapped({"type": "http"}, None, send) is answer + raw = sent[0]["headers"][-1][1] + metadata = arm.observer.parse_metadata(raw.decode(), wrapped.sources) + assert metadata["tail_settle_start_ms"] == pytest.approx(20) + assert metadata["tail_settle_duration_ms"] == pytest.approx(109.384679) + assert metadata["tail_publication_start_ms"] == pytest.approx(129.384679) + assert metadata["tail_publication_duration_ms"] == pytest.approx(7) + finish_end = metadata["finish_start_ms"] + metadata["finish_duration_ms"] + assert metadata["response_start_ms"] - finish_end == pytest.approx(116.384679) + assert calls == [("settle", marker, marker), ("publication", marker, marker)] + assert metadata["request_tail_sha256"] == wrapped.sources["request_tail_sha256"] + assert len(raw) <= 4096 and arm.observer._current.get() is None + # Outside an observed request, these same wrappers only delegate. + assert await arm.tail.settle(marker, selected=answer) is answer + assert await arm.tail._publication_barrier(answer, selected=marker) is answer + assert calls[2:] == [("settle", marker, answer), ("publication", answer, marker)] + assert arm.observer._current.get() is None and sent[0]["headers"][-1][1] == raw + + +async def test_tail_stage_last_outer_completion_replaces_earlier_and_nested_calls( + tail_observer_arm, +): + arm = tail_observer_arm + calls, sent = [], [] + answer, marker = object(), object() + + async def settle(value, *, branch): + calls.append((value, branch)) + if branch == "early": + arm.clock.advance(0.002) + elif branch == "outer": + arm.clock.advance(0.001) + assert await arm.tail.settle(value, branch="inner") is answer + arm.clock.advance(0.004) + else: + arm.clock.advance(0.003) + return answer + + async def publication(value): + assert value is marker + arm.clock.advance(0.005) + return answer + + async def app(scope, receive, send): + assert await arm.tail.settle(marker, branch="early") is answer + assert await arm.tail.settle(marker, branch="outer") is answer + assert await arm.tail._publication_barrier(marker) is answer + await send({"type": "http.response.start", "status": 200, "headers": []}) + + async def send(message): + sent.append(message) + + wrapped = arm.install(app, settle, publication) + await wrapped({"type": "http"}, None, send) + metadata = arm.observer.parse_metadata(sent[0]["headers"][-1][1].decode(), wrapped.sources) + assert calls == [(marker, "early"), (marker, "outer"), (marker, "inner")] + assert metadata["tail_settle_start_ms"] == pytest.approx(2) + assert metadata["tail_settle_duration_ms"] == pytest.approx(8) + assert metadata["tail_publication_start_ms"] == pytest.approx(10) + assert metadata["tail_publication_duration_ms"] == pytest.approx(5) + assert metadata["response_start_ms"] == pytest.approx(15) + + +async def test_tail_stage_sent_header_excludes_later_body_checks_and_final_cleanup( + tail_observer_arm, +): + arm = tail_observer_arm + sent, records, calls = [], [], [] + body = {"type": "http.response.body", "body": b"unchanged response"} + + async def settle(duration): + calls.append(("settle", duration)) + arm.clock.advance(duration) + + async def publication(duration): + calls.append(("publication", duration)) + arm.clock.advance(duration) + + async def app(scope, receive, send): + await arm.tail.settle(0.005) + await arm.tail._publication_barrier(0.003) + await send({"type": "http.response.start", "status": 200, "headers": []}) + await arm.tail._publication_barrier(0.011) + await send(body) + await arm.tail.settle(0.017) + records.append(dict(arm.observer._current.get())) + + async def send(message): + sent.append(message) + + wrapped = arm.install(app, settle, publication) + await wrapped({"type": "http"}, None, send) + raw = sent[0]["headers"][-1][1] + metadata = arm.observer.parse_metadata(raw.decode(), wrapped.sources) + assert metadata["tail_settle_duration_ms"] == pytest.approx(5) + assert metadata["tail_publication_duration_ms"] == pytest.approx(3) + assert metadata["response_start_ms"] == pytest.approx(8) + assert records[0]["tail_settle_start_ms"] == pytest.approx(19) + assert records[0]["tail_settle_duration_ms"] == pytest.approx(17) + assert records[0]["tail_publication_start_ms"] == pytest.approx(8) + assert records[0]["tail_publication_duration_ms"] == pytest.approx(11) + assert sent[1] is body and sent[0]["headers"][-1][1] == raw + assert calls == [ + ("settle", 0.005), + ("publication", 0.003), + ("publication", 0.011), + ("settle", 0.017), + ] + assert arm.observer._current.get() is None + + +@pytest.mark.parametrize("cancelled", [False, True]) +async def test_tail_stage_failure_keeps_exact_error_and_no_response(tail_observer_arm, cancelled): + import asyncio + + arm = tail_observer_arm + error = asyncio.CancelledError("owned cancellation") if cancelled else RuntimeError("private") + marker = object() + calls, records, sent = [], [], [] + + async def settle(value, *, selected): + calls.append((value, selected)) + records.append(arm.observer._current.get()) + arm.clock.advance(0.003) + raise error + + async def publication(value): + pytest.fail("publication follows successful settle only") + + async def app(scope, receive, send): + await arm.tail.settle(marker, selected=marker) + await arm.tail._publication_barrier(marker) + await send({"type": "http.response.start", "status": 200, "headers": []}) + + async def send(message): + sent.append(message) + + wrapped = arm.install(app, settle, publication) + with pytest.raises(type(error)) as caught: + await wrapped({"type": "http"}, None, send) + assert caught.value is error and calls == [(marker, marker)] and sent == [] + assert records[0]["tail_settle_start_ms"] == pytest.approx(0) + assert records[0]["tail_settle_duration_ms"] == pytest.approx(3) + assert "tail_publication_duration_ms" not in records[0] + assert arm.observer._current.get() is None + + +def test_tail_stage_factory_refuses_wrong_tail_origin_before_wrapping(tail_observer_arm): + from types import SimpleNamespace + + arm = tail_observer_arm + observed = [] + + async def settle(value): + observed.append(value) + + async def publication(value): + observed.append(value) + + async def app(scope, receive, send): + pytest.fail("a wrong-origin app must not run") + + original_find = arm.observer.find_spec + arm.observer.find_spec = lambda name: ( + SimpleNamespace(origin=str(arm.tree / "wrong" / "request_tail.py")) + if name == "server.core.request_tail" + else original_find(name) + ) + try: + with pytest.raises(RuntimeError, match="resolved a server outside"): + arm.install(app, settle, publication) + assert arm.tail.settle is settle and arm.tail._publication_barrier is publication + finally: + arm.observer.find_spec = original_find + arm.tail.__file__ = str(arm.tree / "wrong" / "request_tail.py") + with pytest.raises(RuntimeError, match="imported a server outside"): + arm.install(app, settle, publication) + assert arm.tail.settle is settle and arm.tail._publication_barrier is publication + assert observed == [] From 97e11abbf7663f4723b34d131a1c1a8ef44d0146 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Thu, 19 Nov 2026 08:06:47 +0000 Subject: [PATCH 26/28] Prepare both latency cohorts before measuring either build Seeding the second cohort between measured arms changes the shared store under the comparison. Prepare both through the existing API before either arm starts, retaining their distinct identities and all measured requests, visibility checks, worker ownership and cleanup. Signed-off-by: zlareb1 --- scripts/gates/mixed_version.py | 59 ++++++++++++---------- tests/unit/test_mixed_version_gate.py | 71 ++++++++++++++++++++++++++- 2 files changed, 103 insertions(+), 27 deletions(-) diff --git a/scripts/gates/mixed_version.py b/scripts/gates/mixed_version.py index dbcdf0805a..35ec24ff34 100644 --- a/scripts/gates/mixed_version.py +++ b/scripts/gates/mixed_version.py @@ -526,6 +526,38 @@ def value(key: str) -> str: corpus = {f"seed/{i:04}": value(f"seed/{i:04}") for i in range(2048)} light = {f"light/{i:02}": value(f"light/{i:02}") for i in range(16)} + # Both cohorts exist before timing; head does not add its seed corpus after base. + # Measured writes still add heads between arms, so physical snapshots can differ. + prepared: dict[str, list[Any]] = {} + for label in ("base", "head"): + fixtures = [] + for role, contents in (("main", corpus), ("light", light)): + org = f"{self.org}-{label}-{role}" + ident = { + "org": org, + "namespace": "latency", + "agent": "gate", + "user_id": "-", + "session_id": "-", + } + token = mint(org, "latency") + expected = {} + entries = [ + {"memory_key": k, "value": v, "memory_type": "semantic"} + for k, v in contents.items() + ] + for offset in range(0, len(entries), 8): + seed_started = time.monotonic() + batch = entries[offset : offset + 8] + result = self.builds["N-1"].call( + token, "memory.bulk_set", {**ident, "entries": batch} + ) + expected.update(committed_latency_rows("memory.bulk_set", result, batch)) + # Four requests/sec is below the ordinary namespace write rate. + time.sleep(max(0, 0.25 - (time.monotonic() - seed_started))) + fixture = (role, ident, token, expected) + fixtures.append(fixture) + prepared[label] = fixtures for label, source in (("base", base), ("head", head)): tree = self.builds["N-1" if label == "base" else "N"].tree arm = [ @@ -550,7 +582,7 @@ def value(key: str) -> str: } proofs = [] proofs_after = [] - fixtures = [] + fixtures = prepared[label] complete = False try: for srv in arm: @@ -558,30 +590,7 @@ def value(key: str) -> str: assert srv.ready()[0] proofs.append(self.latency_workers(srv)) assert len({w["pid"] for p in proofs for w in p["workers"]}) == 4 - for role, contents in (("main", corpus), ("light", light)): - org = f"{self.org}-{label}-{role}" - ident = { - "org": org, - "namespace": "latency", - "agent": "gate", - "user_id": "-", - "session_id": "-", - } - token = mint(org, "latency") - expected = {} - entries = [ - {"memory_key": k, "value": v, "memory_type": "semantic"} - for k, v in contents.items() - ] - for offset in range(0, len(entries), 8): - seed_started = time.monotonic() - batch = entries[offset : offset + 8] - result = arm[0].call(token, "memory.bulk_set", {**ident, "entries": batch}) - expected.update(committed_latency_rows("memory.bulk_set", result, batch)) - # Four requests/sec is below the ordinary namespace write rate. - time.sleep(max(0, 0.25 - (time.monotonic() - seed_started))) - fixture = (role, ident, token, expected) - fixtures.append(fixture) + for _role, ident, token, expected in fixtures: for srv in arm: self.latency_census(srv, ident, token, expected) diff --git a/tests/unit/test_mixed_version_gate.py b/tests/unit/test_mixed_version_gate.py index 9a13b280b6..828d130de9 100644 --- a/tests/unit/test_mixed_version_gate.py +++ b/tests/unit/test_mixed_version_gate.py @@ -171,8 +171,74 @@ def test_latency_first_search_and_real_worker_identity_positive_controls() -> No assert_latency_worker(*args) +@pytest.fixture +def latency_seed_api(monkeypatch, tmp_path): + from types import SimpleNamespace + + seeded = {} + + def call(token, op, body): + assert token == "fixture-token" and op == "memory.bulk_set" + rows = seeded.setdefault(body["org"], {}) + written = [] + for entry in body["entries"]: + assert entry["memory_key"] not in rows + rows[entry["memory_key"]] = dict(entry) + written.append({**_latency_written(), "memory_key": entry["memory_key"]}) + return {"written": written} + + monkeypatch.setattr(mixed_version, "mint", lambda *_args: "fixture-token") + monkeypatch.setattr(mixed_version.time, "sleep", lambda _seconds: None) + return SimpleNamespace(tree=tmp_path, call=call, seeded=seeded) + + +def test_latency_both_complete_cohorts_exist_before_the_first_measured_group( + monkeypatch: pytest.MonkeyPatch, tmp_path, latency_seed_api +) -> None: + from types import SimpleNamespace + + class FirstGroupReached(RuntimeError): + pass + + stopped = [] + + class MeasuredGroup: + def __init__(self, name, *args, **kwargs): + self.name = name + + def start(self): + expected = { + f"fixture-{arm}-{role}": count + for arm in ("base", "head") + for role, count in (("main", 2048), ("light", 16)) + } + assert {org: len(rows) for org, rows in latency_seed_api.seeded.items()} == expected, ( + "both complete cohorts must exist before the first measured group starts" + ) + for org, rows in latency_seed_api.seeded.items(): + prefix, count, width = ( + ("seed", 2048, 4) if org.endswith("-main") else ("light", 16, 2) + ) + assert set(rows) == {f"{prefix}/{i:0{width}}" for i in range(count)} + raise FirstGroupReached() + + def stop(self): + stopped.append(self.name) + + monkeypatch.setattr(mixed_version, "Server", MeasuredGroup) + scenario = mixed_version.Scenario.__new__(mixed_version.Scenario) + scenario.org = "fixture" + scenario.builds = {"N-1": latency_seed_api, "N": SimpleNamespace(tree=tmp_path)} + scenario.ch_port = 18123 + scenario.owned_logs = tmp_path + scenario.owned_servers = [] + with pytest.raises(FirstGroupReached): + scenario.latency_acceptance("base", "head") + assert stopped == ["latency-base-0", "latency-base-1"] + + def test_latency_partial_start_registers_and_stops_both_owned_groups( - monkeypatch: pytest.MonkeyPatch, tmp_path + monkeypatch: pytest.MonkeyPatch, tmp_path, latency_seed_api ) -> None: from types import SimpleNamespace @@ -192,7 +258,8 @@ def stop(self): monkeypatch.setattr(mixed_version, "Server", RefusedStart) scenario = mixed_version.Scenario.__new__(mixed_version.Scenario) - scenario.builds = {label: SimpleNamespace(tree=tmp_path) for label in ("N-1", "N")} + scenario.org = "fixture" + scenario.builds = {"N-1": latency_seed_api, "N": SimpleNamespace(tree=tmp_path)} scenario.ch_port = 18123 scenario.owned_logs = tmp_path scenario.owned_servers = owned From 5783be69d32747dfc0617f1b8a3063f1524f1055 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Thu, 19 Nov 2026 09:45:49 +0000 Subject: [PATCH 27/28] The timing fixture records namespace key waits Record the ordinary step's interval and terminal state while delegating the existing wait. Verify its defining source and keep later cleanup out of response header snapshots. Signed-off-by: zlareb1 --- scripts/gates/_latency_observer.py | 55 +++- tests/unit/test_mixed_version_gate.py | 368 +++++++++++++++++++++++++- 2 files changed, 420 insertions(+), 3 deletions(-) diff --git a/scripts/gates/_latency_observer.py b/scripts/gates/_latency_observer.py index fec30582d5..afd174d02d 100644 --- a/scripts/gates/_latency_observer.py +++ b/scripts/gates/_latency_observer.py @@ -2,6 +2,7 @@ from __future__ import annotations +import asyncio import hashlib import inspect import json @@ -9,7 +10,7 @@ import os import time from contextvars import ContextVar -from functools import wraps +from functools import partial, wraps from importlib import import_module from importlib.util import find_spec from pathlib import Path @@ -33,6 +34,7 @@ "http_sha256": "server/transport/http.py", "main_sha256": "server/main.py", "request_tail_sha256": "server/core/request_tail.py", + "namespace_write_marks_sha256": "server/core/namespace_write_marks.py", } _FIELDS = { "pid", @@ -41,6 +43,10 @@ "body_delivered_ms", "response_start_ms", "endpoint_start_ms", + "tail_key_step_start_ms", + "tail_key_step_duration_ms", + "tail_key_done_at_entry", + "tail_key_done_at_return", *( name + suffix for _, name in (*_STAGES, *_TAIL_STAGES) @@ -93,6 +99,42 @@ def synchronous(*args: Any, **kwargs: Any) -> Any: return synchronous +def _key_step_timed(function: Any, timeout_s: float) -> Any: + @wraps(function) + async def observed(*args: Any, **kwargs: Any) -> Any: + record = _current.get() + if record is None: + return await function(*args, **kwargs) + # This is the exact callback shape produced by _land_key_before_answer. + # Unrecognized shapes remain unknown, rather than being labelled as keys. + step = args[2] if len(args) == 3 and not kwargs else None + tasks = step.args[0] if type(step) is partial and len(step.args) == 1 else None + task = tasks[0] if type(tasks) is list and len(tasks) == 1 else None + if ( + type(step) is not partial + or step.func is not asyncio.wait + or set(step.keywords) != {"timeout"} + or type(step.keywords["timeout"]) is not float + or step.keywords["timeout"] != timeout_s + or type(task) is not asyncio.Task + or task.get_name() != "loom-write-key" + ): + return await function(*args, **kwargs) + started = time.perf_counter() + done_at_entry = int(task.done()) + try: + return await function(*args, **kwargs) + finally: + # The ordinary step may also yield for required work. This interval + # is its wall time, not a measurement of native query time. + record["tail_key_step_start_ms"] = (started - record["asgi_monotonic_s"]) * 1000 + record["tail_key_step_duration_ms"] = (time.perf_counter() - started) * 1000 + record["tail_key_done_at_entry"] = done_at_entry + record["tail_key_done_at_return"] = int(task.done()) + + return observed + + class _ObservedApp: def __init__(self, app: Any, sources: dict[str, str]) -> None: self.app, self.sources = app, sources @@ -153,22 +195,29 @@ def create_app() -> _ObservedApp: ("server.transport.http", _PATHS["http_sha256"]), ("server.main", _PATHS["main_sha256"]), ("server.core.request_tail", _PATHS["request_tail_sha256"]), + ("server.core.namespace_write_marks", _PATHS["namespace_write_marks_sha256"]), ): spec = find_spec(module) if spec is None or spec.origin is None or Path(spec.origin).resolve() != tree / relative: raise RuntimeError("latency observer resolved a server outside its arm") http, main = import_module("server.transport.http"), import_module("server.main") request_tail = import_module("server.core.request_tail") + namespace_write_marks = import_module("server.core.namespace_write_marks") if ( Path(http.__file__).resolve() != tree / _PATHS["http_sha256"] or Path(main.__file__).resolve() != tree / _PATHS["main_sha256"] or Path(request_tail.__file__).resolve() != tree / _PATHS["request_tail_sha256"] + or Path(namespace_write_marks.__file__).resolve() + != tree / _PATHS["namespace_write_marks_sha256"] ): raise RuntimeError("latency observer imported a server outside its arm") for attribute, name in _STAGES: setattr(http, attribute, _timed(name, getattr(http, attribute))) for attribute, name in _TAIL_STAGES: setattr(request_tail, attribute, _timed(name, getattr(request_tail, attribute))) + request_tail._ordinary_step = _key_step_timed( + request_tail._ordinary_step, namespace_write_marks.KEY_WAIT_S + ) return _ObservedApp(main.app, source_hashes(tree)) @@ -183,6 +232,10 @@ def parse_metadata(raw: str, expected: dict[str, str]) -> dict[str, Any]: for key, value in record.items(): if key in _PATHS: continue + if key in ("tail_key_done_at_entry", "tail_key_done_at_return") and ( + type(value) is not int or value not in (0, 1) + ): + raise ValueError("latency stage metadata has an invalid completion state") if ( isinstance(value, bool) or not isinstance(value, int | float) diff --git a/tests/unit/test_mixed_version_gate.py b/tests/unit/test_mixed_version_gate.py index 828d130de9..9003eb3bd9 100644 --- a/tests/unit/test_mixed_version_gate.py +++ b/tests/unit/test_mixed_version_gate.py @@ -591,6 +591,7 @@ async def test_stage_asgi_metadata_is_bounded_private_and_does_not_change_respon "main_sha256": "a" * 64, "http_sha256": "b" * 64, "request_tail_sha256": "c" * 64, + "namespace_write_marks_sha256": "d" * 64, } request = {"type": "http.request", "body": b"private argument", "more_body": False} body = {"type": "http.response.body", "body": b"complete private response"} @@ -637,6 +638,7 @@ def test_stage_metadata_refuses_untrusted_fields_numbers_and_sources(change): "main_sha256": "a" * 64, "http_sha256": "b" * 64, "request_tail_sha256": "c" * 64, + "namespace_write_marks_sha256": "d" * 64, } record = { **sources, @@ -669,6 +671,7 @@ def test_stage_factory_checks_arm_origins_before_importing_server( "server.transport.http": "server/transport/http.py", "server.main": "server/main.py", "server.core.request_tail": "server/core/request_tail.py", + "server.core.namespace_write_marks": "server/core/namespace_write_marks.py", } for relative in paths.values(): path = tmp_path / relative @@ -683,6 +686,7 @@ def test_stage_factory_checks_arm_origins_before_importing_server( __file__=str(tmp_path / paths["server.core.request_tail"]), settle=lambda *args, **kwargs: None, _publication_barrier=lambda *args, **kwargs: None, + _ordinary_step=lambda *args, **kwargs: None, ) imports, specs = [], [] @@ -700,6 +704,9 @@ def import_module(name): "server.transport.http": http, "server.main": main, "server.core.request_tail": request_tail, + "server.core.namespace_write_marks": SimpleNamespace( + __file__=str(tmp_path / paths["server.core.namespace_write_marks"]), KEY_WAIT_S=1.0 + ), }[name] monkeypatch.chdir(tmp_path) @@ -712,7 +719,12 @@ def import_module(name): else: wrapped = observer.create_app() assert wrapped.app is app and wrapped.sources == observer.source_hashes(tmp_path) - assert imports == ["server.transport.http", "server.main", "server.core.request_tail"] + assert imports == [ + "server.transport.http", + "server.main", + "server.core.request_tail", + "server.core.namespace_write_marks", + ] @pytest.mark.parametrize("observed", [False, True]) @@ -726,6 +738,7 @@ def test_stage_profile_keeps_exact_arm_pythonpath_and_ordinary_launch( "server/main.py", "server/transport/http.py", "server/core/request_tail.py", + "server/core/namespace_write_marks.py", ): path = tree / relative path.parent.mkdir(parents=True, exist_ok=True) @@ -810,6 +823,7 @@ def test_stage_client_metadata_does_not_change_closed_timing_or_complete_result( "main_sha256": "a" * 64, "http_sha256": "b" * 64, "request_tail_sha256": "c" * 64, + "namespace_write_marks_sha256": "d" * 64, } state.server.stage_observer = True state.server.stage_sources = sources @@ -930,6 +944,7 @@ def tail_observer_arm(monkeypatch, tmp_path): "server.transport.http": "server/transport/http.py", "server.main": "server/main.py", "server.core.request_tail": "server/core/request_tail.py", + "server.core.namespace_write_marks": "server/core/namespace_write_marks.py", } for relative in paths.values(): path = tmp_path / relative @@ -939,11 +954,17 @@ def tail_observer_arm(monkeypatch, tmp_path): for attribute, _name in observer._STAGES: setattr(http, attribute, lambda *args, **kwargs: None) main = SimpleNamespace(__file__=str(tmp_path / paths["server.main"]), app=None) - tail = SimpleNamespace(__file__=str(tmp_path / paths["server.core.request_tail"])) + tail = SimpleNamespace( + __file__=str(tmp_path / paths["server.core.request_tail"]), + _ordinary_step=lambda *args, **kwargs: None, + ) modules = { "server.transport.http": http, "server.main": main, "server.core.request_tail": tail, + "server.core.namespace_write_marks": SimpleNamespace( + __file__=str(tmp_path / paths["server.core.namespace_write_marks"]), KEY_WAIT_S=1.0 + ), } monkeypatch.chdir(tmp_path) monkeypatch.setattr( @@ -1171,3 +1192,346 @@ async def app(scope, receive, send): arm.install(app, settle, publication) assert arm.tail.settle is settle and arm.tail._publication_barrier is publication assert observed == [] + + +@pytest.fixture +def key_step_observer_arm(monkeypatch, tmp_path): + from types import SimpleNamespace + + from scripts.gates import _latency_observer as observer + from server.core import request_tail + from tests.unit.clock_fake import ManualClock + + clock = ManualClock(t=10) + monkeypatch.setattr(observer.time, "perf_counter", clock.monotonic) + monkeypatch.setattr(observer.time, "time_ns", lambda: 100) + paths = { + "server": "server/__init__.py", + "server.transport.http": "server/transport/http.py", + "server.main": "server/main.py", + "server.core.request_tail": "server/core/request_tail.py", + "server.core.namespace_write_marks": "server/core/namespace_write_marks.py", + } + for relative in paths.values(): + path = tmp_path / relative + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text("# This arm's source.\n") + http = SimpleNamespace(__file__=str(tmp_path / paths["server.transport.http"])) + for attribute, _name in observer._STAGES: + setattr(http, attribute, lambda *args, **kwargs: None) + main = SimpleNamespace(__file__=str(tmp_path / paths["server.main"]), app=None) + tail = SimpleNamespace(__file__=str(tmp_path / paths["server.core.request_tail"])) + marks = SimpleNamespace( + __file__=str(tmp_path / paths["server.core.namespace_write_marks"]), KEY_WAIT_S=1.0 + ) + modules = { + "server.transport.http": http, + "server.main": main, + "server.core.request_tail": tail, + "server.core.namespace_write_marks": marks, + } + monkeypatch.chdir(tmp_path) + # Both the original and proposed factories resolve their own known modules. + monkeypatch.setattr( + observer, "find_spec", lambda name: SimpleNamespace(origin=str(tmp_path / paths[name])) + ) + monkeypatch.setattr(observer, "import_module", lambda name: modules[name]) + + async def noop(*args, **kwargs): + return None + + def install(app, ordinary): + main.app = app + tail.settle = tail._publication_barrier = noop + tail._ordinary_step = ordinary + return observer.create_app() + + return SimpleNamespace( + observer=observer, + clock=clock, + tail=tail, + marks=marks, + real_ordinary=request_tail._ordinary_step, + install=install, + tree=tmp_path, + ) + + +@pytest.mark.parametrize("completed", [False, True]) +async def test_key_step_header_observes_real_wait_and_done_state(key_step_observer_arm, completed): + import asyncio + from functools import partial + from types import SimpleNamespace + + arm = key_step_observer_arm + calls, sent = [], [] + answer, tag = object(), "private-key-tag" + tail = SimpleNamespace(has_required=False) + + async def child(): + arm.clock.advance(0.012) + return answer + + task = asyncio.create_task(child(), name="loom-write-key") + if completed: + assert await task is answer + step = partial(asyncio.wait, [task], timeout=1.0) + + async def ordinary(*args, **kwargs): + calls.append((args, kwargs)) + assert await arm.real_ordinary(*args, **kwargs) is None + arm.clock.advance(0.003) + return answer + + async def app(scope, receive, send): + assert await arm.tail._ordinary_step(tail, tag, step) is answer + await send({"type": "http.response.start", "status": 200, "headers": []}) + return answer + + async def send(message): + sent.append(message) + + wrapped = arm.install(app, ordinary) + assert await wrapped({"type": "http"}, None, send) is answer + assert calls == [((tail, tag, step), {})] and task.done() and not task.cancelled() + raw = sent[0]["headers"][-1][1] + metadata = arm.observer.parse_metadata(raw.decode(), wrapped.sources) + # Old observer reaches this successful response, then lacks these fields. + assert metadata["tail_key_step_duration_ms"] == pytest.approx(3 if completed else 15) + assert metadata["tail_key_step_start_ms"] == pytest.approx(0) + assert metadata["tail_key_done_at_entry"] == int(completed) + assert metadata["tail_key_done_at_return"] == 1 + assert ( + metadata["namespace_write_marks_sha256"] == wrapped.sources["namespace_write_marks_sha256"] + ) + assert b"private" not in raw and b"loom-write-key" not in raw and len(raw) <= 4096 + assert arm.observer._current.get() is None + + +@pytest.mark.parametrize("cancelled", [False, True]) +async def test_key_step_observation_preserves_exact_error_and_pending_child( + key_step_observer_arm, cancelled +): + import asyncio + from functools import partial + from types import SimpleNamespace + + arm = key_step_observer_arm + answer = object() + gate = asyncio.Event() + records, calls, sent = [], [], [] + error = asyncio.CancelledError("owned cancel") if cancelled else RuntimeError("private") + + async def child(): + await gate.wait() + return answer + + task = asyncio.create_task(child(), name="loom-write-key") + step = partial(asyncio.wait, [task], timeout=1.0) + tail = SimpleNamespace(has_required=False) + + async def ordinary(*args, **kwargs): + calls.append((args, kwargs)) + records.append(arm.observer._current.get()) + arm.clock.advance(0.004) + raise error + + async def app(scope, receive, send): + await arm.tail._ordinary_step(tail, "private", step) + + async def send(message): + sent.append(message) + + wrapped = arm.install(app, ordinary) + try: + with pytest.raises(type(error)) as caught: + await wrapped({"type": "http"}, None, send) + assert caught.value is error and calls == [((tail, "private", step), {})] + assert not task.done() and sent == [] and arm.observer._current.get() is None + assert records[0]["tail_key_step_duration_ms"] == pytest.approx(4) + assert records[0]["tail_key_done_at_entry"] == 0 + assert records[0]["tail_key_done_at_return"] == 0 + finally: + gate.set() + assert await task is answer + + +async def test_key_step_sequential_calls_and_later_cleanup_do_not_rewrite_header( + key_step_observer_arm, +): + import asyncio + from functools import partial + from types import SimpleNamespace + + arm = key_step_observer_arm + tail = SimpleNamespace(has_required=False) + calls, sent, records = [], [], [] + + async def child(): + return None + + task = asyncio.create_task(child(), name="loom-write-key") + await task + step = partial(asyncio.wait, [task], timeout=1.0) + + async def ordinary(*args, **kwargs): + calls.append((args, kwargs)) + await arm.real_ordinary(*args, **kwargs) + arm.clock.advance(0.002 * len(calls)) + + async def app(scope, receive, send): + await arm.tail._ordinary_step(tail, "private", step) + await arm.tail._ordinary_step(tail, "private", step) + await send({"type": "http.response.start", "status": 200, "headers": []}) + await arm.tail._ordinary_step(tail, "private", step) + records.append(dict(arm.observer._current.get())) + + async def send(message): + sent.append(message) + + wrapped = arm.install(app, ordinary) + await wrapped({"type": "http"}, None, send) + raw = sent[0]["headers"][-1][1] + metadata = arm.observer.parse_metadata(raw.decode(), wrapped.sources) + assert metadata["tail_key_step_start_ms"] == pytest.approx(2) + assert metadata["tail_key_step_duration_ms"] == pytest.approx(4) + assert metadata["response_start_ms"] == pytest.approx(6) + assert records[0]["tail_key_step_start_ms"] == pytest.approx(6) + assert records[0]["tail_key_step_duration_ms"] == pytest.approx(6) + assert sent[0]["headers"][-1][1] == raw and len(calls) == 3 + # Outside an observed request the same function only delegates. + await arm.tail._ordinary_step(tail, "private", step) + assert len(calls) == 4 and arm.observer._current.get() is None + + +@pytest.mark.parametrize("ineligible", ["name", "timeout", "keyword", "function"]) +async def test_key_step_unrecognized_callbacks_keep_ordinary_behavior( + key_step_observer_arm, ineligible +): + import asyncio + from functools import partial + from types import SimpleNamespace + + arm = key_step_observer_arm + tail = SimpleNamespace(has_required=False) + calls, sent = [], [] + answer = object() + + async def child(): + return answer + + task = asyncio.create_task(child(), name="other" if ineligible == "name" else "loom-write-key") + assert await task is answer + step = partial(asyncio.wait, [task], timeout=0.5 if ineligible == "timeout" else 1.0) + if ineligible == "keyword": + step = partial(asyncio.wait, fs=[task], timeout=1.0) + elif ineligible == "function": + step = child + + async def ordinary(*args, **kwargs): + calls.append((args, kwargs)) + await arm.real_ordinary(*args, **kwargs) + return answer + + async def app(scope, receive, send): + assert await arm.tail._ordinary_step(tail, "private", step) is answer + await send({"type": "http.response.start", "status": 200, "headers": []}) + + async def send(message): + sent.append(message) + + wrapped = arm.install(app, ordinary) + await wrapped({"type": "http"}, None, send) + metadata = arm.observer.parse_metadata(sent[0]["headers"][-1][1].decode(), wrapped.sources) + assert calls == [((tail, "private", step), {})] + assert not any(key.startswith("tail_key_") for key in metadata) + assert task.done() and not task.cancelled() + + +@pytest.mark.parametrize("value", [True, False, 0.0, 1.0, 2, -1]) +def test_key_step_metadata_done_state_requires_integer_zero_or_one(key_step_observer_arm, value): + import json + + arm = key_step_observer_arm + sources = arm.observer.source_hashes(arm.tree) + record = { + **sources, + "pid": 1, + "asgi_unix_ns": 1, + "asgi_monotonic_s": 1, + "response_start_ms": 1, + "tail_key_done_at_entry": value, + } + with pytest.raises(ValueError): + arm.observer.parse_metadata(json.dumps(record), sources) + + +async def test_key_step_bounded_wait_does_not_complete_or_cancel_pending_child( + key_step_observer_arm, monkeypatch +): + import asyncio + from functools import partial + from types import SimpleNamespace + + arm = key_step_observer_arm + gate = asyncio.Event() + calls, sent = [], [] + + async def child(): + await gate.wait() + + task = asyncio.create_task(child(), name="loom-write-key") + + async def bounded_wait(tasks, **options): + assert options == {"timeout": 1.0} + timeout = options["timeout"] + calls.append((tasks, timeout)) + assert tasks == [task] and timeout == 1.0 and not task.done() + arm.clock.advance(0.008) + return set(), {task} + + monkeypatch.setattr(asyncio, "wait", bounded_wait) + step = partial(asyncio.wait, [task], timeout=1.0) + tail = SimpleNamespace(has_required=False) + + async def app(scope, receive, send): + assert await arm.tail._ordinary_step(tail, "private", step) is None + await send({"type": "http.response.start", "status": 200, "headers": []}) + + async def send(message): + sent.append(message) + + wrapped = arm.install(app, arm.real_ordinary) + try: + await wrapped({"type": "http"}, None, send) + metadata = arm.observer.parse_metadata(sent[0]["headers"][-1][1].decode(), wrapped.sources) + assert metadata["tail_key_step_duration_ms"] == pytest.approx(8) + assert metadata["tail_key_done_at_entry"] == metadata["tail_key_done_at_return"] == 0 + assert calls == [([task], 1.0)] and not task.done() and not task.cancelled() + finally: + gate.set() + await task + + +@pytest.mark.parametrize("before_import", [False, True]) +def test_key_step_defining_source_origin_is_verified( + key_step_observer_arm, monkeypatch, before_import +): + from types import SimpleNamespace + + arm = key_step_observer_arm + original = arm.observer.find_spec + if before_import: + monkeypatch.setattr( + arm.observer, + "find_spec", + lambda name: ( + SimpleNamespace(origin=str(arm.tree / "wrong.py")) + if name == "server.core.namespace_write_marks" + else original(name) + ), + ) + else: + arm.marks.__file__ = str(arm.tree / "wrong.py") + with pytest.raises(RuntimeError, match="outside its arm"): + arm.install(None, lambda *args, **kwargs: None) From c1df6ca43da9be7199edbc9d6389763413488420 Mon Sep 17 00:00:00 2001 From: zlareb1 Date: Thu, 19 Nov 2026 10:19:02 +0000 Subject: [PATCH 28/28] SCIM clock test issues the token before deactivation Signed-off-by: zlareb1 --- tests/unit/test_scim.py | 30 ++++++++++++++++++++++++++---- 1 file changed, 26 insertions(+), 4 deletions(-) diff --git a/tests/unit/test_scim.py b/tests/unit/test_scim.py index b7dd17e092..66b36a43ef 100644 --- a/tests/unit/test_scim.py +++ b/tests/unit/test_scim.py @@ -2621,15 +2621,37 @@ async def sign_in_meanwhile(row: dict[str, Any]) -> None: assert await world.keeps_out("lee", issued_at=int(start.timestamp())) # a token from before +@pytest.mark.parametrize("elapsed_s", [0.0, 0.2], ids=["same-second", "whole-second"]) async def test_a_quick_reactivation_keeps_a_token_an_ahead_clock_issued_before_it_dead( - world, + world, monkeypatch, elapsed_s: float ) -> None: - # A replica whose clock runs ahead issued a token just before the deactivation. + from tests.unit.clock_fake import ManualClock + + clock = ManualClock(t=1000.9) + + class Clock(datetime): + @classmethod + def now(cls, tz: Any = None) -> Any: + return cls.fromtimestamp(clock.time(), tz) + + monkeypatch.setattr(scim, "datetime", Clock) + monkeypatch.setattr(revocation, "datetime", Clock) + monkeypatch.setattr( + revocation, + "time", + SimpleNamespace(time=clock.time, monotonic=time.monotonic, time_ns=time.time_ns), + ) world.person("lee@example.com", "lee") uid = await _create("lee@example.com") + ahead = int(clock.time()) + revocation.CLOCK_SKEW_S await scim.update(uid, active=False) - ahead = int(time.time()) + revocation.CLOCK_SKEW_S - await scim.update(uid, active=True) # at once + clock.advance(elapsed_s) + await scim.update(uid, active=True) + cutoff = world.ours["lee"] + assert cutoff is not None + cutoff_s = int(cutoff.replace(tzinfo=UTC).timestamp()) + assert cutoff_s == 1001 + revocation.CLOCK_SKEW_S + assert not await world.keeps_out("lee", issued_at=cutoff_s) assert await world.keeps_out("lee", issued_at=ahead)