From 976d5d7a9e07c4b5e2457a1b800ac35cd22ff91a Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 9 Oct 2026 10:33:38 -0400 Subject: [PATCH 01/13] Keep the telemetry emitter service alive through spool lock contention Every concurrent build on a host shares one SQLite spool. When another process held its write lock, a starting service's registration gave up after SQLite's 5 s busy wait, the service exited and its SQLAlchemy traceback landed on the build's stderr, so the build ran with no hosted telemetry. - The build passes its readiness deadline (--ready-deadline, Unix time) and the service retries SQLITE_BUSY/SQLITE_LOCKED during spool open and registration with jittered doubling backoff until 1 s before it. If the spool stays locked it prints one line and exits 75 (EX_TEMPFAIL); any other failure prints one line and exits 1. - Opening a spool at head takes no write lock: retention pruning moved from spool open and every append to the delivery worker (prune_if_due, once per interval, failed attempts included). - Migrations run in one BEGIN IMMEDIATE transaction after a read-only head check, so concurrent first opens no longer interleave DDL (23 of 48 such opens failed with "table already exists" or a duplicate alembic_version row) and a failed migration leaves no partial schema. - The delivery worker skips a failed step for one tick instead of dying: lock contention is silent and retried, other errors print one line per type, a failed heartbeat stays due, and the parent check that records a killed build keeps running. Co-Authored-By: Claude Opus 5.5 --- README.md | 8 +- .../telemetry-spool-lock-startup.fixed.md | 1 + .../src/microcosm/build/telemetry_emitter.py | 4 + .../telemetry_emitter_service/constants.py | 23 + .../telemetry_emitter_service/contention.py | 86 ++ .../telemetry_emitter_service/database.py | 34 +- .../build/telemetry_emitter_service/main.py | 112 +- .../telemetry_emitter_service/migrations.py | 29 +- .../telemetry_emitter_service/runtime.py | 103 +- .../build/telemetry_emitter_service/spool.py | 32 +- .../shared/test_telemetry_emitter.py | 4 +- .../shared/test_telemetry_spool_contention.py | 990 ++++++++++++++++++ 12 files changed, 1375 insertions(+), 51 deletions(-) create mode 100644 changelog.d/telemetry-spool-lock-startup.fixed.md create mode 100644 packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/contention.py create mode 100644 packages/microcosm-build/tests/engine_free/shared/test_telemetry_spool_contention.py diff --git a/README.md b/README.md index f25d358cd6..da76ea0d12 100644 --- a/README.md +++ b/README.md @@ -49,8 +49,12 @@ until you restart the kernel, load it by path with Supported US and UK build commands always start the local telemetry emitter service. It reports live progress to the hosted collector when the operator's existing Hugging Face login is accepted; otherwise it retains the events -locally and the build continues. This live event path is independent of the -staging run files described below. +locally and the build continues. Every build on a host shares one local +event spool; a service that finds it locked by another process keeps retrying +for as long as the build waits for it to become ready, and one that still +cannot register prints a single warning line, without a traceback, before the +build continues without hosted telemetry. This live event path is independent +of the staging run files described below. US fiscal refresh builds also write pre-release staging run files **by default**. Progress JSON is uploaded to `policyengine/populace-us-staging` diff --git a/changelog.d/telemetry-spool-lock-startup.fixed.md b/changelog.d/telemetry-spool-lock-startup.fixed.md new file mode 100644 index 0000000000..3dfd0bc83f --- /dev/null +++ b/changelog.d/telemetry-spool-lock-startup.fixed.md @@ -0,0 +1 @@ +A build on a busy host no longer loses its hosted telemetry when another build holds the shared local spool (`~/.cache/microcosm/telemetry/events.sqlite3`) at the moment its emitter service starts. Before, the service's registration gave up after SQLite's 5 s busy wait, the service exited and a SQLAlchemy traceback landed on the build's stderr. Now the build tells the service when it will stop waiting (`--ready-deadline`), and the service retries lock errors with jittered backoff until shortly before then; if the spool is still locked it prints one warning line and exits with status 75. Opening a spool that is already at the current schema takes no write lock: retention pruning moved from spool open and every append to the delivery worker. Schema migrations now run in one `BEGIN IMMEDIATE` transaction, so concurrent first opens of a new spool no longer fail with "table already exists" (about half of eight simultaneous opens did before) and an interrupted migration leaves no partial schema. The delivery worker skips a failed step for one tick instead of dying, so heartbeats, delivery and the check that records a killed build keep running, and errors other than lock contention print one line per error type rather than a thread traceback. diff --git a/packages/microcosm-build/src/microcosm/build/telemetry_emitter.py b/packages/microcosm-build/src/microcosm/build/telemetry_emitter.py index 761f49617f..d6245965a8 100644 --- a/packages/microcosm-build/src/microcosm/build/telemetry_emitter.py +++ b/packages/microcosm-build/src/microcosm/build/telemetry_emitter.py @@ -193,6 +193,10 @@ def start( str(os.getpid()), "--heartbeat-seconds", str(heartbeat_seconds), + # The service retries spool lock contention until just before + # this wall-clock time, when the wait below gives up. + "--ready-deadline", + repr(time.time() + startup_timeout_seconds), ] if development_collector_url is not None: command.extend( diff --git a/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/constants.py b/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/constants.py index e4a55b7d77..f0d5c83a00 100644 --- a/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/constants.py +++ b/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/constants.py @@ -53,6 +53,14 @@ DATABASE_TIMEOUT_SECONDS: Final = 5 PRUNE_INTERVAL_SECONDS: Final = 60.0 +# Every concurrent build on a host shares one spool, so a write can find it +# locked for longer than SQLite's own busy wait. Startup retries such errors +# with jittered, doubling waits until shortly before the build stops waiting +# for readiness; the margin covers binding the socket and the build's ping. +SPOOL_RETRY_INITIAL_SECONDS: Final = 0.05 +SPOOL_RETRY_MAX_SECONDS: Final = 1.0 +READY_DEADLINE_MARGIN_SECONDS: Final = 1.0 +SPOOL_LOCKED_EXIT_STATUS: Final = 75 # EX_TEMPFAIL from sysexits.h DEFAULT_HEARTBEAT_SECONDS: Final = 60.0 DEFAULT_DRAIN_SECONDS: Final = 15.0 @@ -67,3 +75,18 @@ UNSUPPORTED_ACTION_ERROR: Final = "unsupported local telemetry action" LOCAL_MESSAGE_TOO_LARGE_ERROR: Final = "local telemetry message exceeds 1 MiB" FAILURE_CLASS_UNEXPECTED_PROCESS_EXIT: Final = "unexpected_process_exit" + +READY_DEADLINE_ERROR: Final = "ready deadline must be a finite number of seconds" +SPOOL_LOCKED_WARNING: Final = ( + "warning: the local telemetry emitter service could not register this build " + "in its spool {spool}: another process kept the spool locked for " + "{waited_seconds:.1f} s ({error})." +) +SERVICE_FAILED_WARNING: Final = ( + "warning: the local telemetry emitter service stopped: {error_type}: {error}" +) +WORKER_STEP_WARNING: Final = ( + "warning: the local telemetry emitter service's delivery worker hit " + "{error_type} ({error}) and will keep running." +) +MAX_WARNING_ERROR_CHARS: Final = 300 diff --git a/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/contention.py b/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/contention.py new file mode 100644 index 0000000000..280c9661fd --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/contention.py @@ -0,0 +1,86 @@ +"""Bounded retry for lock contention on the shared telemetry spool.""" + +from __future__ import annotations + +import random +import sqlite3 +import time +from collections.abc import Callable + +from sqlalchemy.exc import DBAPIError + +from microcosm.build.telemetry_emitter_service.constants import ( + MAX_WARNING_ERROR_CHARS, + SPOOL_RETRY_INITIAL_SECONDS, + SPOOL_RETRY_MAX_SECONDS, +) + +# SQLite reports extended result codes; the low byte is the primary code. +_PRIMARY_RESULT_CODE_MASK = 0xFF +_TRANSIENT_RESULT_CODES = frozenset({sqlite3.SQLITE_BUSY, sqlite3.SQLITE_LOCKED}) + + +def is_transient_spool_error(error: BaseException) -> bool: + """Return whether an error is SQLite lock contention that a retry can clear. + + That is SQLITE_BUSY ("database is locked") or SQLITE_LOCKED, raised + directly by ``sqlite3`` or wrapped by SQLAlchemy. Schema, constraint and + I/O errors are not transient. + """ + + if isinstance(error, DBAPIError): + error = error.orig + if not isinstance(error, sqlite3.Error): + return False + code = getattr(error, "sqlite_errorcode", None) + return ( + isinstance(code, int) + and code & _PRIMARY_RESULT_CODE_MASK in _TRANSIENT_RESULT_CODES + ) + + +def retry_spool_contention[T]( + operation: Callable[[], T], + *, + deadline: float, + clock: Callable[[], float] = time.monotonic, + sleep: Callable[[float], None] = time.sleep, + jitter: Callable[[float, float], float] = random.uniform, +) -> T: + """Run ``operation``, retrying spool lock contention until ``deadline``. + + ``deadline`` is a ``clock`` reading. The first attempt always runs. After a + transient lock error the next attempt waits a jittered backoff that starts + at ``SPOOL_RETRY_INITIAL_SECONDS`` and doubles up to + ``SPOOL_RETRY_MAX_SECONDS``. When that attempt would not start before + ``deadline``, the last lock error is raised instead. Any other error is + raised at once. Each attempt may itself wait in SQLite's busy handler, so + this bounds when attempts start, not when the last one ends. + """ + + delay = SPOOL_RETRY_INITIAL_SECONDS + while True: + try: + return operation() + except Exception as error: + if not is_transient_spool_error(error): + raise + wait = jitter(delay / 2, delay) + if clock() + wait >= deadline: + raise + sleep(wait) + delay = min(SPOOL_RETRY_MAX_SECONDS, delay * 2) + + +def describe_error(error: BaseException) -> str: + """Return one short line describing an error, for a warning on stderr. + + SQLAlchemy's message repeats the SQL statement and a documentation link on + later lines, so a wrapped driver error is described by the driver's own + message. + """ + + if isinstance(error, DBAPIError) and error.orig is not None: + error = error.orig + lines = str(error).strip().splitlines() + return (lines[0] if lines else type(error).__name__)[:MAX_WARNING_ERROR_CHARS] diff --git a/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/database.py b/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/database.py index 75d32d96ba..ad9cca52cc 100644 --- a/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/database.py +++ b/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/database.py @@ -4,8 +4,8 @@ from pathlib import Path -from sqlalchemy import URL, create_engine -from sqlalchemy.engine import Engine +from sqlalchemy import URL, create_engine, event +from sqlalchemy.engine import Connection, Engine from sqlalchemy.orm import Session, sessionmaker from microcosm.build.telemetry_emitter_service.constants import ( @@ -19,13 +19,37 @@ def sqlite_database_url(path: Path | str) -> URL: return URL.create("sqlite+pysqlite", database=str(Path(path))) -def create_spool_engine(path: Path | str) -> Engine: - """Create the SQLAlchemy engine used by one emitter service process.""" +def create_spool_engine( + path: Path | str, + *, + immediate_transactions: bool = False, +) -> Engine: + """Create the SQLAlchemy engine used by one emitter service process. - return create_engine( + By default Python's sqlite3 module opens a transaction only before a data + change, never before DDL. With ``immediate_transactions`` every SQLAlchemy + transaction is instead one ``BEGIN IMMEDIATE`` transaction: it takes the + write lock at the start, waiting in SQLite's busy handler, and holds it + until commit. This is SQLAlchemy's documented pysqlite recipe for + transactional DDL. + """ + + engine = create_engine( sqlite_database_url(path), connect_args={"timeout": DATABASE_TIMEOUT_SECONDS}, ) + if immediate_transactions: + event.listen(engine, "connect", _disable_driver_transactions) + event.listen(engine, "begin", _begin_immediate) + return engine + + +def _disable_driver_transactions(dbapi_connection, connection_record) -> None: + dbapi_connection.isolation_level = None + + +def _begin_immediate(connection: Connection) -> None: + connection.exec_driver_sql("BEGIN IMMEDIATE") def create_spool_session_factory(engine: Engine) -> sessionmaker[Session]: diff --git a/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/main.py b/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/main.py index 2872bf91a8..b2cffe040f 100644 --- a/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/main.py +++ b/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/main.py @@ -4,17 +4,37 @@ import argparse import json +import math +import sys +import time from pathlib import Path from microcosm.build.telemetry_emitter_service.collector import CollectorDelivery from microcosm.build.telemetry_emitter_service.constants import ( DEFAULT_HEARTBEAT_SECONDS, + READY_DEADLINE_ERROR, + READY_DEADLINE_MARGIN_SECONDS, + SERVICE_FAILED_WARNING, + SPOOL_LOCKED_EXIT_STATUS, + SPOOL_LOCKED_WARNING, +) +from microcosm.build.telemetry_emitter_service.contention import ( + describe_error, + is_transient_spool_error, + retry_spool_contention, ) from microcosm.build.telemetry_emitter_service.resources import ProcessTreeSampler from microcosm.build.telemetry_emitter_service.runtime import EmitterService from microcosm.build.telemetry_emitter_service.spool import EventSpool +def _finite_seconds(value: str) -> float: + seconds = float(value) + if not math.isfinite(seconds): + raise argparse.ArgumentTypeError(READY_DEADLINE_ERROR) + return seconds + + def build_parser() -> argparse.ArgumentParser: """Create the service command-line parser.""" @@ -29,25 +49,89 @@ def build_parser() -> argparse.ArgumentParser: type=float, default=DEFAULT_HEARTBEAT_SECONDS, ) + parser.add_argument( + "--ready-deadline", + type=_finite_seconds, + help=( + "Unix time at which the build stops waiting for readiness. Until " + "shortly before then, startup retries spool lock contention; " + "without it, startup makes one attempt." + ), + ) return parser +def startup_deadline( + ready_deadline: float | None, + *, + now_unix: float, + now_monotonic: float, +) -> float: + """Return the ``time.monotonic()`` reading at which startup stops retrying. + + The build passes its own give-up time as Unix time, the clock both + processes share. The margin leaves time to bind the socket and answer the + build's ping before the build gives up. + """ + + if ready_deadline is None: + return -math.inf + return now_monotonic + (ready_deadline - now_unix) - READY_DEADLINE_MARGIN_SECONDS + + def main(argv: list[str] | None = None) -> int: - """Run the telemetry emitter service until the build disconnects.""" + """Run the telemetry emitter service until the build disconnects. + + Its stderr is the build's, so a failure is one line, never a traceback. + Exit status 75 means another process kept the spool locked past the + startup deadline; 1 means any other failure. + """ args = build_parser().parse_args(argv) - registration = json.loads(args.registration_json) - spool = EventSpool(args.spool) - service = EmitterService( - socket_path=args.socket, - registration=registration, - spool=spool, - delivery=CollectorDelivery( - spool, - development_collector_url=args.development_collector_url, - ), - sampler=ProcessTreeSampler(args.parent_pid), - heartbeat_seconds=args.heartbeat_seconds, + started_at = time.monotonic() + deadline = startup_deadline( + args.ready_deadline, + now_unix=time.time(), + now_monotonic=started_at, ) - service.run() + try: + registration = json.loads(args.registration_json) + spool = retry_spool_contention( + lambda: EventSpool(args.spool), + deadline=deadline, + ) + service = EmitterService( + socket_path=args.socket, + registration=registration, + spool=spool, + delivery=CollectorDelivery( + spool, + development_collector_url=args.development_collector_url, + ), + sampler=ProcessTreeSampler(args.parent_pid), + heartbeat_seconds=args.heartbeat_seconds, + startup_deadline=deadline, + ) + service.run() + except Exception as error: + if is_transient_spool_error(error): + print( + SPOOL_LOCKED_WARNING.format( + spool=args.spool, + waited_seconds=time.monotonic() - started_at, + error=describe_error(error), + ), + file=sys.stderr, + flush=True, + ) + return SPOOL_LOCKED_EXIT_STATUS + print( + SERVICE_FAILED_WARNING.format( + error_type=type(error).__name__, + error=describe_error(error), + ), + file=sys.stderr, + flush=True, + ) + return 1 return 0 diff --git a/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/migrations.py b/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/migrations.py index c1c69c61c2..f92161f42d 100644 --- a/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/migrations.py +++ b/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/migrations.py @@ -35,11 +35,30 @@ def alembic_config( def upgrade_spool_database(engine: Engine) -> None: - """Apply every committed spool migration to an engine.""" - - with engine.begin() as connection: - with alembic_config(connection=connection) as config: - command.upgrade(config, _MIGRATION_TARGET) + """Bring a spool to the packaged head, one migrating process at a time. + + A spool already at head is only read, so opening it takes no write lock. + Otherwise the whole upgrade, DDL and version stamp, runs in one + ``BEGIN IMMEDIATE`` transaction. Without it two services opening a new + spool at once interleave their DDL, and one fails with "table already + exists". A service that waits for that lock finds the spool at head when + it gets it, and Alembic then changes nothing. + """ + + with engine.connect() as connection: + current = MigrationContext.configure(connection).get_current_revision() + if current == migration_head_revision(): + return + migration_engine = create_spool_engine( + engine.url.database, + immediate_transactions=True, + ) + try: + with migration_engine.begin() as connection: + with alembic_config(connection=connection) as config: + command.upgrade(config, _MIGRATION_TARGET) + finally: + migration_engine.dispose() def current_database_revision(path: Path | str) -> str | None: diff --git a/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/runtime.py b/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/runtime.py index 42fbe231d2..fdcf02687c 100644 --- a/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/runtime.py +++ b/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/runtime.py @@ -3,11 +3,13 @@ from __future__ import annotations import json +import math import os import socket +import sys import threading import time -from collections.abc import Mapping +from collections.abc import Callable, Mapping from pathlib import Path from typing import Any @@ -24,6 +26,12 @@ SOCKET_LISTEN_BACKLOG, UNSUPPORTED_ACTION_ERROR, WORKER_INTERVAL_SECONDS, + WORKER_STEP_WARNING, +) +from microcosm.build.telemetry_emitter_service.contention import ( + describe_error, + is_transient_spool_error, + retry_spool_contention, ) from microcosm.build.telemetry_emitter_service.resources import ProcessTreeSampler from microcosm.build.telemetry_emitter_service.spool import EventSpool @@ -86,6 +94,7 @@ def __init__( sampler: ProcessTreeSampler, heartbeat_seconds: float, drain_seconds: float = DEFAULT_DRAIN_SECONDS, + startup_deadline: float = -math.inf, ) -> None: self.socket_path = socket_path self.registration = dict(registration) @@ -97,13 +106,20 @@ def __init__( heartbeat_seconds, ) self.drain_seconds = max(0.0, drain_seconds) + # A time.monotonic() reading: registration retries spool lock + # contention until then. The default allows one attempt. + self.startup_deadline = startup_deadline self._stop = threading.Event() self._last_stage = STAGE_CREATED + self._reported_error_types: set[str] = set() def run(self) -> None: """Serve local messages until the client closes or exits.""" - self.spool.register(self.registration) + retry_spool_contention( + lambda: self.spool.register(self.registration), + deadline=self.startup_deadline, + ) self.socket_path.parent.mkdir(parents=True, exist_ok=True) try: self.socket_path.unlink(missing_ok=True) @@ -184,34 +200,87 @@ def _handle(self, message: Mapping[str, Any]) -> None: raise ValueError(UNSUPPORTED_ACTION_ERROR) def _worker(self) -> None: + # A failed step is skipped for this tick only. The parent check below + # must keep running: it is the only thing that stops a service whose + # build died without closing it. next_heartbeat = time.monotonic() + self.heartbeat_seconds while not self._stop.wait(WORKER_INTERVAL_SECONDS): now = time.monotonic() # Sample every worker iteration so short-lived build children are # much less likely to disappear between stage and heartbeat events. - self.sampler.sample() - if now >= next_heartbeat: - self.spool.append( - self.registration, - _heartbeat_event(self._last_stage), - resources=self.sampler.sample(), - ) + self._attempt(self.sampler.sample) + # A heartbeat that fails stays due, so the next tick retries it. + if now >= next_heartbeat and self._attempt(self._append_heartbeat): next_heartbeat = now + self.heartbeat_seconds - self.delivery.flush_once() + self._attempt(self.spool.prune_if_due) + self._attempt(self.delivery.flush_once) if not self.sampler.parent_alive(): - self.spool.append( - self.registration, - _unexpected_exit_event(self._last_stage), - resources=self.sampler.sample(), - ) + self._attempt(self._append_unexpected_exit) self._stop.set() break self._drain() + def _append_heartbeat(self) -> None: + self.spool.append( + self.registration, + _heartbeat_event(self._last_stage), + resources=self.sampler.sample(), + ) + + def _append_unexpected_exit(self) -> None: + # This is the only record of a build that was killed, so it waits out + # lock contention for as long as the shutdown drain may take. + retry_spool_contention( + lambda: self.spool.append( + self.registration, + _unexpected_exit_event(self._last_stage), + resources=self.sampler.sample(), + ), + deadline=time.monotonic() + self.drain_seconds, + clock=time.monotonic, + sleep=time.sleep, + ) + + def _attempt(self, step: Callable[[], object]) -> bool: + """Run one worker step, reporting rather than raising its failure.""" + + try: + step() + except Exception as error: + self._report(error) + return False + return True + + def _report(self, error: Exception) -> None: + # Lock contention clears by itself and the step runs again on a later + # tick, so it is not worth a line in the build's log. Anything else is + # reported once per error type. + if is_transient_spool_error(error): + return + error_type = type(error).__name__ + if error_type in self._reported_error_types: + return + self._reported_error_types.add(error_type) + print( + WORKER_STEP_WARNING.format( + error_type=error_type, + error=describe_error(error), + ), + file=sys.stderr, + flush=True, + ) + def _drain(self) -> None: deadline = time.monotonic() + self.drain_seconds - while self.spool.has_deliverable() and time.monotonic() < deadline: - if not self.delivery.flush_once(): + while time.monotonic() < deadline: + progressed = False + try: + if not self.spool.has_deliverable(): + return + progressed = self.delivery.flush_once() + except Exception as error: + self._report(error) + if not progressed: remaining = deadline - time.monotonic() if remaining > 0: time.sleep(min(DRAIN_RETRY_SECONDS, remaining)) diff --git a/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/spool.py b/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/spool.py index 1ef43c5269..515abae9b1 100644 --- a/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/spool.py +++ b/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/spool.py @@ -48,11 +48,18 @@ def __init__(self, path: Path | str) -> None: except OSError: pass self._engine = create_spool_engine(self.path) - upgrade_spool_database(self._engine) + try: + upgrade_spool_database(self._engine) + except BaseException: + self._engine.dispose() + raise self._session_factory = create_spool_session_factory(self._engine) self._lock = threading.RLock() - self._last_prune_at = 0.0 - self.prune() + # Retention is enforced by the service's delivery worker through + # prune_if_due, not here: pruning needs the write lock whenever a row + # has expired, and this constructor runs before the build's readiness + # ping is answered. + self._last_prune_at: float | None = None def register(self, registration: Mapping[str, Any]) -> None: """Create or refresh a producer registration.""" @@ -124,8 +131,6 @@ def append( ) run.next_sequence = sequence + 1 run.updated_at = utc_now() - if time.monotonic() - self._last_prune_at >= PRUNE_INTERVAL_SECONDS: - self.prune() return payload def pending_runs(self) -> list[dict[str, Any]]: @@ -210,6 +215,22 @@ def has_pending(self) -> bool: with self._lock, self._session_factory() as session: return session.scalar(statement) is not None + def prune_if_due(self) -> None: + """Prune unless an attempt began less than ``PRUNE_INTERVAL_SECONDS`` ago. + + The first call always prunes. A failed attempt still counts, so a spool + that another process keeps locked is not rescanned every worker tick. + """ + + now = time.monotonic() + if ( + self._last_prune_at is not None + and now - self._last_prune_at < PRUNE_INTERVAL_SECONDS + ): + return + self._last_prune_at = now + self.prune() + def prune(self) -> None: """Enforce the age and total-size retention limits.""" @@ -242,7 +263,6 @@ def prune(self) -> None: ) for run in expired_runs: session.delete(run) - self._last_prune_at = time.monotonic() @staticmethod def _remove_oldest_bytes(session: Session, excess: int) -> None: diff --git a/packages/microcosm-build/tests/engine_free/shared/test_telemetry_emitter.py b/packages/microcosm-build/tests/engine_free/shared/test_telemetry_emitter.py index 65f40770b1..0fe9f44c91 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_telemetry_emitter.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_telemetry_emitter.py @@ -61,8 +61,8 @@ def _recent_timestamp() -> str: """A stored-row timestamp inside the spool's retention window. - The spool prunes rows older than ``RETENTION_DAYS`` when it opens, so a - fixed calendar date in a migrated-spool fixture expires a week later. + The service's delivery worker prunes rows older than ``RETENTION_DAYS``, + so a fixed calendar date in a migrated-spool fixture expires a week later. """ return (datetime.now(UTC) - timedelta(minutes=5)).isoformat() diff --git a/packages/microcosm-build/tests/engine_free/shared/test_telemetry_spool_contention.py b/packages/microcosm-build/tests/engine_free/shared/test_telemetry_spool_contention.py new file mode 100644 index 0000000000..deda150253 --- /dev/null +++ b/packages/microcosm-build/tests/engine_free/shared/test_telemetry_spool_contention.py @@ -0,0 +1,990 @@ +"""The telemetry emitter service under lock contention on its shared spool. + +Every concurrent build on a host shares one SQLite spool. These tests hold +its write lock from a second connection (``BEGIN IMMEDIATE``), the way another +build's service does while it writes, and check that a starting service waits +the lock out instead of exiting, that concurrent first opens migrate one at a +time, and that nothing it prints on the build's stderr is a traceback. +""" + +from __future__ import annotations + +import contextlib +import io +import json +import math +import os +import socket +import sqlite3 +import subprocess +import sys +import tempfile +import threading +import time +from datetime import UTC, datetime, timedelta +from pathlib import Path +from types import SimpleNamespace + +import alembic.op +import pytest +from alembic import command +from hypothesis import HealthCheck, given, settings +from hypothesis import strategies as st +from sqlalchemy import inspect +from sqlalchemy.exc import IntegrityError, OperationalError + +from microcosm.build.telemetry_emitter import LocalTelemetryEmitter, TelemetryRun +from microcosm.build.telemetry_emitter_constants import TELEMETRY_SERVICE_MODULE +from microcosm.build.telemetry_emitter_service import database as database_module +from microcosm.build.telemetry_emitter_service import main as main_module +from microcosm.build.telemetry_emitter_service import runtime as runtime_module +from microcosm.build.telemetry_emitter_service import spool as spool_module +from microcosm.build.telemetry_emitter_service.constants import ( + DATABASE_TIMEOUT_SECONDS, + PRUNE_INTERVAL_SECONDS, + READY_DEADLINE_MARGIN_SECONDS, + RETENTION_DAYS, + SPOOL_LOCKED_EXIT_STATUS, + SPOOL_RETRY_INITIAL_SECONDS, + SPOOL_RETRY_MAX_SECONDS, +) +from microcosm.build.telemetry_emitter_service.contention import ( + describe_error, + is_transient_spool_error, + retry_spool_contention, +) +from microcosm.build.telemetry_emitter_service.database import create_spool_engine +from microcosm.build.telemetry_emitter_service.migrations import ( + alembic_config, + current_database_revision, + migration_head_revision, +) +from microcosm.build.telemetry_emitter_service.models import ( + SpoolModel, + TelemetryEventRecord, +) +from microcosm.build.telemetry_emitter_service.runtime import EmitterService +from microcosm.build.telemetry_emitter_service.spool import EventSpool +from microcosm.build.telemetry_protocol import ( + BUILD_COMPLETED_MESSAGE, + BUILD_STARTED_MESSAGE, + LOCAL_PING_MESSAGE, +) + +_LOOPBACK_COLLECTOR = "http://127.0.0.1:9" +_SPOOL_STATES = ("fresh", "at_head", "at_head_with_expired_rows") + + +def _registration(run_id: str = "run-a") -> dict[str, object]: + return TelemetryRun( + run_id=run_id, + country_code="US", + pipeline="test-pipeline", + producer_id="producer-a", + ).as_registration() + + +def _event(stage_id: str = "compile") -> dict[str, object]: + return {"event_type": "stage", "stage_id": stage_id, "status": "started"} + + +def _close(spool: EventSpool) -> None: + spool._engine.dispose() + + +def _prepare_spool(path: Path, state: str) -> None: + """Leave ``path`` as a new file, a spool at head, or one with expired rows.""" + + if state == "fresh": + sqlite3.connect(path).close() + return + spool = EventSpool(path) + if state == "at_head_with_expired_rows": + registration = _registration("expired-run") + spool.register(registration) + spool.append(registration, _event()) + expired = (datetime.now(UTC) - timedelta(days=RETENTION_DAYS + 1)).isoformat() + with spool._session_factory.begin() as session: + session.query(TelemetryEventRecord).update({"created_at": expired}) + _close(spool) + + +@contextlib.contextmanager +def _write_lock(path: Path): + """Hold the spool's write lock from another connection, as a writer would.""" + + connection = sqlite3.connect(path, isolation_level=None) + try: + connection.execute("BEGIN IMMEDIATE") + yield connection + finally: + if connection.in_transaction: + connection.rollback() + connection.close() + + +def _short_socket_path() -> Path: + # macOS limits AF_UNIX paths to roughly 100 bytes. + return Path(tempfile.mkdtemp(prefix="microcosm-test-", dir="/tmp")) / "e.sock" + + +def _ping(socket_path: Path) -> bool: + try: + with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as client: + client.settimeout(0.5) + client.connect(str(socket_path)) + client.sendall(LOCAL_PING_MESSAGE) + return client.recv(16) == b"ok\n" + except OSError: + return False + + +def _close_service(socket_path: Path) -> None: + with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as client: + client.settimeout(2) + client.connect(str(socket_path)) + client.sendall(b'{"action":"close"}\n') + assert client.recv(16) == b"ok\n" + + +def _service_arguments( + socket_path: Path, + spool_path: Path, + *, + ready_deadline: float | None, +) -> list[str]: + arguments = [ + "--socket", + str(socket_path), + "--spool", + str(spool_path), + "--registration-json", + json.dumps(_registration()), + "--parent-pid", + str(os.getpid()), + "--development-collector-url", + _LOOPBACK_COLLECTOR, + ] + if ready_deadline is not None: + arguments += ["--ready-deadline", repr(ready_deadline)] + return arguments + + +def _lock_error(code: int = sqlite3.SQLITE_BUSY) -> OperationalError: + driver_error = sqlite3.OperationalError("database is locked") + driver_error.sqlite_errorcode = code + return OperationalError("INSERT INTO telemetry_runs ...", {}, driver_error) + + +# --- Startup waits out a held lock ------------------------------------------ + + +@pytest.mark.parametrize("state", _SPOOL_STATES) +def test_service_registers_once_a_held_spool_lock_is_released( + tmp_path, monkeypatch, capsys, state +) -> None: + """In process, with a short busy timeout so the lock outlasts many waits.""" + + monkeypatch.setattr(database_module, "DATABASE_TIMEOUT_SECONDS", 0.1) + spool_path = tmp_path / "events.sqlite3" + _prepare_spool(spool_path, state) + socket_path = _short_socket_path() + outcomes: list[tuple[str, float, BaseException | None]] = [] + + def recording(name, operation): + def wrapped(*args, **kwargs): + try: + result = operation(*args, **kwargs) + except BaseException as error: + outcomes.append((name, time.monotonic(), error)) + raise + outcomes.append((name, time.monotonic(), None)) + return result + + return wrapped + + monkeypatch.setattr(EventSpool, "__init__", recording("open", EventSpool.__init__)) + monkeypatch.setattr( + EventSpool, "register", recording("register", EventSpool.register) + ) + status: list[int] = [] + arguments = _service_arguments( + socket_path, spool_path, ready_deadline=time.time() + 60 + ) + service = threading.Thread( + target=lambda: status.append(main_module.main(arguments)) + ) + + with _write_lock(spool_path): + service.start() + time.sleep(1.0) + assert not socket_path.exists() + assert service.is_alive() + released_at = time.monotonic() + deadline = time.monotonic() + 20 + while not _ping(socket_path): + assert time.monotonic() < deadline, "service never became ready" + assert service.is_alive(), f"service exited with {status}" + time.sleep(0.02) + ready_at = time.monotonic() + _close_service(socket_path) + service.join(timeout=10) + + assert status == [0] + failures = [error for _, _, error in outcomes if error is not None] + # Ten busy timeouts fit in the hold, so a single SQLite wait cannot + # explain the success: startup retried. + assert len(failures) >= 2 + assert all(is_transient_spool_error(error) for error in failures) + name, finished_at, error = outcomes[-1] + assert (name, error) == ("register", None) + assert finished_at >= released_at + assert ready_at >= released_at + assert "Traceback" not in capsys.readouterr().err + + +def test_service_exits_75_with_one_line_while_the_spool_stays_locked( + tmp_path, monkeypatch, capsys +) -> None: + monkeypatch.setattr(database_module, "DATABASE_TIMEOUT_SECONDS", 0.1) + spool_path = tmp_path / "events.sqlite3" + _prepare_spool(spool_path, "at_head") + socket_path = _short_socket_path() + arguments = _service_arguments( + socket_path, + spool_path, + ready_deadline=time.time() + READY_DEADLINE_MARGIN_SECONDS + 0.5, + ) + + with _write_lock(spool_path): + started = time.monotonic() + status = main_module.main(arguments) + waited = time.monotonic() - started + + assert status == SPOOL_LOCKED_EXIT_STATUS + assert waited < 0.5 + 2 * DATABASE_TIMEOUT_SECONDS + assert not socket_path.exists() + error_output = capsys.readouterr().err + assert error_output.count("\n") == 1 + assert error_output.startswith( + "warning: the local telemetry emitter service could not register" + ) + assert str(spool_path) in error_output + assert "(database is locked)" in error_output + assert "Traceback" not in error_output + + +def test_unexpected_service_failure_is_one_line_with_status_1(tmp_path, capsys) -> None: + occupied = tmp_path / "not-a-directory" + occupied.write_text("") + arguments = _service_arguments( + _short_socket_path(), + occupied / "events.sqlite3", + ready_deadline=time.time() + 60, + ) + + assert main_module.main(arguments) == 1 + + error_output = capsys.readouterr().err + assert error_output.count("\n") == 1 + assert error_output.startswith( + "warning: the local telemetry emitter service stopped: " + ) + assert "Traceback" not in error_output + + +@pytest.mark.parametrize("value", ["nan", "inf", "-inf"]) +def test_ready_deadline_must_be_finite(value) -> None: + with pytest.raises(SystemExit) as raised: + main_module.build_parser().parse_args( + [ + "--socket", + "s", + "--spool", + "q", + "--registration-json", + "{}", + "--parent-pid", + "1", + "--ready-deadline", + value, + ] + ) + assert raised.value.code == 2 + + +def test_build_keeps_telemetry_when_the_spool_is_locked_past_a_busy_wait( + tmp_path, capfd, real_local_telemetry +) -> None: + """End to end: the real client starts the real service process.""" + + spool_path = tmp_path / "spool" / "events.sqlite3" + spool_directory = spool_path.parent + _prepare_spool(spool_path, "at_head") + # Opening the spool sets its directory to 0o700, which tells this test + # when the service has reached the spool without a process inspector. + spool_directory.chmod(0o750) + started: dict[str, object] = {} + + def start() -> None: + started["emitter"] = LocalTelemetryEmitter.start( + run_id="locked-spool-run", + country_code="US", + pipeline="test-pipeline", + development_collector_url=_LOOPBACK_COLLECTOR, + spool_path=spool_path, + heartbeat_seconds=60, + startup_timeout_seconds=90, + ) + started["returned_at"] = time.monotonic() + + client = threading.Thread(target=start) + with _write_lock(spool_path): + client.start() + deadline = time.monotonic() + 60 + while spool_directory.stat().st_mode & 0o777 != 0o700: + assert time.monotonic() < deadline, "service never opened the spool" + assert client.is_alive(), "client returned before the service opened" + time.sleep(0.02) + # One full SQLite busy wait, and then some, passes with the lock held. + time.sleep(DATABASE_TIMEOUT_SECONDS + 1.0) + assert client.is_alive(), "client stopped waiting while the spool was locked" + released_at = time.monotonic() + client.join(timeout=90) + + emitter = started["emitter"] + assert isinstance(emitter, LocalTelemetryEmitter) + assert emitter.available + assert started["returned_at"] > released_at + emitter.stage("compile", message="Compiling.") + emitter.complete() + assert emitter._process is not None + assert emitter._process.wait(timeout=30) == 0 + + spool = EventSpool(spool_path) + try: + events = spool.batch("locked-spool-run", emitter.run.producer_id) + finally: + _close(spool) + assert [event["sequence"] for event in events] == list(range(1, len(events) + 1)) + assert events[0]["message"] == BUILD_STARTED_MESSAGE + assert events[-1]["message"] == BUILD_COMPLETED_MESSAGE + assert "Traceback" not in capfd.readouterr().err + + +def test_service_process_reports_a_locked_spool_in_one_line(tmp_path) -> None: + """The real process: exit 75 and a single stderr line, no traceback.""" + + spool_path = tmp_path / "events.sqlite3" + _prepare_spool(spool_path, "at_head") + socket_path = _short_socket_path() + with _write_lock(spool_path): + completed = subprocess.run( + [ + sys.executable, + "-m", + TELEMETRY_SERVICE_MODULE, + # A deadline already past allows one attempt: SQLite's own wait. + *_service_arguments( + socket_path, spool_path, ready_deadline=time.time() + ), + ], + stdin=subprocess.DEVNULL, + capture_output=True, + text=True, + timeout=180, + ) + + assert completed.returncode == SPOOL_LOCKED_EXIT_STATUS + lines = [line for line in completed.stderr.splitlines() if line.strip()] + assert len(lines) == 1, completed.stderr + assert lines[0].startswith( + "warning: the local telemetry emitter service could not register" + ) + assert not socket_path.exists() + + +def test_client_passes_its_wait_as_a_wall_clock_ready_deadline( + tmp_path, monkeypatch, real_local_telemetry +) -> None: + commands: list[list[str]] = [] + + class ExitedProcess: + def poll(self): + return 1 + + def wait(self, timeout=None): + return 1 + + def popen(command, **kwargs): + commands.append(command) + return ExitedProcess() + + monkeypatch.setattr("microcosm.build.telemetry_emitter.subprocess.Popen", popen) + before = time.time() + LocalTelemetryEmitter.start( + run_id="deadline", + country_code="US", + pipeline="test-pipeline", + development_collector_url=_LOOPBACK_COLLECTOR, + spool_path=tmp_path / "events.sqlite3", + startup_timeout_seconds=42, + ) + after = time.time() + + (command,) = commands + ready_deadline = float(command[command.index("--ready-deadline") + 1]) + assert before + 42 <= ready_deadline <= after + 42 + + +@given( + ready_deadline=st.floats(-1e6, 1e6), + now_unix=st.floats(-1e6, 1e6), + now_monotonic=st.floats(0, 1e6), +) +def test_startup_deadline_is_the_ready_deadline_less_the_margin_on_this_clock( + ready_deadline, now_unix, now_monotonic +) -> None: + deadline = main_module.startup_deadline( + ready_deadline, now_unix=now_unix, now_monotonic=now_monotonic + ) + remaining = ready_deadline - now_unix - READY_DEADLINE_MARGIN_SECONDS + assert deadline - now_monotonic == pytest.approx(remaining, abs=1e-6) + assert ( + main_module.startup_deadline( + None, now_unix=now_unix, now_monotonic=now_monotonic + ) + == -math.inf + ) + + +# --- Migrations run one process at a time, atomically ----------------------- + + +def test_opening_a_spool_at_head_takes_no_write_lock(tmp_path) -> None: + """Retention runs in the worker, so expired rows don't lock the opener.""" + + spool_path = tmp_path / "events.sqlite3" + _prepare_spool(spool_path, "at_head_with_expired_rows") + + with _write_lock(spool_path): + started = time.monotonic() + spool = EventSpool(spool_path) + opened_in = time.monotonic() - started + try: + assert opened_in < DATABASE_TIMEOUT_SECONDS + assert spool.has_pending() + finally: + _close(spool) + + +def test_a_failed_migration_leaves_no_partial_schema(tmp_path, monkeypatch) -> None: + spool_path = tmp_path / "events.sqlite3" + + def fail(*args, **kwargs): + raise RuntimeError("migration interrupted") + + monkeypatch.setattr(alembic.op, "create_index", fail) + with pytest.raises(RuntimeError, match="migration interrupted"): + EventSpool(spool_path) + + connection = sqlite3.connect(spool_path) + try: + tables = connection.execute( + "SELECT name FROM sqlite_master WHERE type = 'table'" + ).fetchall() + finally: + connection.close() + assert tables == [] + + +def test_an_opener_waits_for_a_migration_in_progress(tmp_path) -> None: + """A second opener waits, then finds the spool at head and changes nothing. + + Before migrations took the write lock first, the waiting opener had + already read an empty schema and failed with "table already exists". + """ + + spool_path = tmp_path / "events.sqlite3" + sqlite3.connect(spool_path).close() + migrator = create_spool_engine(spool_path, immediate_transactions=True) + opened: list[EventSpool] = [] + failures: list[BaseException] = [] + + def open_spool() -> None: + try: + opened.append(EventSpool(spool_path)) + except BaseException as error: + failures.append(error) + + second = threading.Thread(target=open_spool) + try: + with migrator.begin() as connection: + second.start() + time.sleep(0.5) + assert second.is_alive() + SpoolModel.metadata.create_all(connection) + connection.exec_driver_sql( + "CREATE TABLE alembic_version (version_num VARCHAR(32) NOT NULL, " + "CONSTRAINT alembic_version_pkc PRIMARY KEY (version_num))" + ) + connection.exec_driver_sql( + "INSERT INTO alembic_version (version_num) VALUES (?)", + (migration_head_revision(),), + ) + second.join(timeout=DATABASE_TIMEOUT_SECONDS + 5) + finally: + migrator.dispose() + + assert failures == [] + (spool,) = opened + _close(spool) + assert current_database_revision(spool_path) == migration_head_revision() + check = create_spool_engine(spool_path) + try: + with ( + check.begin() as connection, + alembic_config(connection=connection) as config, + ): + command.check(config) + assert set(inspect(check).get_table_names()) == { + "alembic_version", + "telemetry_events", + "telemetry_runs", + } + finally: + check.dispose() + + +_CONCURRENT_OPEN = """ +import sys, time +from pathlib import Path +from microcosm.build.telemetry_emitter_service.spool import EventSpool +barrier, path, index = Path(sys.argv[1]), sys.argv[2], sys.argv[3] +(barrier / ("ready-" + index)).touch() +while not (barrier / "go").exists(): + time.sleep(0.001) +spool = EventSpool(path) +spool.register({"run_id": "run-" + index, "producer_id": "producer-" + index}) +""" + + +def test_concurrent_first_opens_of_a_new_spool_all_succeed(tmp_path) -> None: + """Before migrations took the write lock, about half of these failed.""" + + spool_path = tmp_path / "events.sqlite3" + barrier = tmp_path / "barrier" + barrier.mkdir() + count = 6 + processes = [ + subprocess.Popen( + [sys.executable, "-c", _CONCURRENT_OPEN, str(barrier), str(spool_path)] + + [str(index)], + stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + for index in range(count) + ] + try: + # Release every process at once, after all of them have imported. + deadline = time.monotonic() + 120 + while len(list(barrier.glob("ready-*"))) < count: + assert time.monotonic() < deadline, "workers never became ready" + assert all(process.poll() is None for process in processes) + time.sleep(0.01) + (barrier / "go").touch() + results = [process.communicate(timeout=120) for process in processes] + finally: + for process in processes: + if process.poll() is None: + process.kill() + process.wait() + + assert [process.returncode for process in processes] == [0] * count, results + assert current_database_revision(spool_path) == migration_head_revision() + connection = sqlite3.connect(spool_path) + try: + (registered,) = connection.execute( + "SELECT count(*) FROM telemetry_runs" + ).fetchone() + finally: + connection.close() + assert registered == count + + +# --- Retention moved off the readiness and event paths ----------------------- + + +def test_appends_never_prune(tmp_path, monkeypatch) -> None: + spool = EventSpool(tmp_path / "events.sqlite3") + pruned: list[float] = [] + monkeypatch.setattr(spool, "prune", lambda: pruned.append(time.monotonic())) + registration = _registration() + spool.register(registration) + try: + for _ in range(3): + spool.append(registration, _event()) + assert pruned == [] + finally: + _close(spool) + + +@given( + gaps=st.lists(st.floats(0, 3 * PRUNE_INTERVAL_SECONDS), max_size=30), + failures=st.lists(st.booleans(), max_size=30), +) +@settings(suppress_health_check=[HealthCheck.function_scoped_fixture]) +def test_prune_runs_at_most_once_per_interval_counting_failures( + monkeypatch, gaps, failures +) -> None: + clock = SimpleNamespace(now=1_000.0) + monkeypatch.setattr( + spool_module, "time", SimpleNamespace(monotonic=lambda: clock.now) + ) + attempts: list[float] = [] + outcomes = iter(failures) + + def prune() -> None: + attempts.append(clock.now) + if next(outcomes, False): + raise _lock_error() + + # prune_if_due reads only the schedule and prune, so no database is needed. + spool = EventSpool.__new__(EventSpool) + spool._last_prune_at = None + spool.prune = prune + calls = [clock.now] + for gap in gaps: + calls.append(calls[-1] + gap) + for now in calls: + clock.now = now + with contextlib.suppress(OperationalError): + spool.prune_if_due() + + # Reference schedule: the first call prunes; later calls prune once the + # interval has passed since the previous attempt, whether or not it failed. + expected: list[float] = [] + for now in calls: + if not expected or now - expected[-1] >= PRUNE_INTERVAL_SECONDS: + expected.append(now) + assert attempts == expected + + +# --- Lock-error classification ------------------------------------------------ + + +@given(code=st.integers(0, 2**16)) +def test_only_busy_and_locked_result_codes_are_transient(code) -> None: + error = sqlite3.OperationalError("message") + error.sqlite_errorcode = code + transient = code & 0xFF in {sqlite3.SQLITE_BUSY, sqlite3.SQLITE_LOCKED} + assert is_transient_spool_error(error) is transient + assert is_transient_spool_error(_lock_error(code)) is transient + + +def test_real_sqlite_errors_are_classified_by_what_retrying_can_fix(tmp_path) -> None: + """Differential check against errors SQLite itself raises.""" + + path = tmp_path / "errors.sqlite3" + engine = create_spool_engine(path) + impatient = sqlite3.connect(path, timeout=0, isolation_level=None) + try: + with engine.begin() as connection: + connection.exec_driver_sql("CREATE TABLE t (id INTEGER PRIMARY KEY)") + connection.exec_driver_sql("INSERT INTO t VALUES (1)") + with _write_lock(path): + with pytest.raises(sqlite3.OperationalError) as locked: + impatient.execute("INSERT INTO t VALUES (2)") + with pytest.raises(OperationalError) as exists: + with engine.begin() as connection: + connection.exec_driver_sql("CREATE TABLE t (id INTEGER)") + with pytest.raises(IntegrityError) as duplicate: + with engine.begin() as connection: + connection.exec_driver_sql("INSERT INTO t VALUES (1)") + finally: + impatient.close() + engine.dispose() + + assert is_transient_spool_error(locked.value) + assert not is_transient_spool_error(exists.value) + assert not is_transient_spool_error(duplicate.value) + assert not is_transient_spool_error(RuntimeError("database is locked")) + assert describe_error(exists.value) == "table t already exists" + + +# --- The retry helper --------------------------------------------------------- + +_OUTCOME = st.sampled_from(["ok", "locked", "fatal"]) + + +@given( + outcomes=st.lists( + st.tuples(_OUTCOME, st.floats(0, 6)), + min_size=1, + max_size=40, + ), + window=st.one_of(st.just(-math.inf), st.floats(-5, 60)), + jitter_fraction=st.floats(0, 1), +) +def test_retry_respects_its_deadline_and_backoff(outcomes, window, jitter_fraction): + clock = SimpleNamespace(now=100.0) + deadline = clock.now + window + attempt_starts: list[float] = [] + jitters: list[tuple[float, float, float, float]] = [] # (at, low, high, wait) + waits: list[tuple[float, float]] = [] # (started, seconds) + + def operation(): + index = len(attempt_starts) + attempt_starts.append(clock.now) + kind, duration = outcomes[min(index, len(outcomes) - 1)] + clock.now += duration + if kind == "locked": + raise _lock_error() + if kind == "fatal": + raise ValueError("schema error") + return index + + def jitter(low, high): + wait = low + jitter_fraction * (high - low) + jitters.append((clock.now, low, high, wait)) + return wait + + def sleep(seconds): + waits.append((clock.now, seconds)) + clock.now += seconds + + try: + result = retry_spool_contention( + operation, + deadline=deadline, + clock=lambda: clock.now, + sleep=sleep, + jitter=jitter, + ) + except ValueError: + outcome = "fatal" + except OperationalError: + outcome = "locked" + else: + outcome = "ok" + + kinds = [outcomes[min(i, len(outcomes) - 1)][0] for i in range(len(attempt_starts))] + # Every attempt but the last was a lock error, and the result is the last's. + assert all(kind == "locked" for kind in kinds[:-1]) + assert kinds[-1] == outcome + if outcome == "ok": + assert result == len(attempt_starts) - 1 + # The first attempt always runs; every later one starts before the deadline + # and no wait reaches it. + assert all(start < deadline for start in attempt_starts[1:]) + assert all(started + seconds < deadline for started, seconds in waits) + # Each lock error draws one wait in [delay / 2, delay]; delays double to a + # cap; every wait drawn is slept except one that would reach the deadline. + assert len(jitters) == kinds.count("locked") + assert len(waits) == len(attempt_starts) - 1 + for index, (_, low, high, wait) in enumerate(jitters): + assert high == min( + SPOOL_RETRY_MAX_SECONDS, SPOOL_RETRY_INITIAL_SECONDS * 2**index + ) + assert low == high / 2 + assert low <= wait <= high + assert [seconds for _, seconds in waits] == [ + wait for _, _, _, wait in jitters[: len(waits)] + ] + # A lock error escapes only when its retry could not start before the + # deadline. + if outcome == "locked": + at, _, _, wait = jitters[-1] + assert at + wait >= deadline + + +def test_retry_with_no_deadline_makes_one_attempt() -> None: + attempts: list[int] = [] + + def operation(): + attempts.append(1) + raise _lock_error() + + with pytest.raises(OperationalError): + retry_spool_contention(operation, deadline=-math.inf, sleep=pytest.fail) + assert attempts == [1] + + +# --- The delivery worker keeps running ---------------------------------------- + + +class _Outcomes: + """Hands each worker step the next scripted outcome, recording failures.""" + + def __init__(self, script: list[str]) -> None: + self._script = iter(script) + self.raised: list[str] = [] + + def next(self, value=None): + kind = next(self._script, "ok") + if kind == "locked": + self.raised.append("OperationalError") + raise _lock_error() + if kind == "runtime": + self.raised.append("RuntimeError") + raise RuntimeError("collector bug") + if kind == "value": + self.raised.append("ValueError") + raise ValueError("bad response") + return value + + +@settings(suppress_health_check=[HealthCheck.function_scoped_fixture]) +@given( + script=st.lists( + st.sampled_from(["ok", "ok", "locked", "runtime", "value"]), max_size=200 + ), + ticks=st.integers(1, 30), + parent_dies_at=st.one_of(st.none(), st.integers(1, 30)), + heartbeat_seconds=st.sampled_from([1.0, 2.0, 3.5]), +) +def test_worker_survives_any_step_failure( + monkeypatch, script, ticks, parent_dies_at, heartbeat_seconds +) -> None: + clock = SimpleNamespace(now=0.0, tick=0) + monkeypatch.setattr( + runtime_module, + "time", + SimpleNamespace( + monotonic=lambda: clock.now, + sleep=lambda seconds: setattr(clock, "now", clock.now + seconds), + ), + ) + outcomes = _Outcomes(script) + appended: list[tuple[int, str]] = [] + parent_checks: list[int] = [] + + class Stop: + def __init__(self) -> None: + self.flag = False + + def wait(self, timeout): + clock.now += timeout + clock.tick += 1 + return self.flag or clock.tick > ticks + + def set(self): + self.flag = True + + def is_set(self): + return self.flag + + def append(registration, event, *, resources=None): + outcomes.next() + appended.append((clock.tick, event["event_type"] + ":" + event["status"])) + + def parent_alive(): + parent_checks.append(clock.tick) + return parent_dies_at is None or clock.tick < parent_dies_at + + service = EmitterService( + socket_path=Path("/unused"), + registration=_registration(), + spool=SimpleNamespace( + append=append, + prune_if_due=outcomes.next, + has_deliverable=lambda: outcomes.next(False), + ), + delivery=SimpleNamespace(flush_once=lambda: outcomes.next(False)), + sampler=SimpleNamespace( + sample=lambda: outcomes.next({}), parent_alive=parent_alive + ), + heartbeat_seconds=heartbeat_seconds, + drain_seconds=2.0, + ) + service._stop = Stop() + heartbeat_attempts: list[tuple[int, float, bool]] = [] + append_heartbeat = service._append_heartbeat + + def recorded_heartbeat() -> None: + try: + append_heartbeat() + except Exception: + heartbeat_attempts.append((clock.tick, clock.now, False)) + raise + heartbeat_attempts.append((clock.tick, clock.now, True)) + + service._append_heartbeat = recorded_heartbeat + exit_records: list[int] = [] + append_unexpected_exit = service._append_unexpected_exit + + def recorded_unexpected_exit() -> None: + exit_records.append(clock.tick) + append_unexpected_exit() + + service._append_unexpected_exit = recorded_unexpected_exit + error_output = io.StringIO() + + with contextlib.redirect_stderr(error_output): + service._worker() # never raises + + last_tick = ticks if parent_dies_at is None else min(ticks, parent_dies_at) + # The parent is checked on every tick until it is found dead. + assert parent_checks == list(range(1, last_tick + 1)) + # A dead parent is recorded once, on the tick it is found, and stops the + # loop. + parent_died = parent_dies_at is not None and parent_dies_at <= ticks + assert exit_records == ([parent_dies_at] if parent_died else []) + assert service._stop.is_set() is parent_died + # Reference heartbeat schedule: due one interval after the start or the + # last success; a failed heartbeat stays due and is retried next tick. + due = heartbeat_seconds + expected_ticks: list[int] = [] + attempts = iter(heartbeat_attempts) + for tick in range(1, last_tick + 1): + now = float(tick) + if now >= due: + expected_ticks.append(tick) + attempt_tick, _, succeeded = next(attempts) + assert attempt_tick == tick + if succeeded: + due = now + heartbeat_seconds + assert [tick for tick, _, _ in heartbeat_attempts] == expected_ticks + # One line per error type other than lock contention, which is silent. + reported = { + name for name in outcomes.raised if name in {"RuntimeError", "ValueError"} + } + lines = error_output.getvalue().splitlines() + assert len(lines) == len(reported) + assert {line.split(" hit ")[1].split(" ")[0] for line in lines} == reported + assert not any("Traceback" in line for line in lines) + + +def test_unexpected_exit_waits_out_lock_contention(monkeypatch) -> None: + clock = SimpleNamespace(now=0.0) + monkeypatch.setattr( + runtime_module, + "time", + SimpleNamespace( + monotonic=lambda: clock.now, + sleep=lambda seconds: setattr(clock, "now", clock.now + seconds), + ), + ) + attempts: list[float] = [] + + def append(registration, event, *, resources=None): + attempts.append(clock.now) + clock.now += 0.2 + if len(attempts) < 4: + raise _lock_error() + + service = EmitterService( + socket_path=Path("/unused"), + registration=_registration(), + spool=SimpleNamespace(append=append), + delivery=SimpleNamespace(), + sampler=SimpleNamespace(sample=dict), + heartbeat_seconds=60, + drain_seconds=15, + ) + + assert service._attempt(service._append_unexpected_exit) + assert len(attempts) == 4 + assert attempts[-1] < 15 From f510f2d63554302496b03f69a9d3b04150488f0b Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 9 Oct 2026 10:46:54 -0400 Subject: [PATCH 02/13] Bound startup waits, chunk the prune, and keep the worker's errors contained From the design review of the first commit: - Startup statements wait at most 0.25 s each in SQLite (a busy timeout applied per connection checkout), so the retry loop, not a 5 s busy wait, decides when to give up, and the service exits with its one-line reason before the build stops waiting (margin now 2 s). The wait is capped at 60 s whatever the deadline or the wall clock says; an infinite deadline is accepted and capped, NaN is refused. - main opens and registers the spool (open_registered_spool), then restores the normal 5 s wait; EmitterService.run expects a registered producer. - The client computes the deadline with datetime.now(UTC), the module's existing wall clock, so #1166's fake-clock startup tests keep passing once both land. - prune deletes in batches of 500 rows, each in its own transaction, so the first prune in the worker never holds the write lock (or the lock the event path shares) for long. - An unknown spool revision (a later migration from another checkout) is refused from the read-only check, without taking the write lock. - Warnings go through write_warning, which never raises, so a closed stderr pipe cannot kill the worker; serving stops if the worker ever dies; an unexpected delivery error backs off like a collector failure instead of repeating every tick. Co-Authored-By: Claude Opus 5.5 --- .../src/microcosm/build/telemetry_emitter.py | 2 +- .../telemetry_emitter_service/collector.py | 20 +- .../telemetry_emitter_service/constants.py | 17 +- .../telemetry_emitter_service/contention.py | 15 - .../telemetry_emitter_service/database.py | 13 + .../telemetry_emitter_service/diagnostics.py | 38 ++ .../build/telemetry_emitter_service/main.py | 71 ++-- .../telemetry_emitter_service/migrations.py | 44 ++- .../telemetry_emitter_service/runtime.py | 35 +- .../build/telemetry_emitter_service/spool.py | 118 +++--- .../shared/test_telemetry_emitter.py | 1 + .../shared/test_telemetry_spool_contention.py | 359 ++++++++++++++++-- 12 files changed, 587 insertions(+), 146 deletions(-) create mode 100644 packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/diagnostics.py diff --git a/packages/microcosm-build/src/microcosm/build/telemetry_emitter.py b/packages/microcosm-build/src/microcosm/build/telemetry_emitter.py index d6245965a8..8bd365d4a3 100644 --- a/packages/microcosm-build/src/microcosm/build/telemetry_emitter.py +++ b/packages/microcosm-build/src/microcosm/build/telemetry_emitter.py @@ -196,7 +196,7 @@ def start( # The service retries spool lock contention until just before # this wall-clock time, when the wait below gives up. "--ready-deadline", - repr(time.time() + startup_timeout_seconds), + repr(datetime.now(UTC).timestamp() + startup_timeout_seconds), ] if development_collector_url is not None: command.extend( diff --git a/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/collector.py b/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/collector.py index 795c61a177..fa62e89edc 100644 --- a/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/collector.py +++ b/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/collector.py @@ -4,7 +4,6 @@ import json import os -import sys import time import urllib.error import urllib.request @@ -40,6 +39,10 @@ TOKEN_EXCHANGE_PATH, TOKEN_REFRESH_MARGIN_SECONDS, ) +from microcosm.build.telemetry_emitter_service.contention import ( + is_transient_spool_error, +) +from microcosm.build.telemetry_emitter_service.diagnostics import write_warning from microcosm.build.telemetry_emitter_service.spool import EventSpool @@ -154,6 +157,17 @@ def flush_once(self) -> bool: if time.monotonic() < self._next_attempt_at: return False + try: + return self._flush_pending_runs() + except Exception as error: + # The worker runs this every second and survives the error, so an + # unexpected one backs off like a collector failure rather than + # repeating requests every tick. A busy spool is retried next tick. + if not is_transient_spool_error(error): + self._defer_retry() + raise + + def _flush_pending_runs(self) -> bool: made_progress = False for registration in self.spool.pending_runs(): run_id = str(registration["run_id"]) @@ -200,7 +214,7 @@ def _collector_token(self, registration: Mapping[str, Any]) -> str | None: hf_token = _huggingface_token() if not hf_token: if not self._warned_no_token: - print(NO_CREDENTIAL_MESSAGE, file=sys.stderr, flush=True) + write_warning(NO_CREDENTIAL_MESSAGE) self._warned_no_token = True self._make_local_only(registration, LOCAL_ONLY_MISSING_CREDENTIAL) return None @@ -269,5 +283,5 @@ def _defer_retry(self) -> None: def _warn_denied(self, registration_key: str) -> None: if registration_key in self._warned_denied: return - print(REJECTED_CREDENTIAL_MESSAGE, file=sys.stderr, flush=True) + write_warning(REJECTED_CREDENTIAL_MESSAGE) self._warned_denied.add(registration_key) diff --git a/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/constants.py b/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/constants.py index f0d5c83a00..8b245dfe03 100644 --- a/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/constants.py +++ b/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/constants.py @@ -53,13 +53,18 @@ DATABASE_TIMEOUT_SECONDS: Final = 5 PRUNE_INTERVAL_SECONDS: Final = 60.0 +PRUNE_BATCH_ROWS: Final = 500 # Every concurrent build on a host shares one spool, so a write can find it # locked for longer than SQLite's own busy wait. Startup retries such errors # with jittered, doubling waits until shortly before the build stops waiting -# for readiness; the margin covers binding the socket and the build's ping. +# for readiness. While it does, each SQLite statement waits at most +# STARTUP_BUSY_TIMEOUT_SECONDS, so an attempt begun before that deadline ends +# within the margin, which also covers binding the socket and the build's ping. +STARTUP_BUSY_TIMEOUT_SECONDS: Final = 0.25 SPOOL_RETRY_INITIAL_SECONDS: Final = 0.05 -SPOOL_RETRY_MAX_SECONDS: Final = 1.0 -READY_DEADLINE_MARGIN_SECONDS: Final = 1.0 +SPOOL_RETRY_MAX_SECONDS: Final = 0.25 +READY_DEADLINE_MARGIN_SECONDS: Final = 2.0 +STARTUP_RETRY_LIMIT_SECONDS: Final = 60.0 SPOOL_LOCKED_EXIT_STATUS: Final = 75 # EX_TEMPFAIL from sysexits.h DEFAULT_HEARTBEAT_SECONDS: Final = 60.0 @@ -76,7 +81,11 @@ LOCAL_MESSAGE_TOO_LARGE_ERROR: Final = "local telemetry message exceeds 1 MiB" FAILURE_CLASS_UNEXPECTED_PROCESS_EXIT: Final = "unexpected_process_exit" -READY_DEADLINE_ERROR: Final = "ready deadline must be a finite number of seconds" +READY_DEADLINE_ERROR: Final = "ready deadline must be a number of seconds" +UNKNOWN_SPOOL_REVISION_ERROR: Final = ( + "telemetry spool schema revision {revision!r} is not in this microcosm's " + "migration history, whose head is {head!r}" +) SPOOL_LOCKED_WARNING: Final = ( "warning: the local telemetry emitter service could not register this build " "in its spool {spool}: another process kept the spool locked for " diff --git a/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/contention.py b/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/contention.py index 280c9661fd..06dc14d700 100644 --- a/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/contention.py +++ b/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/contention.py @@ -10,7 +10,6 @@ from sqlalchemy.exc import DBAPIError from microcosm.build.telemetry_emitter_service.constants import ( - MAX_WARNING_ERROR_CHARS, SPOOL_RETRY_INITIAL_SECONDS, SPOOL_RETRY_MAX_SECONDS, ) @@ -70,17 +69,3 @@ def retry_spool_contention[T]( raise sleep(wait) delay = min(SPOOL_RETRY_MAX_SECONDS, delay * 2) - - -def describe_error(error: BaseException) -> str: - """Return one short line describing an error, for a warning on stderr. - - SQLAlchemy's message repeats the SQL statement and a documentation link on - later lines, so a wrapped driver error is described by the driver's own - message. - """ - - if isinstance(error, DBAPIError) and error.orig is not None: - error = error.orig - lines = str(error).strip().splitlines() - return (lines[0] if lines else type(error).__name__)[:MAX_WARNING_ERROR_CHARS] diff --git a/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/database.py b/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/database.py index ad9cca52cc..8b94747358 100644 --- a/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/database.py +++ b/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/database.py @@ -2,6 +2,7 @@ from __future__ import annotations +from collections.abc import Callable from pathlib import Path from sqlalchemy import URL, create_engine, event @@ -23,9 +24,14 @@ def create_spool_engine( path: Path | str, *, immediate_transactions: bool = False, + busy_timeout_seconds: Callable[[], float] | None = None, ) -> Engine: """Create the SQLAlchemy engine used by one emitter service process. + ``busy_timeout_seconds`` is read each time a connection is checked out and + caps how long one statement waits in SQLite's busy handler for another + process's lock; by default that is ``DATABASE_TIMEOUT_SECONDS``. + By default Python's sqlite3 module opens a transaction only before a data change, never before DDL. With ``immediate_transactions`` every SQLAlchemy transaction is instead one ``BEGIN IMMEDIATE`` transaction: it takes the @@ -38,6 +44,13 @@ def create_spool_engine( sqlite_database_url(path), connect_args={"timeout": DATABASE_TIMEOUT_SECONDS}, ) + if busy_timeout_seconds is not None: + + def apply_busy_timeout(dbapi_connection, connection_record, proxy) -> None: + milliseconds = max(0, round(busy_timeout_seconds() * 1000)) + dbapi_connection.execute(f"PRAGMA busy_timeout = {milliseconds}") + + event.listen(engine, "checkout", apply_busy_timeout) if immediate_transactions: event.listen(engine, "connect", _disable_driver_transactions) event.listen(engine, "begin", _begin_immediate) diff --git a/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/diagnostics.py b/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/diagnostics.py new file mode 100644 index 0000000000..5eaa5d52ee --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/diagnostics.py @@ -0,0 +1,38 @@ +"""One-line warnings on the build's stderr, which the service inherits.""" + +from __future__ import annotations + +import sys + +from sqlalchemy.exc import DBAPIError + +from microcosm.build.telemetry_emitter_service.constants import ( + MAX_WARNING_ERROR_CHARS, +) + + +def describe_error(error: BaseException) -> str: + """Return one short line describing an error, for a warning on stderr. + + SQLAlchemy's message repeats the SQL statement and its parameters, which + can hold a registration or event payload, and a documentation link on later + lines, so a wrapped driver error is described by the driver's own message. + """ + + if isinstance(error, DBAPIError) and error.orig is not None: + error = error.orig + lines = str(error).strip().splitlines() + return (lines[0] if lines else type(error).__name__)[:MAX_WARNING_ERROR_CHARS] + + +def write_warning(message: str) -> None: + """Print one line to stderr without ever raising. + + The service runs in its own session, so it can outlive a build whose + stderr was a pipe that has since closed; a failed write must not stop it. + """ + + try: + print(message, file=sys.stderr, flush=True) + except (OSError, ValueError): + pass diff --git a/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/main.py b/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/main.py index b2cffe040f..4005078b62 100644 --- a/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/main.py +++ b/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/main.py @@ -5,32 +5,37 @@ import argparse import json import math -import sys import time from pathlib import Path from microcosm.build.telemetry_emitter_service.collector import CollectorDelivery from microcosm.build.telemetry_emitter_service.constants import ( + DATABASE_TIMEOUT_SECONDS, DEFAULT_HEARTBEAT_SECONDS, READY_DEADLINE_ERROR, READY_DEADLINE_MARGIN_SECONDS, SERVICE_FAILED_WARNING, SPOOL_LOCKED_EXIT_STATUS, SPOOL_LOCKED_WARNING, + STARTUP_BUSY_TIMEOUT_SECONDS, + STARTUP_RETRY_LIMIT_SECONDS, ) from microcosm.build.telemetry_emitter_service.contention import ( - describe_error, is_transient_spool_error, retry_spool_contention, ) +from microcosm.build.telemetry_emitter_service.diagnostics import ( + describe_error, + write_warning, +) from microcosm.build.telemetry_emitter_service.resources import ProcessTreeSampler from microcosm.build.telemetry_emitter_service.runtime import EmitterService from microcosm.build.telemetry_emitter_service.spool import EventSpool -def _finite_seconds(value: str) -> float: +def _seconds(value: str) -> float: seconds = float(value) - if not math.isfinite(seconds): + if math.isnan(seconds): raise argparse.ArgumentTypeError(READY_DEADLINE_ERROR) return seconds @@ -51,7 +56,7 @@ def build_parser() -> argparse.ArgumentParser: ) parser.add_argument( "--ready-deadline", - type=_finite_seconds, + type=_seconds, help=( "Unix time at which the build stops waiting for readiness. Until " "shortly before then, startup retries spool lock contention; " @@ -71,20 +76,48 @@ def startup_deadline( The build passes its own give-up time as Unix time, the clock both processes share. The margin leaves time to bind the socket and answer the - build's ping before the build gives up. + build's ping before the build gives up. The limit bounds the wait however + far the deadline or the wall clock moves, so a service whose build has died + still gives up. """ if ready_deadline is None: return -math.inf - return now_monotonic + (ready_deadline - now_unix) - READY_DEADLINE_MARGIN_SECONDS + remaining = min(ready_deadline - now_unix, STARTUP_RETRY_LIMIT_SECONDS) + return now_monotonic + remaining - READY_DEADLINE_MARGIN_SECONDS + + +def open_registered_spool( + path: Path, + registration: dict[str, object], + *, + deadline: float, +) -> EventSpool: + """Open the spool and register this producer, waiting out lock contention. + + Each SQLite statement waits at most ``STARTUP_BUSY_TIMEOUT_SECONDS`` for + another process's lock, so this retry loop, bounded by ``deadline``, decides + when to give up. The spool it returns waits the normal + ``DATABASE_TIMEOUT_SECONDS``. + """ + + spool = retry_spool_contention( + lambda: EventSpool(path, busy_timeout_seconds=STARTUP_BUSY_TIMEOUT_SECONDS), + deadline=deadline, + ) + retry_spool_contention(lambda: spool.register(registration), deadline=deadline) + spool.busy_timeout_seconds = DATABASE_TIMEOUT_SECONDS + return spool def main(argv: list[str] | None = None) -> int: """Run the telemetry emitter service until the build disconnects. - Its stderr is the build's, so a failure is one line, never a traceback. - Exit status 75 means another process kept the spool locked past the - startup deadline; 1 means any other failure. + The socket is bound only once this build is registered, so a build that + finds the service ready can queue events. The service's stderr is the + build's, so a failure is one line, never a traceback. Exit status 75 means + another process kept the spool locked past the startup deadline; 1 means + any other failure. """ args = build_parser().parse_args(argv) @@ -96,10 +129,7 @@ def main(argv: list[str] | None = None) -> int: ) try: registration = json.loads(args.registration_json) - spool = retry_spool_contention( - lambda: EventSpool(args.spool), - deadline=deadline, - ) + spool = open_registered_spool(args.spool, registration, deadline=deadline) service = EmitterService( socket_path=args.socket, registration=registration, @@ -110,28 +140,23 @@ def main(argv: list[str] | None = None) -> int: ), sampler=ProcessTreeSampler(args.parent_pid), heartbeat_seconds=args.heartbeat_seconds, - startup_deadline=deadline, ) service.run() except Exception as error: if is_transient_spool_error(error): - print( + write_warning( SPOOL_LOCKED_WARNING.format( spool=args.spool, waited_seconds=time.monotonic() - started_at, error=describe_error(error), - ), - file=sys.stderr, - flush=True, + ) ) return SPOOL_LOCKED_EXIT_STATUS - print( + write_warning( SERVICE_FAILED_WARNING.format( error_type=type(error).__name__, error=describe_error(error), - ), - file=sys.stderr, - flush=True, + ) ) return 1 return 0 diff --git a/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/migrations.py b/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/migrations.py index f92161f42d..e744a9365f 100644 --- a/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/migrations.py +++ b/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/migrations.py @@ -13,6 +13,10 @@ from alembic.script import ScriptDirectory from sqlalchemy.engine import Connection, Engine +from microcosm.build.telemetry_emitter_service.constants import ( + DATABASE_TIMEOUT_SECONDS, + UNKNOWN_SPOOL_REVISION_ERROR, +) from microcosm.build.telemetry_emitter_service.database import create_spool_engine _MIGRATION_TARGET = "head" @@ -34,24 +38,38 @@ def alembic_config( yield config -def upgrade_spool_database(engine: Engine) -> None: +def upgrade_spool_database( + engine: Engine, + *, + busy_timeout_seconds: float = DATABASE_TIMEOUT_SECONDS, +) -> None: """Bring a spool to the packaged head, one migrating process at a time. - A spool already at head is only read, so opening it takes no write lock. - Otherwise the whole upgrade, DDL and version stamp, runs in one - ``BEGIN IMMEDIATE`` transaction. Without it two services opening a new - spool at once interleave their DDL, and one fails with "table already - exists". A service that waits for that lock finds the spool at head when - it gets it, and Alembic then changes nothing. + A spool already at head is only read, so opening it takes no write lock; + neither does refusing one stamped with a revision this checkout's history + does not contain, such as a later migration from a newer checkout sharing + the spool. Otherwise the + whole upgrade, DDL and version stamp, runs in one ``BEGIN IMMEDIATE`` + transaction whose statements each wait at most ``busy_timeout_seconds`` + for another process's lock. Without that transaction two services opening + a new spool at once interleave their DDL, and one fails with "table + already exists". A service that waits for the lock finds the spool at head + when it gets it, and Alembic then changes nothing. """ with engine.connect() as connection: current = MigrationContext.configure(connection).get_current_revision() - if current == migration_head_revision(): + head = migration_head_revision() + if current == head: return + if current is not None and current not in migration_revisions(): + raise RuntimeError( + UNKNOWN_SPOOL_REVISION_ERROR.format(revision=current, head=head) + ) migration_engine = create_spool_engine( engine.url.database, immediate_transactions=True, + busy_timeout_seconds=lambda: busy_timeout_seconds, ) try: with migration_engine.begin() as connection: @@ -77,3 +95,13 @@ def migration_head_revision() -> str | None: with alembic_config() as config: return ScriptDirectory.from_config(config).get_current_head() + + +def migration_revisions() -> frozenset[str]: + """Return every revision in the packaged migration history.""" + + with alembic_config() as config: + return frozenset( + script.revision + for script in ScriptDirectory.from_config(config).walk_revisions() + ) diff --git a/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/runtime.py b/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/runtime.py index fdcf02687c..e8ee754811 100644 --- a/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/runtime.py +++ b/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/runtime.py @@ -3,10 +3,8 @@ from __future__ import annotations import json -import math import os import socket -import sys import threading import time from collections.abc import Callable, Mapping @@ -29,10 +27,13 @@ WORKER_STEP_WARNING, ) from microcosm.build.telemetry_emitter_service.contention import ( - describe_error, is_transient_spool_error, retry_spool_contention, ) +from microcosm.build.telemetry_emitter_service.diagnostics import ( + describe_error, + write_warning, +) from microcosm.build.telemetry_emitter_service.resources import ProcessTreeSampler from microcosm.build.telemetry_emitter_service.spool import EventSpool from microcosm.build.telemetry_emitter_service.timestamps import utc_now @@ -94,7 +95,6 @@ def __init__( sampler: ProcessTreeSampler, heartbeat_seconds: float, drain_seconds: float = DEFAULT_DRAIN_SECONDS, - startup_deadline: float = -math.inf, ) -> None: self.socket_path = socket_path self.registration = dict(registration) @@ -106,20 +106,16 @@ def __init__( heartbeat_seconds, ) self.drain_seconds = max(0.0, drain_seconds) - # A time.monotonic() reading: registration retries spool lock - # contention until then. The default allows one attempt. - self.startup_deadline = startup_deadline self._stop = threading.Event() self._last_stage = STAGE_CREATED self._reported_error_types: set[str] = set() def run(self) -> None: - """Serve local messages until the client closes or exits.""" + """Serve local messages until the client closes or exits. + + The spool must already hold this producer's registration. + """ - retry_spool_contention( - lambda: self.spool.register(self.registration), - deadline=self.startup_deadline, - ) self.socket_path.parent.mkdir(parents=True, exist_ok=True) try: self.socket_path.unlink(missing_ok=True) @@ -130,7 +126,7 @@ def run(self) -> None: server.settimeout(SOCKET_ACCEPT_TIMEOUT_SECONDS) worker = threading.Thread(target=self._worker, daemon=True) worker.start() - self._serve(server) + self._serve(server, worker) worker.join(timeout=self.drain_seconds + WORKER_INTERVAL_SECONDS) finally: self.socket_path.unlink(missing_ok=True) @@ -139,8 +135,11 @@ def run(self) -> None: except OSError: pass - def _serve(self, server: socket.socket) -> None: - while not self._stop.is_set(): + def _serve(self, server: socket.socket, worker: threading.Thread) -> None: + # Only the worker notices that the build died without closing, so a + # service whose worker has stopped must stop too, or it would serve a + # dead build forever. + while not self._stop.is_set() and worker.is_alive(): try: connection, _ = server.accept() except TimeoutError: @@ -261,13 +260,11 @@ def _report(self, error: Exception) -> None: if error_type in self._reported_error_types: return self._reported_error_types.add(error_type) - print( + write_warning( WORKER_STEP_WARNING.format( error_type=error_type, error=describe_error(error), - ), - file=sys.stderr, - flush=True, + ) ) def _drain(self) -> None: diff --git a/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/spool.py b/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/spool.py index 515abae9b1..20c0b44e23 100644 --- a/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/spool.py +++ b/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/spool.py @@ -10,12 +10,13 @@ from pathlib import Path from typing import Any -from sqlalchemy import func, select -from sqlalchemy.orm import Session +from sqlalchemy import Integer, delete, func, select from microcosm.build.telemetry_emitter_service.constants import ( BATCH_SIZE, + DATABASE_TIMEOUT_SECONDS, MAX_QUEUED_BYTES, + PRUNE_BATCH_ROWS, PRUNE_INTERVAL_SECONDS, RETENTION_DAYS, UPLOAD_STATE_LOCAL_ONLY, @@ -31,7 +32,6 @@ from microcosm.build.telemetry_emitter_service.models import ( TelemetryEventRecord, TelemetryRunRecord, - serialized_json_length, ) from microcosm.build.telemetry_emitter_service.timestamps import utc_now from microcosm.build.telemetry_protocol import TELEMETRY_SCHEMA_VERSION @@ -40,16 +40,31 @@ class EventSpool: """Small SQLite queue shared by successive emitter service processes.""" - def __init__(self, path: Path | str) -> None: + def __init__( + self, + path: Path | str, + *, + busy_timeout_seconds: float = DATABASE_TIMEOUT_SECONDS, + ) -> None: self.path = Path(path) self.path.parent.mkdir(parents=True, exist_ok=True) try: self.path.parent.chmod(0o700) except OSError: pass - self._engine = create_spool_engine(self.path) + # How long one statement waits for another process's lock. Startup + # lowers it so its own retry loop decides when to give up; it applies + # from each connection's next checkout. + self.busy_timeout_seconds = busy_timeout_seconds + self._engine = create_spool_engine( + self.path, + busy_timeout_seconds=lambda: self.busy_timeout_seconds, + ) try: - upgrade_spool_database(self._engine) + upgrade_spool_database( + self._engine, + busy_timeout_seconds=busy_timeout_seconds, + ) except BaseException: self._engine.dispose() raise @@ -232,47 +247,64 @@ def prune_if_due(self) -> None: self.prune() def prune(self) -> None: - """Enforce the age and total-size retention limits.""" + """Enforce the age and total-size retention limits. + + Rows go in batches of ``PRUNE_BATCH_ROWS``, each its own short + transaction, so a large backlog never holds the spool's write lock, or + the lock this process's event path shares, for long. + """ cutoff = (datetime.now(UTC) - timedelta(days=RETENTION_DAYS)).isoformat() + expired = ( + select(TelemetryEventRecord.event_id) + .where(TelemetryEventRecord.created_at < cutoff) + .limit(PRUNE_BATCH_ROWS) + ) + while self._delete_events(expired) == PRUNE_BATCH_ROWS: + pass + oldest_first = self._oldest_events_over_size_limit() + for start in range(0, len(oldest_first), PRUNE_BATCH_ROWS): + self._delete_events(oldest_first[start : start + PRUNE_BATCH_ROWS]) with self._lock, self._session_factory.begin() as session: - expired_events = session.scalars( - select(TelemetryEventRecord).where( - TelemetryEventRecord.created_at < cutoff - ) - ) - for event in expired_events: - session.delete(event) - session.flush() - stored_characters = session.scalar( - select( - func.coalesce( - func.sum(func.length(TelemetryEventRecord.payload)), - 0, - ) - ) - ) - excess = int(stored_characters or 0) - MAX_QUEUED_BYTES - if excess > 0: - self._remove_oldest_bytes(session, excess) - expired_runs = session.scalars( - select(TelemetryRunRecord).where( + session.execute( + delete(TelemetryRunRecord) + .where( TelemetryRunRecord.updated_at < cutoff, ~TelemetryRunRecord.events.any(), ) + .execution_options(synchronize_session=False) ) - for run in expired_runs: - session.delete(run) - - @staticmethod - def _remove_oldest_bytes(session: Session, excess: int) -> None: - statement = select(TelemetryEventRecord).order_by( - TelemetryEventRecord.created_at, - TelemetryEventRecord.sequence, - ) - removed = 0 - for event in session.scalars(statement): - session.delete(event) - removed += serialized_json_length(event.payload) - if removed >= excess: - break + + def _delete_events(self, event_ids) -> int: + """Delete the selected or listed events in one transaction.""" + + with self._lock, self._session_factory.begin() as session: + return session.execute( + delete(TelemetryEventRecord) + .where(TelemetryEventRecord.event_id.in_(event_ids)) + .execution_options(synchronize_session=False) + ).rowcount + + def _oldest_events_over_size_limit(self) -> list[str]: + """Return the oldest events whose removal brings storage under the cap.""" + + stored = func.length(TelemetryEventRecord.payload, type_=Integer) + with self._lock, self._session_factory() as session: + excess = ( + session.scalar(select(func.coalesce(func.sum(stored), 0))) + - MAX_QUEUED_BYTES + ) + doomed: list[str] = [] + if excess <= 0: + return doomed + oldest = select(TelemetryEventRecord.event_id, stored).order_by( + TelemetryEventRecord.created_at, + TelemetryEventRecord.sequence, + TelemetryEventRecord.event_id, + ) + for event_id, length in session.execute(oldest): + doomed.append(event_id) + excess -= length + if excess <= 0: + break + return doomed diff --git a/packages/microcosm-build/tests/engine_free/shared/test_telemetry_emitter.py b/packages/microcosm-build/tests/engine_free/shared/test_telemetry_emitter.py index 0fe9f44c91..10411461df 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_telemetry_emitter.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_telemetry_emitter.py @@ -595,6 +595,7 @@ def test_local_socket_acknowledges_after_durable_queue( ) spool = EventSpool(tmp_path / "events.sqlite3") registration = _registration() + spool.register(registration) emitter = EmitterService( socket_path=socket_path, registration=registration, diff --git a/packages/microcosm-build/tests/engine_free/shared/test_telemetry_spool_contention.py b/packages/microcosm-build/tests/engine_free/shared/test_telemetry_spool_contention.py index deda150253..06751eb25f 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_telemetry_spool_contention.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_telemetry_spool_contention.py @@ -30,15 +30,16 @@ from alembic import command from hypothesis import HealthCheck, given, settings from hypothesis import strategies as st -from sqlalchemy import inspect +from sqlalchemy import inspect, select from sqlalchemy.exc import IntegrityError, OperationalError from microcosm.build.telemetry_emitter import LocalTelemetryEmitter, TelemetryRun from microcosm.build.telemetry_emitter_constants import TELEMETRY_SERVICE_MODULE -from microcosm.build.telemetry_emitter_service import database as database_module +from microcosm.build.telemetry_emitter_service import collector as collector_module from microcosm.build.telemetry_emitter_service import main as main_module from microcosm.build.telemetry_emitter_service import runtime as runtime_module from microcosm.build.telemetry_emitter_service import spool as spool_module +from microcosm.build.telemetry_emitter_service.collector import CollectorDelivery from microcosm.build.telemetry_emitter_service.constants import ( DATABASE_TIMEOUT_SECONDS, PRUNE_INTERVAL_SECONDS, @@ -47,13 +48,18 @@ SPOOL_LOCKED_EXIT_STATUS, SPOOL_RETRY_INITIAL_SECONDS, SPOOL_RETRY_MAX_SECONDS, + STARTUP_BUSY_TIMEOUT_SECONDS, + STARTUP_RETRY_LIMIT_SECONDS, ) from microcosm.build.telemetry_emitter_service.contention import ( - describe_error, is_transient_spool_error, retry_spool_contention, ) from microcosm.build.telemetry_emitter_service.database import create_spool_engine +from microcosm.build.telemetry_emitter_service.diagnostics import ( + describe_error, + write_warning, +) from microcosm.build.telemetry_emitter_service.migrations import ( alembic_config, current_database_revision, @@ -62,6 +68,7 @@ from microcosm.build.telemetry_emitter_service.models import ( SpoolModel, TelemetryEventRecord, + TelemetryRunRecord, ) from microcosm.build.telemetry_emitter_service.runtime import EmitterService from microcosm.build.telemetry_emitter_service.spool import EventSpool @@ -183,9 +190,8 @@ def _lock_error(code: int = sqlite3.SQLITE_BUSY) -> OperationalError: def test_service_registers_once_a_held_spool_lock_is_released( tmp_path, monkeypatch, capsys, state ) -> None: - """In process, with a short busy timeout so the lock outlasts many waits.""" + """In process: the lock outlasts many of startup's short SQLite waits.""" - monkeypatch.setattr(database_module, "DATABASE_TIMEOUT_SECONDS", 0.1) spool_path = tmp_path / "events.sqlite3" _prepare_spool(spool_path, state) socket_path = _short_socket_path() @@ -217,7 +223,7 @@ def wrapped(*args, **kwargs): with _write_lock(spool_path): service.start() - time.sleep(1.0) + time.sleep(1.5) assert not socket_path.exists() assert service.is_alive() released_at = time.monotonic() @@ -232,8 +238,8 @@ def wrapped(*args, **kwargs): assert status == [0] failures = [error for _, _, error in outcomes if error is not None] - # Ten busy timeouts fit in the hold, so a single SQLite wait cannot - # explain the success: startup retried. + # Six startup waits fit in the hold, so a single SQLite wait cannot explain + # the success: startup retried. assert len(failures) >= 2 assert all(is_transient_spool_error(error) for error in failures) name, finished_at, error = outcomes[-1] @@ -243,10 +249,9 @@ def wrapped(*args, **kwargs): assert "Traceback" not in capsys.readouterr().err -def test_service_exits_75_with_one_line_while_the_spool_stays_locked( - tmp_path, monkeypatch, capsys +def test_service_exits_75_with_one_line_before_the_build_stops_waiting( + tmp_path, capsys ) -> None: - monkeypatch.setattr(database_module, "DATABASE_TIMEOUT_SECONDS", 0.1) spool_path = tmp_path / "events.sqlite3" _prepare_spool(spool_path, "at_head") socket_path = _short_socket_path() @@ -262,7 +267,9 @@ def test_service_exits_75_with_one_line_while_the_spool_stays_locked( waited = time.monotonic() - started assert status == SPOOL_LOCKED_EXIT_STATUS - assert waited < 0.5 + 2 * DATABASE_TIMEOUT_SECONDS + # The build would give up at margin + 0.5 s; the service has already + # said why and exited. + assert waited < READY_DEADLINE_MARGIN_SECONDS + 0.5 assert not socket_path.exists() error_output = capsys.readouterr().err assert error_output.count("\n") == 1 @@ -293,25 +300,64 @@ def test_unexpected_service_failure_is_one_line_with_status_1(tmp_path, capsys) assert "Traceback" not in error_output -@pytest.mark.parametrize("value", ["nan", "inf", "-inf"]) -def test_ready_deadline_must_be_finite(value) -> None: +def _parse_ready_deadline(value: str): + return main_module.build_parser().parse_args( + [ + "--socket", + "s", + "--spool", + "q", + "--registration-json", + "{}", + "--parent-pid", + "1", + "--ready-deadline", + value, + ] + ) + + +def test_ready_deadline_rejects_nan_and_caps_infinity() -> None: with pytest.raises(SystemExit) as raised: - main_module.build_parser().parse_args( - [ - "--socket", - "s", - "--spool", - "q", - "--registration-json", - "{}", - "--parent-pid", - "1", - "--ready-deadline", - value, - ] - ) + _parse_ready_deadline("nan") assert raised.value.code == 2 + arguments = _parse_ready_deadline("inf") + deadline = main_module.startup_deadline( + arguments.ready_deadline, now_unix=time.time(), now_monotonic=100.0 + ) + assert deadline == ( + 100.0 + STARTUP_RETRY_LIMIT_SECONDS - READY_DEADLINE_MARGIN_SECONDS + ) + + +def test_startup_statements_wait_briefly_and_the_spool_then_waits_normally( + tmp_path, +) -> None: + spool_path = tmp_path / "events.sqlite3" + _prepare_spool(spool_path, "at_head") + spool = EventSpool(spool_path, busy_timeout_seconds=STARTUP_BUSY_TIMEOUT_SECONDS) + try: + with _write_lock(spool_path): + started = time.monotonic() + with pytest.raises(OperationalError): + spool.register(_registration()) + waited = time.monotonic() - started + assert waited < 4 * STARTUP_BUSY_TIMEOUT_SECONDS + finally: + _close(spool) + + registered = main_module.open_registered_spool( + spool_path, _registration(), deadline=time.monotonic() + 10 + ) + try: + assert registered.busy_timeout_seconds == DATABASE_TIMEOUT_SECONDS + with registered._engine.connect() as connection: + milliseconds = connection.exec_driver_sql("PRAGMA busy_timeout").scalar() + assert milliseconds == DATABASE_TIMEOUT_SECONDS * 1000 + finally: + _close(registered) + def test_build_keeps_telemetry_when_the_spool_is_locked_past_a_busy_wait( tmp_path, capfd, real_local_telemetry @@ -448,8 +494,12 @@ def test_startup_deadline_is_the_ready_deadline_less_the_margin_on_this_clock( deadline = main_module.startup_deadline( ready_deadline, now_unix=now_unix, now_monotonic=now_monotonic ) - remaining = ready_deadline - now_unix - READY_DEADLINE_MARGIN_SECONDS - assert deadline - now_monotonic == pytest.approx(remaining, abs=1e-6) + remaining = min(ready_deadline - now_unix, STARTUP_RETRY_LIMIT_SECONDS) + assert deadline - now_monotonic == pytest.approx( + remaining - READY_DEADLINE_MARGIN_SECONDS, abs=1e-6 + ) + # However far away the build's deadline is, startup stops within the limit. + assert deadline - now_monotonic <= STARTUP_RETRY_LIMIT_SECONDS assert ( main_module.startup_deadline( None, now_unix=now_unix, now_monotonic=now_monotonic @@ -988,3 +1038,252 @@ def append(registration, event, *, resources=None): assert service._attempt(service._append_unexpected_exit) assert len(attempts) == 4 assert attempts[-1] < 15 + + +def test_service_stops_serving_when_its_worker_dies(tmp_path) -> None: + """Only the worker notices a dead build, so serving must not outlive it.""" + + spool = EventSpool(tmp_path / "events.sqlite3") + spool.register(_registration()) + socket_path = _short_socket_path() + service = EmitterService( + socket_path=socket_path, + registration=_registration(), + spool=spool, + delivery=SimpleNamespace(flush_once=lambda: False), + sampler=SimpleNamespace(sample=dict, parent_alive=lambda: True), + heartbeat_seconds=60, + drain_seconds=0, + ) + service._worker = lambda: None # returns at once without stopping the service + serving = threading.Thread(target=service.run) + try: + serving.start() + serving.join(timeout=10) + assert not serving.is_alive() + assert not socket_path.exists() + finally: + _close(spool) + + +class _GoneStream: + """A stderr whose reader has exited, like a build piped through ``tee``.""" + + def write(self, text): + raise BrokenPipeError(32, "Broken pipe") + + def flush(self): + raise BrokenPipeError(32, "Broken pipe") + + +def test_warnings_never_raise_when_the_builds_stderr_is_gone(monkeypatch) -> None: + monkeypatch.setattr(sys, "stderr", _GoneStream()) + write_warning("the service keeps running") + closed = io.StringIO() + closed.close() + monkeypatch.setattr(sys, "stderr", closed) + write_warning("the service keeps running") + + monkeypatch.setattr(sys, "stderr", _GoneStream()) + service = EmitterService( + socket_path=Path("/unused"), + registration=_registration(), + spool=SimpleNamespace(), + delivery=SimpleNamespace(), + sampler=SimpleNamespace(), + heartbeat_seconds=60, + ) + + def bug() -> None: + raise RuntimeError("collector bug") + + assert service._attempt(bug) is False + + +def test_delivery_backs_off_after_an_unexpected_error(monkeypatch) -> None: + """The worker survives the error, so it must not turn into a request a tick.""" + + requests: list[str] = [] + + def post(url, payload, bearer_token, **kwargs): + requests.append(url) + return 200, {"access_token": "collector-token", "expires_in": None} + + monkeypatch.setattr(collector_module, "_http_post", post) + monkeypatch.setenv("HF_TOKEN", "hf-test-token") + delivery = CollectorDelivery( + SimpleNamespace(pending_runs=lambda: [_registration()]), + development_collector_url=_LOOPBACK_COLLECTOR, + ) + + with pytest.raises(TypeError): + delivery.flush_once() + assert delivery.flush_once() is False + assert len(requests) == 1 + + +def test_delivery_retries_a_locked_spool_on_the_next_tick(monkeypatch) -> None: + attempts: list[int] = [] + + def pending_runs(): + attempts.append(1) + raise _lock_error() + + delivery = CollectorDelivery( + SimpleNamespace(pending_runs=pending_runs), + development_collector_url=_LOOPBACK_COLLECTOR, + ) + for _ in range(3): + with pytest.raises(OperationalError): + delivery.flush_once() + assert len(attempts) == 3 + + +def test_a_spool_from_an_unknown_migration_is_refused_without_the_write_lock( + tmp_path, +) -> None: + spool_path = tmp_path / "events.sqlite3" + _prepare_spool(spool_path, "at_head") + connection = sqlite3.connect(spool_path) + connection.execute("UPDATE alembic_version SET version_num = 'from_elsewhere'") + connection.commit() + connection.close() + + with _write_lock(spool_path): + started = time.monotonic() + with pytest.raises(RuntimeError, match="'from_elsewhere' is not in"): + EventSpool(spool_path) + assert time.monotonic() - started < DATABASE_TIMEOUT_SECONDS + + +# --- Retention in short batches, matching the single-pass rule ---------------- + +_STAMP_ORIGIN = datetime(2026, 10, 9, 12, tzinfo=UTC) + + +def _stamp(age_days: float, index: int) -> str: + # The index keeps stamps distinct, so the oldest-first order is unambiguous. + return ( + _STAMP_ORIGIN - timedelta(days=age_days) + timedelta(microseconds=index) + ).isoformat() + + +@settings(max_examples=40, deadline=None) +@given( + events=st.lists( + st.tuples( + st.integers(0, 2), # run + st.floats(0, 2 * RETENTION_DAYS), # age in days + st.integers(0, 60), # padding characters + ), + max_size=18, + ), + run_ages=st.lists(st.floats(0, 2 * RETENTION_DAYS), min_size=3, max_size=3), + cap_fraction=st.floats(0, 1.2), + batch_rows=st.integers(1, 4), +) +def test_batched_prune_matches_the_single_pass_retention_rule( + tmp_path_factory, events, run_ages, cap_fraction, batch_rows +) -> None: + """Differential: the batched prune keeps exactly what one pass would. + + The reference is the retention rule the single-transaction prune applied: + drop events past ``RETENTION_DAYS``; then, oldest first, drop events until + the stored payloads fit ``MAX_QUEUED_BYTES``; then drop expired runs that + have no events left. + """ + + from unittest import mock + + spool = EventSpool(tmp_path_factory.mktemp("prune") / "events.sqlite3") + keys = [(f"run-{index}", f"producer-{index}") for index in range(3)] + rows: list[dict[str, object]] = [] + try: + with spool._session_factory.begin() as session: + for index, (run_id, producer_id) in enumerate(keys): + session.add( + TelemetryRunRecord( + run_id=run_id, + producer_id=producer_id, + registration={"run_id": run_id, "producer_id": producer_id}, + next_sequence=1, + upload_state="pending", + local_only_reason=None, + updated_at=_stamp(run_ages[index], index), + ) + ) + for index, (run, age, padding) in enumerate(events): + run_id, producer_id = keys[run] + payload = {"pad": "x" * padding} + row = { + "event_id": f"event-{index:02d}", + "run": keys[run], + "created_at": _stamp(age, index), + "sequence": index, + "length": len( + json.dumps(payload, separators=(",", ":"), sort_keys=True) + ), + } + rows.append(row) + session.add( + TelemetryEventRecord( + event_id=row["event_id"], + run_id=run_id, + producer_id=producer_id, + sequence=index, + payload=payload, + created_at=row["created_at"], + ) + ) + cutoff = (_STAMP_ORIGIN - timedelta(days=RETENTION_DAYS)).isoformat() + live = [row for row in rows if row["created_at"] >= cutoff] + cap = int(cap_fraction * sum(row["length"] for row in live)) + batches: list[int] = [] + delete_events = spool._delete_events + + def recording_delete(event_ids) -> int: + deleted = delete_events(event_ids) + batches.append(deleted) + return deleted + + spool._delete_events = recording_delete + frozen_now = mock.Mock(wraps=datetime) + frozen_now.now.return_value = _STAMP_ORIGIN + with ( + mock.patch.object(spool_module, "MAX_QUEUED_BYTES", cap), + mock.patch.object(spool_module, "PRUNE_BATCH_ROWS", batch_rows), + mock.patch.object(spool_module, "datetime", frozen_now), + ): + spool.prune() + with spool._session_factory() as session: + kept_events = set( + session.scalars(select(TelemetryEventRecord.event_id)).all() + ) + kept_runs = set( + session.execute( + select( + TelemetryRunRecord.run_id, + TelemetryRunRecord.producer_id, + ) + ).all() + ) + finally: + _close(spool) + + expected_events = {row["event_id"] for row in live} + excess = sum(row["length"] for row in live) - cap + for row in sorted(live, key=lambda row: (row["created_at"], row["sequence"])): + if excess <= 0: + break + expected_events.discard(row["event_id"]) + excess -= row["length"] + expected_runs = { + key + for index, key in enumerate(keys) + if _stamp(run_ages[index], index) >= cutoff + or any(row["run"] == key and row["event_id"] in expected_events for row in rows) + } + assert kept_events == expected_events + assert kept_runs == expected_runs + # Each transaction deleted at most one batch. + assert all(deleted <= batch_rows for deleted in batches) From e87cf03bcc03e0ebe7dd6bf4340203ddf4bd92fd Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 9 Oct 2026 10:50:59 -0400 Subject: [PATCH 03/13] Describe the final startup and retention behaviour in the docs Co-Authored-By: Claude Opus 5.5 --- README.md | 6 +++--- .../telemetry-spool-lock-startup.fixed.md | 6 +++++- .../telemetry_emitter_service/migrations.py | 20 +++++++++---------- 3 files changed, 18 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index da76ea0d12..4dd5b985da 100644 --- a/README.md +++ b/README.md @@ -51,9 +51,9 @@ service. It reports live progress to the hosted collector when the operator's existing Hugging Face login is accepted; otherwise it retains the events locally and the build continues. Every build on a host shares one local event spool; a service that finds it locked by another process keeps retrying -for as long as the build waits for it to become ready, and one that still -cannot register prints a single warning line, without a traceback, before the -build continues without hosted telemetry. This live event path is independent +until shortly before the build stops waiting for it, and one that still cannot +register prints a single warning line, without a traceback, before the build +continues without hosted telemetry. This live event path is independent of the staging run files described below. US fiscal refresh builds also write pre-release staging run files **by diff --git a/changelog.d/telemetry-spool-lock-startup.fixed.md b/changelog.d/telemetry-spool-lock-startup.fixed.md index 3dfd0bc83f..31c65a653a 100644 --- a/changelog.d/telemetry-spool-lock-startup.fixed.md +++ b/changelog.d/telemetry-spool-lock-startup.fixed.md @@ -1 +1,5 @@ -A build on a busy host no longer loses its hosted telemetry when another build holds the shared local spool (`~/.cache/microcosm/telemetry/events.sqlite3`) at the moment its emitter service starts. Before, the service's registration gave up after SQLite's 5 s busy wait, the service exited and a SQLAlchemy traceback landed on the build's stderr. Now the build tells the service when it will stop waiting (`--ready-deadline`), and the service retries lock errors with jittered backoff until shortly before then; if the spool is still locked it prints one warning line and exits with status 75. Opening a spool that is already at the current schema takes no write lock: retention pruning moved from spool open and every append to the delivery worker. Schema migrations now run in one `BEGIN IMMEDIATE` transaction, so concurrent first opens of a new spool no longer fail with "table already exists" (about half of eight simultaneous opens did before) and an interrupted migration leaves no partial schema. The delivery worker skips a failed step for one tick instead of dying, so heartbeats, delivery and the check that records a killed build keep running, and errors other than lock contention print one line per error type rather than a thread traceback. +A build on a busy host no longer loses its hosted telemetry when another build holds the shared local spool (`~/.cache/microcosm/telemetry/events.sqlite3`) while its emitter service starts. Before, the service's registration gave up after SQLite's 5 s busy wait, the service exited, and a SQLAlchemy traceback landed on the build's stderr. Now the build passes the time it will stop waiting (`--ready-deadline`). The service retries lock errors with jittered backoff until 2 s before then, with each SQLite statement waiting at most 0.25 s, and never for more than 60 s in all. If the spool is still locked, it prints one warning line and exits with status 75 before the build gives up. Any other startup failure is also one line, with status 1. + +Opening a spool already at the current schema takes no write lock. Retention pruning moved from spool open and every append to the delivery worker, and it deletes in batches of 500 rows, each in its own short transaction. Migrations run in one `BEGIN IMMEDIATE` transaction. Concurrent first opens of a new spool no longer fail with "table already exists"; about half of eight simultaneous opens did before. An interrupted migration leaves no partial schema, and a spool stamped by a migration this checkout does not know is refused without taking the lock. + +The delivery worker skips a failed step for one tick instead of dying. Heartbeats, delivery, and the check that records a killed build keep running. Lock contention is retried silently, and any other error prints one line per error type. An unexpected delivery error backs off like a collector failure. Warnings never raise on a closed stderr, and the service stops serving if its worker ever dies. diff --git a/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/migrations.py b/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/migrations.py index e744a9365f..3550c07149 100644 --- a/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/migrations.py +++ b/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/migrations.py @@ -45,16 +45,16 @@ def upgrade_spool_database( ) -> None: """Bring a spool to the packaged head, one migrating process at a time. - A spool already at head is only read, so opening it takes no write lock; - neither does refusing one stamped with a revision this checkout's history - does not contain, such as a later migration from a newer checkout sharing - the spool. Otherwise the - whole upgrade, DDL and version stamp, runs in one ``BEGIN IMMEDIATE`` - transaction whose statements each wait at most ``busy_timeout_seconds`` - for another process's lock. Without that transaction two services opening - a new spool at once interleave their DDL, and one fails with "table - already exists". A service that waits for the lock finds the spool at head - when it gets it, and Alembic then changes nothing. + A spool already at head is only read, so opening it takes no write lock. + Neither does refusing one stamped with a revision missing from this + checkout's history, such as a later migration from a newer checkout that + shares the spool. Otherwise the whole upgrade, DDL and version stamp, runs + in one ``BEGIN IMMEDIATE`` transaction whose statements each wait at most + ``busy_timeout_seconds`` for another process's lock. Without that + transaction two services opening a new spool at once interleave their DDL, + and one fails with "table already exists". A service that waits for the + lock finds the spool at head when it gets it, and Alembic then changes + nothing. """ with engine.connect() as connection: From bf2ec8e5775df486b566d6b6aae2486eab5f3bfe Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 9 Oct 2026 11:08:08 -0400 Subject: [PATCH 04/13] Make the contention properties reach their boundaries and never hang The retry and classifier properties only sometimes generated the cases that tell a correct implementation from a broken one: a wait crossing a deadline 0.04 s away, a second retry, an extended SQLITE_BUSY code. Lock errors and short attempts are now weighted up, busy and locked primaries are drawn with every extension, and the boundary cases are explicit examples. Threaded tests stop their service in a finally block and use daemon threads, so a failing run cannot hang pytest at exit. Co-Authored-By: Claude Opus 5.5 --- .../shared/test_telemetry_spool_contention.py | 78 +++++++++++++------ 1 file changed, 55 insertions(+), 23 deletions(-) diff --git a/packages/microcosm-build/tests/engine_free/shared/test_telemetry_spool_contention.py b/packages/microcosm-build/tests/engine_free/shared/test_telemetry_spool_contention.py index 06751eb25f..e8feac6f5e 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_telemetry_spool_contention.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_telemetry_spool_contention.py @@ -28,7 +28,7 @@ import alembic.op import pytest from alembic import command -from hypothesis import HealthCheck, given, settings +from hypothesis import HealthCheck, example, given, settings from hypothesis import strategies as st from sqlalchemy import inspect, select from sqlalchemy.exc import IntegrityError, OperationalError @@ -218,23 +218,27 @@ def wrapped(*args, **kwargs): socket_path, spool_path, ready_deadline=time.time() + 60 ) service = threading.Thread( - target=lambda: status.append(main_module.main(arguments)) + target=lambda: status.append(main_module.main(arguments)), daemon=True ) - with _write_lock(spool_path): - service.start() - time.sleep(1.5) - assert not socket_path.exists() - assert service.is_alive() - released_at = time.monotonic() - deadline = time.monotonic() + 20 - while not _ping(socket_path): - assert time.monotonic() < deadline, "service never became ready" - assert service.is_alive(), f"service exited with {status}" - time.sleep(0.02) - ready_at = time.monotonic() - _close_service(socket_path) - service.join(timeout=10) + try: + with _write_lock(spool_path): + service.start() + time.sleep(1.5) + assert not socket_path.exists() + assert service.is_alive() + released_at = time.monotonic() + deadline = time.monotonic() + 20 + while not _ping(socket_path): + assert time.monotonic() < deadline, "service never became ready" + assert service.is_alive(), f"service exited with {status}" + time.sleep(0.02) + ready_at = time.monotonic() + finally: + # Stop the service even when an assertion above failed. + if socket_path.exists(): + _close_service(socket_path) + service.join(timeout=10) assert status == [0] failures = [error for _, _, error in outcomes if error is not None] @@ -384,7 +388,7 @@ def start() -> None: ) started["returned_at"] = time.monotonic() - client = threading.Thread(target=start) + client = threading.Thread(target=start, daemon=True) with _write_lock(spool_path): client.start() deadline = time.monotonic() + 60 @@ -567,7 +571,7 @@ def open_spool() -> None: except BaseException as error: failures.append(error) - second = threading.Thread(target=open_spool) + second = threading.Thread(target=open_spool, daemon=True) try: with migrator.begin() as connection: second.start() @@ -725,7 +729,24 @@ def prune() -> None: # --- Lock-error classification ------------------------------------------------ -@given(code=st.integers(0, 2**16)) +# Extended result codes keep the primary code in the low byte, so draw busy and +# locked primaries with every extension as well as arbitrary codes. +_RESULT_CODES = st.one_of( + st.integers(0, 2**16), + st.builds( + lambda primary, extension: primary | extension << 8, + st.sampled_from([sqlite3.SQLITE_BUSY, sqlite3.SQLITE_LOCKED]), + st.integers(0, 255), + ), +) + + +@given(code=_RESULT_CODES) +@example(code=261) # SQLITE_BUSY_RECOVERY +@example(code=262) # SQLITE_LOCKED_SHAREDCACHE +@example(code=517) # SQLITE_BUSY_SNAPSHOT +@example(code=773) # SQLITE_BUSY_TIMEOUT +@example(code=1555) # SQLITE_CONSTRAINT_PRIMARYKEY def test_only_busy_and_locked_result_codes_are_transient(code) -> None: error = sqlite3.OperationalError("message") error.sqlite_errorcode = code @@ -766,18 +787,26 @@ def test_real_sqlite_errors_are_classified_by_what_retrying_can_fix(tmp_path) -> # --- The retry helper --------------------------------------------------------- -_OUTCOME = st.sampled_from(["ok", "locked", "fatal"]) +# Lock errors are weighted up and attempts are often short, so most examples +# retry several times and many give up within one wait of the deadline. +_OUTCOME = st.sampled_from(["locked", "locked", "locked", "ok", "fatal"]) +_ATTEMPT_SECONDS = st.one_of(st.just(0.0), st.floats(0, 0.3), st.floats(0, 6)) +_LOCKED_THEN_OK = [("locked", 0.0)] * 6 + [("ok", 0.0)] @given( outcomes=st.lists( - st.tuples(_OUTCOME, st.floats(0, 6)), + st.tuples(_OUTCOME, _ATTEMPT_SECONDS), min_size=1, max_size=40, ), - window=st.one_of(st.just(-math.inf), st.floats(-5, 60)), + window=st.one_of(st.just(-math.inf), st.floats(-1, 3), st.floats(-5, 60)), jitter_fraction=st.floats(0, 1), ) +# The first retry's full wait (0.05 s) would cross a deadline 0.04 s away. +@example(outcomes=_LOCKED_THEN_OK, window=0.04, jitter_fraction=1.0) +# Six retries fit, so the delays must double up to the cap. +@example(outcomes=_LOCKED_THEN_OK, window=10.0, jitter_fraction=0.5) def test_retry_respects_its_deadline_and_backoff(outcomes, window, jitter_fraction): clock = SimpleNamespace(now=100.0) deadline = clock.now + window @@ -1056,13 +1085,16 @@ def test_service_stops_serving_when_its_worker_dies(tmp_path) -> None: drain_seconds=0, ) service._worker = lambda: None # returns at once without stopping the service - serving = threading.Thread(target=service.run) + serving = threading.Thread(target=service.run, daemon=True) try: serving.start() serving.join(timeout=10) assert not serving.is_alive() assert not socket_path.exists() finally: + # A failing run must not leave a server thread behind. + service._stop.set() + serving.join(timeout=5) _close(spool) From 4e088a13979ce73505fff019c2d10890b00b7001 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 9 Oct 2026 11:55:38 -0400 Subject: [PATCH 05/13] Check the shutdown drain against any failure sequence The worker property reached the drain with failures left in its script only some of the time, so an unguarded drain survived one of three fresh Hypothesis runs. A dedicated property drives the drain directly: it never raises, never runs past its deadline, and reports each error type once. Co-Authored-By: Claude Opus 5.5 --- .../shared/test_telemetry_spool_contention.py | 56 +++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/packages/microcosm-build/tests/engine_free/shared/test_telemetry_spool_contention.py b/packages/microcosm-build/tests/engine_free/shared/test_telemetry_spool_contention.py index e8feac6f5e..391a7ce704 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_telemetry_spool_contention.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_telemetry_spool_contention.py @@ -1036,6 +1036,62 @@ def recorded_unexpected_exit() -> None: assert not any("Traceback" in line for line in lines) +@settings(suppress_health_check=[HealthCheck.function_scoped_fixture]) +@given( + script=st.lists(st.sampled_from(["ok", "locked", "runtime", "value"]), max_size=60), + deliverable=st.lists(st.booleans(), max_size=60), + drain_seconds=st.sampled_from([0.0, 0.4, 2.0, 5.0]), +) +def test_drain_survives_any_failure_and_keeps_its_deadline( + monkeypatch, script, deliverable, drain_seconds +) -> None: + clock = SimpleNamespace(now=0.0) + monkeypatch.setattr( + runtime_module, + "time", + SimpleNamespace( + monotonic=lambda: clock.now, + sleep=lambda seconds: setattr(clock, "now", clock.now + seconds), + ), + ) + outcomes = _Outcomes(script) + answers = iter(deliverable) + checks: list[float] = [] + + def has_deliverable() -> bool: + checks.append(clock.now) + outcomes.next() + return next(answers, False) + + def flush_once() -> bool: + clock.now += 0.01 + outcomes.next() + return False + + service = EmitterService( + socket_path=Path("/unused"), + registration=_registration(), + spool=SimpleNamespace(has_deliverable=has_deliverable), + delivery=SimpleNamespace(flush_once=flush_once), + sampler=SimpleNamespace(), + heartbeat_seconds=60, + drain_seconds=drain_seconds, + ) + error_output = io.StringIO() + + with contextlib.redirect_stderr(error_output): + service._drain() # never raises + + # It never runs past its deadline, and checks again after every failure + # until nothing is deliverable or the deadline passes. + assert clock.now <= drain_seconds + 0.01 + 1e-9 + assert all(check < drain_seconds for check in checks) + reported = { + name for name in outcomes.raised if name in {"RuntimeError", "ValueError"} + } + assert len(error_output.getvalue().splitlines()) == len(reported) + + def test_unexpected_exit_waits_out_lock_contention(monkeypatch) -> None: clock = SimpleNamespace(now=0.0) monkeypatch.setattr( From fe736772334cd02c0fff30ca0fa561ddc2af2de7 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 9 Oct 2026 12:24:44 -0400 Subject: [PATCH 06/13] Tell a clean close from a crash, and keep every exit path to one line From the in-session review: - A build that sent close and exited while a worker step was running was recorded as an unexpected exit after its completion. The worker now checks the build only while it has not been closed (pre-existing). - A parent check that raises is reported and treated as a dead build, so the worker never dies on it. - Invalid arguments print one line (exit 2) instead of a usage block. - The client passes --ready-deadline= as one token, so a past or negative deadline parses. - SIGINT ends the service at once, as SIGTERM does, instead of printing a KeyboardInterrupt traceback. Co-Authored-By: Claude Opus 5.5 --- .../src/microcosm/build/telemetry_emitter.py | 7 +- .../telemetry_emitter_service/__main__.py | 6 + .../telemetry_emitter_service/constants.py | 3 + .../build/telemetry_emitter_service/main.py | 16 ++- .../telemetry_emitter_service/runtime.py | 13 +- .../shared/test_telemetry_spool_contention.py | 124 +++++++++++++++--- 6 files changed, 147 insertions(+), 22 deletions(-) diff --git a/packages/microcosm-build/src/microcosm/build/telemetry_emitter.py b/packages/microcosm-build/src/microcosm/build/telemetry_emitter.py index 8bd365d4a3..65d4518515 100644 --- a/packages/microcosm-build/src/microcosm/build/telemetry_emitter.py +++ b/packages/microcosm-build/src/microcosm/build/telemetry_emitter.py @@ -194,9 +194,10 @@ def start( "--heartbeat-seconds", str(heartbeat_seconds), # The service retries spool lock contention until just before - # this wall-clock time, when the wait below gives up. - "--ready-deadline", - repr(datetime.now(UTC).timestamp() + startup_timeout_seconds), + # this wall-clock time, when the wait below gives up. One token, + # so a past or negative deadline is not read as an option. + "--ready-deadline=" + + repr(datetime.now(UTC).timestamp() + startup_timeout_seconds), ] if development_collector_url is not None: command.extend( diff --git a/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/__main__.py b/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/__main__.py index 2981038314..3b150ef5c1 100644 --- a/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/__main__.py +++ b/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/__main__.py @@ -1,5 +1,11 @@ """Module execution entry point for the telemetry emitter service.""" +import signal + from microcosm.build.telemetry_emitter_service.main import main +# The service runs in its own session, so an interrupt reaches it only when +# sent on purpose. End at once, as SIGTERM does, rather than print a +# KeyboardInterrupt traceback on the build's stderr. +signal.signal(signal.SIGINT, signal.SIG_DFL) raise SystemExit(main()) diff --git a/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/constants.py b/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/constants.py index 8b245dfe03..f54620415c 100644 --- a/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/constants.py +++ b/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/constants.py @@ -91,6 +91,9 @@ "in its spool {spool}: another process kept the spool locked for " "{waited_seconds:.1f} s ({error})." ) +SERVICE_ARGUMENTS_WARNING: Final = ( + "warning: the local telemetry emitter service could not start: {error}" +) SERVICE_FAILED_WARNING: Final = ( "warning: the local telemetry emitter service stopped: {error_type}: {error}" ) diff --git a/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/main.py b/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/main.py index 4005078b62..068180bd27 100644 --- a/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/main.py +++ b/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/main.py @@ -7,6 +7,7 @@ import math import time from pathlib import Path +from typing import NoReturn from microcosm.build.telemetry_emitter_service.collector import CollectorDelivery from microcosm.build.telemetry_emitter_service.constants import ( @@ -14,6 +15,7 @@ DEFAULT_HEARTBEAT_SECONDS, READY_DEADLINE_ERROR, READY_DEADLINE_MARGIN_SECONDS, + SERVICE_ARGUMENTS_WARNING, SERVICE_FAILED_WARNING, SPOOL_LOCKED_EXIT_STATUS, SPOOL_LOCKED_WARNING, @@ -40,10 +42,18 @@ def _seconds(value: str) -> float: return seconds +class _ArgumentParser(argparse.ArgumentParser): + """Report invalid arguments in one line, like every other startup failure.""" + + def error(self, message: str) -> NoReturn: + write_warning(SERVICE_ARGUMENTS_WARNING.format(error=message)) + raise SystemExit(2) + + def build_parser() -> argparse.ArgumentParser: """Create the service command-line parser.""" - parser = argparse.ArgumentParser() + parser = _ArgumentParser() parser.add_argument("--socket", type=Path, required=True) parser.add_argument("--spool", type=Path, required=True) parser.add_argument("--development-collector-url") @@ -116,8 +126,8 @@ def main(argv: list[str] | None = None) -> int: The socket is bound only once this build is registered, so a build that finds the service ready can queue events. The service's stderr is the build's, so a failure is one line, never a traceback. Exit status 75 means - another process kept the spool locked past the startup deadline; 1 means - any other failure. + another process kept the spool locked past the startup deadline, 2 that + the arguments were invalid, and 1 any other failure. """ args = build_parser().parse_args(argv) diff --git a/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/runtime.py b/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/runtime.py index e8ee754811..0954385f18 100644 --- a/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/runtime.py +++ b/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/runtime.py @@ -213,12 +213,23 @@ def _worker(self) -> None: next_heartbeat = now + self.heartbeat_seconds self._attempt(self.spool.prune_if_due) self._attempt(self.delivery.flush_once) - if not self.sampler.parent_alive(): + # A build that closed while this tick ran may already have exited + # cleanly; only a build that never closed died unexpectedly. + if not self._stop.is_set() and not self._parent_alive(): self._attempt(self._append_unexpected_exit) self._stop.set() break self._drain() + def _parent_alive(self) -> bool: + # A parent that cannot be checked is treated as gone: the alternative + # is a service that may outlive its build. + try: + return self.sampler.parent_alive() + except Exception as error: + self._report(error) + return False + def _append_heartbeat(self) -> None: self.spool.append( self.registration, diff --git a/packages/microcosm-build/tests/engine_free/shared/test_telemetry_spool_contention.py b/packages/microcosm-build/tests/engine_free/shared/test_telemetry_spool_contention.py index 391a7ce704..0c06f7773b 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_telemetry_spool_contention.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_telemetry_spool_contention.py @@ -14,6 +14,7 @@ import json import math import os +import signal import socket import sqlite3 import subprocess @@ -173,7 +174,7 @@ def _service_arguments( _LOOPBACK_COLLECTOR, ] if ready_deadline is not None: - arguments += ["--ready-deadline", repr(ready_deadline)] + arguments.append(f"--ready-deadline={ready_deadline!r}") return arguments @@ -321,10 +322,15 @@ def _parse_ready_deadline(value: str): ) -def test_ready_deadline_rejects_nan_and_caps_infinity() -> None: +def test_ready_deadline_rejects_nan_and_caps_infinity(capsys) -> None: with pytest.raises(SystemExit) as raised: _parse_ready_deadline("nan") assert raised.value.code == 2 + # Invalid arguments are one line on the build's stderr, not a usage block. + assert capsys.readouterr().err == ( + "warning: the local telemetry emitter service could not start: " + "argument --ready-deadline: ready deadline must be a number of seconds\n" + ) arguments = _parse_ready_deadline("inf") deadline = main_module.startup_deadline( @@ -434,7 +440,7 @@ def test_service_process_reports_a_locked_spool_in_one_line(tmp_path) -> None: sys.executable, "-m", TELEMETRY_SERVICE_MODULE, - # A deadline already past allows one attempt: SQLite's own wait. + # A deadline already past allows one attempt. *_service_arguments( socket_path, spool_path, ready_deadline=time.time() ), @@ -454,6 +460,46 @@ def test_service_process_reports_a_locked_spool_in_one_line(tmp_path) -> None: assert not socket_path.exists() +def test_an_interrupted_service_exits_without_a_traceback(tmp_path) -> None: + """SIGINT ends the real process as SIGTERM does, with nothing on stderr.""" + + spool_path = tmp_path / "spool" / "events.sqlite3" + _prepare_spool(spool_path, "at_head") + # Opening the spool sets its directory to 0o700: the service is retrying. + spool_path.parent.chmod(0o750) + socket_path = _short_socket_path() + with _write_lock(spool_path): + process = subprocess.Popen( + [ + sys.executable, + "-m", + TELEMETRY_SERVICE_MODULE, + *_service_arguments( + socket_path, spool_path, ready_deadline=time.time() + 60 + ), + ], + stdin=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, + stderr=subprocess.PIPE, + text=True, + ) + try: + deadline = time.monotonic() + 120 + while spool_path.parent.stat().st_mode & 0o777 != 0o700: + assert time.monotonic() < deadline, "service never opened the spool" + assert process.poll() is None, "service exited before opening" + time.sleep(0.02) + process.send_signal(signal.SIGINT) + _, error_output = process.communicate(timeout=30) + finally: + if process.poll() is None: + process.kill() + process.wait() + + assert process.returncode == -signal.SIGINT + assert error_output == "" + + def test_client_passes_its_wait_as_a_wall_clock_ready_deadline( tmp_path, monkeypatch, real_local_telemetry ) -> None: @@ -483,8 +529,19 @@ def popen(command, **kwargs): after = time.time() (command,) = commands - ready_deadline = float(command[command.index("--ready-deadline") + 1]) + (argument,) = [part for part in command if part.startswith("--ready-deadline=")] + ready_deadline = float(argument.removeprefix("--ready-deadline=")) assert before + 42 <= ready_deadline <= after + 42 + # One token, so the service parses even a deadline that is already past. + assert ( + main_module.build_parser() + .parse_args( + ["--socket", "s", "--spool", "q", "--registration-json", "{}"] + + ["--parent-pid", "1", "--ready-deadline=-inf"] + ) + .ready_deadline + == -math.inf + ) @given( @@ -922,11 +979,28 @@ def next(self, value=None): ), ticks=st.integers(1, 30), parent_dies_at=st.one_of(st.none(), st.integers(1, 30)), + parent_check_raises=st.booleans(), + closed_at=st.one_of(st.none(), st.integers(1, 30)), heartbeat_seconds=st.sampled_from([1.0, 2.0, 3.5]), ) def test_worker_survives_any_step_failure( - monkeypatch, script, ticks, parent_dies_at, heartbeat_seconds + monkeypatch, + script, + ticks, + parent_dies_at, + parent_check_raises, + closed_at, + heartbeat_seconds, ) -> None: + """The worker never dies, checks its build every tick, and tells a crash + from a clean close. + + ``closed_at`` is the tick on which the build's close arrives while a step + runs, as it does when the accept thread handles it mid-tick; the build may + then exit before the tick ends. ``parent_check_raises`` makes the check of + a dead build raise instead of returning False. + """ + clock = SimpleNamespace(now=0.0, tick=0) monkeypatch.setattr( runtime_module, @@ -961,7 +1035,15 @@ def append(registration, event, *, resources=None): def parent_alive(): parent_checks.append(clock.tick) - return parent_dies_at is None or clock.tick < parent_dies_at + alive = parent_dies_at is None or clock.tick < parent_dies_at + if not alive and parent_check_raises: + raise OverflowError("signed integer is greater than maximum") + return alive + + def flush_once(): + if clock.tick == closed_at: + service._stop.set() + return outcomes.next(False) service = EmitterService( socket_path=Path("/unused"), @@ -971,7 +1053,7 @@ def parent_alive(): prune_if_due=outcomes.next, has_deliverable=lambda: outcomes.next(False), ), - delivery=SimpleNamespace(flush_once=lambda: outcomes.next(False)), + delivery=SimpleNamespace(flush_once=flush_once), sampler=SimpleNamespace( sample=lambda: outcomes.next({}), parent_alive=parent_alive ), @@ -1004,14 +1086,24 @@ def recorded_unexpected_exit() -> None: with contextlib.redirect_stderr(error_output): service._worker() # never raises - last_tick = ticks if parent_dies_at is None else min(ticks, parent_dies_at) - # The parent is checked on every tick until it is found dead. - assert parent_checks == list(range(1, last_tick + 1)) - # A dead parent is recorded once, on the tick it is found, and stops the - # loop. - parent_died = parent_dies_at is not None and parent_dies_at <= ticks - assert exit_records == ([parent_dies_at] if parent_died else []) - assert service._stop.is_set() is parent_died + closed = closed_at is not None and closed_at <= ticks + last_tick = min( + tick for tick in (ticks, parent_dies_at, closed_at) if tick is not None + ) + # The parent is checked on every tick until it is found dead, except on the + # tick the build closed. + assert parent_checks == [ + tick for tick in range(1, last_tick + 1) if not (closed and tick == closed_at) + ] + # A build that died without closing is recorded once, on the tick it is + # found, which stops the loop; a build that closed first never is. + crashed = ( + parent_dies_at is not None + and parent_dies_at <= ticks + and (not closed or parent_dies_at < closed_at) + ) + assert exit_records == ([parent_dies_at] if crashed else []) + assert service._stop.is_set() is (crashed or closed) # Reference heartbeat schedule: due one interval after the start or the # last success; a failed heartbeat stays due and is retried next tick. due = heartbeat_seconds @@ -1030,6 +1122,8 @@ def recorded_unexpected_exit() -> None: reported = { name for name in outcomes.raised if name in {"RuntimeError", "ValueError"} } + if crashed and parent_check_raises: + reported.add("OverflowError") lines = error_output.getvalue().splitlines() assert len(lines) == len(reported) assert {line.split(" hit ")[1].split(" ")[0] for line in lines} == reported From f35b015fe24d0e5d292b929d7246c60bd1940c2c Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 9 Oct 2026 12:31:28 -0400 Subject: [PATCH 07/13] Queue telemetry events in memory and write the spool from one thread The emitter service appended each event to the shared SQLite spool before replying, so another build holding the spool's write lock stalled the build's sends past the client's 0.2 s timeout and dropped events once SQLite's 5 s busy wait ran out. The socket thread now validates an event, queues it in a bounded in-memory queue and replies at once. One writer thread appends queued events in order, in batches, retrying lock errors for as long as they last. Heartbeats and the killed-build record use the same queue. On close or parent death the service drains the queue within its drain window and reports in one line anything it could not write. Co-Authored-By: Claude Opus 5.5 --- README.md | 12 +- changelog.d/telemetry-event-queue.fixed.md | 3 + .../src/microcosm/build/telemetry_emitter.py | 18 +- .../telemetry_emitter_service/constants.py | 40 + .../telemetry_emitter_service/contention.py | 17 +- .../telemetry_emitter_service/event_queue.py | 340 ++++++ .../telemetry_emitter_service/runtime.py | 196 +++- .../build/telemetry_emitter_service/spool.py | 104 +- .../shared/test_telemetry_emitter.py | 9 +- .../shared/test_telemetry_event_queue.py | 1045 +++++++++++++++++ .../shared/test_telemetry_spool_contention.py | 61 +- 11 files changed, 1744 insertions(+), 101 deletions(-) create mode 100644 changelog.d/telemetry-event-queue.fixed.md create mode 100644 packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/event_queue.py create mode 100644 packages/microcosm-build/tests/engine_free/shared/test_telemetry_event_queue.py diff --git a/README.md b/README.md index 4dd5b985da..071b678895 100644 --- a/README.md +++ b/README.md @@ -53,8 +53,16 @@ locally and the build continues. Every build on a host shares one local event spool; a service that finds it locked by another process keeps retrying until shortly before the build stops waiting for it, and one that still cannot register prints a single warning line, without a traceback, before the build -continues without hosted telemetry. This live event path is independent -of the staging run files described below. +continues without hosted telemetry. Once it is running, the service replies to +each event as soon as the event is queued in its memory, and a separate writer +thread appends queued events to the spool in the order they arrived, retrying +for as long as another process holds the spool's lock, so a busy spool never +delays the build. The queue holds at most 10,000 events or 16 MiB, keeping +room for the run's started, completed and failed events; when it is full the +service refuses further updates and prints one warning line. When the build +closes or dies, the service spends up to 15 s writing what is still queued and +reports in one line anything it could not write. This live event path is +independent of the staging run files described below. US fiscal refresh builds also write pre-release staging run files **by default**. Progress JSON is uploaded to `policyengine/populace-us-staging` diff --git a/changelog.d/telemetry-event-queue.fixed.md b/changelog.d/telemetry-event-queue.fixed.md new file mode 100644 index 0000000000..33eb87243c --- /dev/null +++ b/changelog.d/telemetry-event-queue.fixed.md @@ -0,0 +1,3 @@ +A build's telemetry sends no longer wait for the shared local spool. Before, the emitter service appended each event to the SQLite spool before replying. When another build held the spool's write lock, a send waited up to SQLite's 5 s busy wait while the build's client gave up after 0.2 s and printed "could not queue an update", and an append still locked after 5 s dropped the event. Now the service replies once the event is queued in memory, and one writer thread appends queued events in arrival order, up to 100 per transaction, retrying lock errors for as long as they last. Heartbeats and the record of a killed build go through the same queue, so the killed-build record follows every event the build sent. + +The queue holds at most 10,000 events and 16 MiB of their encoded JSON, with 16 events and 256 KiB kept for run events (started, completed, failed), so a queue that progress updates have filled still records how the build ended. A full queue refuses updates and prints one warning line. When the build closes or dies, the service spends up to its 15 s drain writing the queue, starts no append in the last 0.25 s of it, and reports in one line how many events it could not write. diff --git a/packages/microcosm-build/src/microcosm/build/telemetry_emitter.py b/packages/microcosm-build/src/microcosm/build/telemetry_emitter.py index 8bd365d4a3..998cc88800 100644 --- a/packages/microcosm-build/src/microcosm/build/telemetry_emitter.py +++ b/packages/microcosm-build/src/microcosm/build/telemetry_emitter.py @@ -262,7 +262,12 @@ def emit( message: str | None = None, details: Mapping[str, Any] | None = None, ) -> None: - """Queue one event locally, never raising into the build.""" + """Queue one event with the local service, never raising into the build. + + The service acknowledges the event once it is queued in the service's + memory, behind every event acknowledged before it, without waiting for + the shared spool; see ``_send``. + """ if not self.available: return @@ -424,6 +429,17 @@ def _close_transition_stage( self.stage(stage_id, status=status, message=message, **details) def _send(self, payload: Mapping[str, Any]) -> None: + """Send one message and wait at most the send timeout for its reply. + + ``ok`` to an event means the service has queued it in memory behind + the events acknowledged before it; a writer thread then appends it to + the spool in that order, retrying while another build holds the + spool's lock. The reply never waits for the spool, so a locked spool + does not delay the build. ``error``, or no reply within the timeout, + means the event may not be queued: the first such failure prints one + warning, and the build continues either way. + """ + if self._socket_path is None: return try: diff --git a/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/constants.py b/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/constants.py index 8b245dfe03..04b42f5624 100644 --- a/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/constants.py +++ b/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/constants.py @@ -67,6 +67,24 @@ STARTUP_RETRY_LIMIT_SECONDS: Final = 60.0 SPOOL_LOCKED_EXIT_STATUS: Final = 75 # EX_TEMPFAIL from sysexits.h +# The socket hands each accepted event to a writer thread through an ordered +# in-memory queue, so a spool that another process keeps locked never delays +# the build's sends. The queue holds at most QUEUE_MAX_EVENTS events and +# QUEUE_MAX_BYTES of their encoded JSON. The last QUEUE_RESERVED_EVENTS and +# QUEUE_RESERVED_BYTES of that room take only run events (started, completed, +# failed), so a queue that progress updates have filled still records how the +# build ended. +QUEUE_MAX_EVENTS: Final = 10_000 +QUEUE_MAX_BYTES: Final = 16 * 1024 * 1024 +QUEUE_RESERVED_EVENTS: Final = 16 +QUEUE_RESERVED_BYTES: Final = 256 * 1024 +# The writer appends up to WRITER_BATCH_EVENTS queued events per transaction. +# Each of its statements waits at most WRITER_BUSY_TIMEOUT_SECONDS for another +# process's lock, so its retry loop decides how long to keep trying and it +# holds the spool's in-process lock only briefly. +WRITER_BATCH_EVENTS: Final = 100 +WRITER_BUSY_TIMEOUT_SECONDS: Final = 0.25 + DEFAULT_HEARTBEAT_SECONDS: Final = 60.0 DEFAULT_DRAIN_SECONDS: Final = 15.0 MINIMUM_HEARTBEAT_SECONDS: Final = 1.0 @@ -77,6 +95,9 @@ DRAIN_RETRY_SECONDS: Final = 0.5 EVENT_OBJECT_ERROR: Final = "event must be an object" +EVENT_FIELDS_ERROR: Final = "event must have an event_type and a status" +QUEUE_FULL_ERROR: Final = "local telemetry queue is full" +QUEUE_CLOSED_ERROR: Final = "local telemetry queue is closed" UNSUPPORTED_ACTION_ERROR: Final = "unsupported local telemetry action" LOCAL_MESSAGE_TOO_LARGE_ERROR: Final = "local telemetry message exceeds 1 MiB" FAILURE_CLASS_UNEXPECTED_PROCESS_EXIT: Final = "unexpected_process_exit" @@ -98,4 +119,23 @@ "warning: the local telemetry emitter service's delivery worker hit " "{error_type} ({error}) and will keep running." ) +QUEUE_FULL_WARNING: Final = ( + "warning: the local telemetry emitter service's queue is full ({events} " + "updates, {megabytes:.1f} MiB, waiting for its spool); it will refuse " + "updates until the spool takes them." +) +UNWRITTEN_EVENTS_WARNING: Final = ( + "warning: the local telemetry emitter service could not write {count} " + "queued update(s) to its spool before its {seconds:g} s shutdown drain " + "ended ({reason})." +) +DRAIN_TIME_REASON: Final = "the drain ran out of time" +EVENT_DROPPED_WARNING: Final = ( + "warning: the local telemetry emitter service's spool refused an update, " + "which was dropped: {error_type} ({error})." +) +WRITER_STOPPED_WARNING: Final = ( + "warning: the local telemetry emitter service's spool writer stopped: " + "{error_type}: {error}" +) MAX_WARNING_ERROR_CHARS: Final = 300 diff --git a/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/contention.py b/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/contention.py index 06dc14d700..e6441feb98 100644 --- a/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/contention.py +++ b/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/contention.py @@ -41,22 +41,25 @@ def is_transient_spool_error(error: BaseException) -> bool: def retry_spool_contention[T]( operation: Callable[[], T], *, - deadline: float, + deadline: float | Callable[[], float], clock: Callable[[], float] = time.monotonic, sleep: Callable[[float], None] = time.sleep, jitter: Callable[[float, float], float] = random.uniform, ) -> T: """Run ``operation``, retrying spool lock contention until ``deadline``. - ``deadline`` is a ``clock`` reading. The first attempt always runs. After a - transient lock error the next attempt waits a jittered backoff that starts - at ``SPOOL_RETRY_INITIAL_SECONDS`` and doubles up to - ``SPOOL_RETRY_MAX_SECONDS``. When that attempt would not start before - ``deadline``, the last lock error is raised instead. Any other error is + ``deadline`` is a ``clock`` reading, or a function returning one. A + function is read again after every lock error, so its caller can bring the + deadline forward, from infinity say, while this waits. The first attempt + always runs. After a transient lock error the next attempt waits a + jittered backoff that starts at ``SPOOL_RETRY_INITIAL_SECONDS`` and doubles + up to ``SPOOL_RETRY_MAX_SECONDS``. When that attempt would not start before + the deadline, the last lock error is raised instead. Any other error is raised at once. Each attempt may itself wait in SQLite's busy handler, so this bounds when attempts start, not when the last one ends. """ + current_deadline = deadline if callable(deadline) else lambda: deadline delay = SPOOL_RETRY_INITIAL_SECONDS while True: try: @@ -65,7 +68,7 @@ def retry_spool_contention[T]( if not is_transient_spool_error(error): raise wait = jitter(delay / 2, delay) - if clock() + wait >= deadline: + if clock() + wait >= current_deadline(): raise sleep(wait) delay = min(SPOOL_RETRY_MAX_SECONDS, delay * 2) diff --git a/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/event_queue.py b/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/event_queue.py new file mode 100644 index 0000000000..341b9e9d3c --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/event_queue.py @@ -0,0 +1,340 @@ +"""Ordered, bounded hand-off of accepted events from the socket to the spool. + +The service's socket thread validates each event, queues it here and replies +at once. One writer thread appends queued events to the shared spool, oldest +first, and is the only code that appends this producer's events, so the spool +assigns sequences in queue order. Lock contention on the spool therefore +delays only the writer, never a build's send. +""" + +from __future__ import annotations + +import itertools +import json +import math +import threading +import time +from collections import deque +from collections.abc import Callable, Mapping +from dataclasses import dataclass +from typing import Any + +from microcosm.build.telemetry_emitter_service.constants import ( + QUEUE_CLOSED_ERROR, + QUEUE_FULL_ERROR, + QUEUE_MAX_BYTES, + QUEUE_MAX_EVENTS, + QUEUE_RESERVED_BYTES, + QUEUE_RESERVED_EVENTS, + WRITER_BATCH_EVENTS, + WRITER_BUSY_TIMEOUT_SECONDS, + WRITER_STOPPED_WARNING, +) +from microcosm.build.telemetry_emitter_service.contention import ( + is_transient_spool_error, + retry_spool_contention, +) +from microcosm.build.telemetry_emitter_service.diagnostics import ( + describe_error, + write_warning, +) +from microcosm.build.telemetry_protocol import EVENT_TYPE_RUN + + +class EventRefusedError(Exception): + """The queue did not take an event.""" + + +class QueueFullError(EventRefusedError): + """Taking the event would exceed the queue's bound.""" + + def __init__(self) -> None: + super().__init__(QUEUE_FULL_ERROR) + + +class QueueClosedError(EventRefusedError): + """The queue takes no more events: the service is shutting down.""" + + def __init__(self) -> None: + super().__init__(QUEUE_CLOSED_ERROR) + + +class DrainDeadlineError(Exception): + """The writer may not start another append before the drain deadline.""" + + +@dataclass(frozen=True, slots=True) +class QueuedEvent: + """One accepted event and its resource sample, held as compact JSON. + + Holding bytes rather than parsed objects makes the queue's byte bound a + bound on the memory its events occupy. + """ + + encoded: bytes + run_event: bool + + @classmethod + def encode( + cls, + event: Mapping[str, Any], + resources: Mapping[str, Any] | None, + ) -> QueuedEvent: + """Encode an event; raises ``ValueError`` for values JSON cannot hold.""" + + encoded = json.dumps( + [event, resources], separators=(",", ":"), allow_nan=False + ).encode() + return cls(encoded, event.get("event_type") == EVENT_TYPE_RUN) + + @property + def size(self) -> int: + return len(self.encoded) + + def decode(self) -> tuple[dict[str, Any], dict[str, Any] | None]: + event, resources = json.loads(self.encoded) + return event, resources + + +class EventQueue: + """Accepted events, oldest first, bounded in count and in encoded bytes. + + Safe to share between threads. Its lock is held only for in-memory work, + never while the spool is written. Only the writer removes events, and only + from the front, so the events it reads with ``oldest`` are still the oldest + when it removes them. + """ + + def __init__( + self, + *, + max_events: int = QUEUE_MAX_EVENTS, + max_bytes: int = QUEUE_MAX_BYTES, + reserved_events: int = QUEUE_RESERVED_EVENTS, + reserved_bytes: int = QUEUE_RESERVED_BYTES, + ) -> None: + self.max_events = max_events + self.max_bytes = max_bytes + self.reserved_events = reserved_events + self.reserved_bytes = reserved_bytes + self._events: deque[QueuedEvent] = deque() + self._bytes = 0 + self._closed = False + self._changed = threading.Condition() + + def __len__(self) -> int: + with self._changed: + return len(self._events) + + @property + def queued_bytes(self) -> int: + with self._changed: + return self._bytes + + @property + def closed(self) -> bool: + with self._changed: + return self._closed + + def offer(self, event: QueuedEvent, *, close: bool = False) -> None: + """Queue ``event`` behind every event queued before it. + + A run event may use the reserved room; any other event must leave it + free. Raises ``QueueFullError`` when the event does not fit and + ``QueueClosedError`` once the queue is closed. With ``close`` the queue + closes after this offer, whether or not it took the event, so nothing + can be queued behind it. + """ + + with self._changed: + try: + if self._closed: + raise QueueClosedError() + if not self._fits(event): + raise QueueFullError() + self._events.append(event) + self._bytes += event.size + finally: + if close: + self._closed = True + self._changed.notify_all() + + def _fits(self, event: QueuedEvent) -> bool: + max_events = self.max_events + max_bytes = self.max_bytes + if not event.run_event: + max_events -= self.reserved_events + max_bytes -= self.reserved_bytes + return len(self._events) < max_events and self._bytes + event.size <= max_bytes + + def close(self) -> None: + """Refuse every later offer; queued events stay for the writer.""" + + with self._changed: + self._closed = True + self._changed.notify_all() + + def wait_for_events(self) -> bool: + """Block until an event is queued, or return False once closed and empty.""" + + with self._changed: + while not self._events and not self._closed: + self._changed.wait() + return bool(self._events) + + def oldest(self, limit: int) -> list[QueuedEvent]: + """Return up to ``limit`` of the oldest events, leaving them queued.""" + + with self._changed: + return list(itertools.islice(self._events, limit)) + + def remove(self, count: int) -> None: + """Remove the ``count`` oldest events, once they are written or dropped.""" + + with self._changed: + for _ in range(count): + self._bytes -= self._events.popleft().size + self._changed.notify_all() + + +class SpoolWriter: + """Appends one producer's queued events to the spool, in order. + + While the service serves, a lock error is retried for as long as it lasts: + an event leaves the queue only once the spool has stored it, or once the + spool has refused it for another reason. ``finish`` sets a drain deadline + and closes the queue; the writer then starts no append after + ``deadline - WRITER_BUSY_TIMEOUT_SECONDS``, and since each statement waits + at most that long for a lock, its last append ends by the deadline plus the + time its own inserts take. + """ + + def __init__( + self, + spool: Any, + registration: Mapping[str, Any], + queue: EventQueue, + *, + report_dropped: Callable[[Exception], None], + clock: Callable[[], float] = time.monotonic, + sleep: Callable[[float], None] = time.sleep, + ) -> None: + self.spool = spool + self.registration = dict(registration) + self.queue = queue + self._report_dropped = report_dropped + self._clock = clock + self._sleep = sleep + self._drain_deadline = math.inf + self._thread: threading.Thread | None = None + # Why the writer last stopped with events still queued. + self.stopped_by: BaseException | None = None + # Events the spool refused for a reason other than lock contention. + self.dropped = 0 + + def start(self) -> None: + """Write queued events on a daemon thread until ``finish``.""" + + self._thread = threading.Thread( + target=self._run, name="telemetry-spool-writer", daemon=True + ) + self._thread.start() + + def is_alive(self) -> bool: + return self._thread is not None and self._thread.is_alive() + + def finish(self, deadline: float) -> int: + """Close the queue and write what it holds, until ``deadline``. + + ``deadline`` is a reading of this writer's clock. Returns how many + events were left unwritten. A writer whose thread was never started, + such as one a test drives step by step, writes on the calling thread. + """ + + self._drain_deadline = deadline + self.queue.close() + if self._thread is None: + if len(self.queue): + self.write_queued() + else: + # The thread's last append ends by the deadline plus its inserts; + # this waits for that and gives up a statement wait later. + self._thread.join( + timeout=max(0.0, deadline - self._clock()) + WRITER_BUSY_TIMEOUT_SECONDS + ) + return len(self.queue) + + def _run(self) -> None: + try: + while self.queue.wait_for_events(): + if not self.write_queued(): + return + except Exception as error: + # A bug, not contention: one line, never a traceback, on the + # build's stderr. The service stops serving once this thread ends. + self.stopped_by = error + write_warning( + WRITER_STOPPED_WARNING.format( + error_type=type(error).__name__, error=describe_error(error) + ) + ) + + def write_queued(self) -> bool: + """Append queued events, oldest first and in batches, until none is left. + + Returns False when the drain deadline stopped it with events queued. + """ + + while events := self.queue.oldest(WRITER_BATCH_EVENTS): + try: + self._write_oldest(events) + except Exception as error: + if not ( + isinstance(error, DrainDeadlineError) + or is_transient_spool_error(error) + ): + raise + self.stopped_by = error + return False + return True + + def _write_oldest(self, events: list[QueuedEvent]) -> None: + """Append the queue's oldest ``events`` in one transaction, then remove them. + + A lock error is retried until the drain deadline and then raised, with + the events still queued. An event the spool refuses for any other + reason is reported and dropped on its own, so it cannot take the rest + of its batch with it. + """ + + try: + self._append(events) + except Exception as error: + if isinstance(error, DrainDeadlineError) or is_transient_spool_error(error): + raise + if len(events) > 1: + for event in events: + self._write_oldest([event]) + return + self.dropped += 1 + self._report_dropped(error) + self.queue.remove(len(events)) + + def _append(self, events: list[QueuedEvent]) -> None: + if self._clock() >= self._last_attempt_start(): + raise DrainDeadlineError() + pairs = [event.decode() for event in events] + retry_spool_contention( + lambda: self.spool.append_many( + self.registration, + pairs, + busy_timeout_seconds=WRITER_BUSY_TIMEOUT_SECONDS, + ), + deadline=self._last_attempt_start, + clock=self._clock, + sleep=self._sleep, + ) + + def _last_attempt_start(self) -> float: + # Infinite until finish sets a drain deadline. + return self._drain_deadline - WRITER_BUSY_TIMEOUT_SECONDS diff --git a/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/runtime.py b/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/runtime.py index e8ee754811..6b7a26af5e 100644 --- a/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/runtime.py +++ b/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/runtime.py @@ -15,25 +15,37 @@ from microcosm.build.telemetry_emitter_service.constants import ( DEFAULT_DRAIN_SECONDS, DRAIN_RETRY_SECONDS, + DRAIN_TIME_REASON, + EVENT_DROPPED_WARNING, + EVENT_FIELDS_ERROR, EVENT_OBJECT_ERROR, FAILURE_CLASS_UNEXPECTED_PROCESS_EXIT, LOCAL_MESSAGE_TOO_LARGE_ERROR, MINIMUM_HEARTBEAT_SECONDS, + QUEUE_FULL_WARNING, SOCKET_ACCEPT_TIMEOUT_SECONDS, SOCKET_CONNECTION_TIMEOUT_SECONDS, SOCKET_LISTEN_BACKLOG, UNSUPPORTED_ACTION_ERROR, + UNWRITTEN_EVENTS_WARNING, WORKER_INTERVAL_SECONDS, WORKER_STEP_WARNING, ) from microcosm.build.telemetry_emitter_service.contention import ( is_transient_spool_error, - retry_spool_contention, ) from microcosm.build.telemetry_emitter_service.diagnostics import ( describe_error, write_warning, ) +from microcosm.build.telemetry_emitter_service.event_queue import ( + DrainDeadlineError, + EventQueue, + EventRefusedError, + QueuedEvent, + QueueFullError, + SpoolWriter, +) from microcosm.build.telemetry_emitter_service.resources import ProcessTreeSampler from microcosm.build.telemetry_emitter_service.spool import EventSpool from microcosm.build.telemetry_emitter_service.timestamps import utc_now @@ -83,7 +95,30 @@ def _unexpected_exit_event(stage_id: str) -> dict[str, Any]: class EmitterService: - """Private local socket server with a concurrent delivery worker.""" + """Private local socket server with a spool writer and a delivery worker. + + Three threads share the work. The socket thread validates each message, + queues events in memory and replies. The writer thread appends queued + events to the spool, oldest first. The worker samples resources, queues + heartbeats, prunes, delivers to the collector and notices a dead build. + + The reply is the acknowledgement contract with the build: + + - ``ok`` to an event means the event was valid and is queued in memory + behind every event acknowledged before it. The writer appends queued + events to the spool in that order, and the spool assigns their producer + sequences as it does, so sequence order is acknowledgement order. + - ``ok`` does not mean the event is on disk, and the reply never waits for + the spool. When the spool is free the event is written moments later. + While another process holds the spool's lock the event waits in the + queue, and the writer retries for as long as the contention lasts. + - An acknowledged event is lost only if this process is killed before + writing it, if the spool refuses it for a reason other than lock + contention (reported as one warning line), or if the spool stays locked + through the whole shutdown drain (reported with a count). + - ``error`` means the message was malformed, or the queue was full or + closed. The event was not queued. + """ def __init__( self, @@ -95,6 +130,7 @@ def __init__( sampler: ProcessTreeSampler, heartbeat_seconds: float, drain_seconds: float = DEFAULT_DRAIN_SECONDS, + event_queue: EventQueue | None = None, ) -> None: self.socket_path = socket_path self.registration = dict(registration) @@ -108,12 +144,31 @@ def __init__( self.drain_seconds = max(0.0, drain_seconds) self._stop = threading.Event() self._last_stage = STAGE_CREATED + # Warnings already printed, shared by the three threads. + self._warning_lock = threading.Lock() self._reported_error_types: set[str] = set() + self._dropped_error_types: set[str] = set() + self._warned_queue_full = False + self._drain_lock = threading.Lock() + self._drain_started = False + self.event_queue = event_queue if event_queue is not None else EventQueue() + # The writer reads this module's clock when it runs, so a test that + # replaces the module's time drives the writer too. + self.writer = SpoolWriter( + spool, + self.registration, + self.event_queue, + report_dropped=self._report_dropped, + clock=lambda: time.monotonic(), + sleep=lambda seconds: time.sleep(seconds), + ) def run(self) -> None: """Serve local messages until the client closes or exits. - The spool must already hold this producer's registration. + The spool must already hold this producer's registration. Queued + events are written to the spool before this returns, within + ``drain_seconds`` of the stop. """ self.socket_path.parent.mkdir(parents=True, exist_ok=True) @@ -124,10 +179,15 @@ def run(self) -> None: os.chmod(self.socket_path, 0o600) server.listen(SOCKET_LISTEN_BACKLOG) server.settimeout(SOCKET_ACCEPT_TIMEOUT_SECONDS) + self.writer.start() worker = threading.Thread(target=self._worker, daemon=True) worker.start() self._serve(server, worker) + self._stop.set() worker.join(timeout=self.drain_seconds + WORKER_INTERVAL_SECONDS) + # The worker drains on its way out. One that died or hung + # before then leaves the queue to this thread. + self._drain() finally: self.socket_path.unlink(missing_ok=True) try: @@ -138,8 +198,9 @@ def run(self) -> None: def _serve(self, server: socket.socket, worker: threading.Thread) -> None: # Only the worker notices that the build died without closing, so a # service whose worker has stopped must stop too, or it would serve a - # dead build forever. - while not self._stop.is_set() and worker.is_alive(): + # dead build forever. Without the writer, acknowledged events would + # never reach the spool. + while not self._stop.is_set() and worker.is_alive() and self.writer.is_alive(): try: connection, _ = server.accept() except TimeoutError: @@ -153,6 +214,8 @@ def _serve(self, server: socket.socket, worker: threading.Thread) -> None: pass def _serve_connection(self, connection: socket.socket) -> bytes: + """Read and act on one message, returning the reply the class describes.""" + try: message = self._read_message(connection) self._handle(message) @@ -175,29 +238,55 @@ def _read_message(connection: socket.socket) -> dict[str, Any]: return json.loads(bytes(chunks).split(LOCAL_MESSAGE_DELIMITER, 1)[0]) def _handle(self, message: Mapping[str, Any]) -> None: + """Act on one message; raising makes the reply ``error``. + + An event is validated here and queued, never written: the reply must + not wait for the spool. + """ + action = message.get("action") if action == ACTION_EVENT: event = message.get("event") if not isinstance(event, Mapping): raise ValueError(EVENT_OBJECT_ERROR) + if "event_type" not in event or "status" not in event: + raise ValueError(EVENT_FIELDS_ERROR) stage_id = event.get("stage_id") if isinstance(stage_id, str) and stage_id not in { STAGE_COMPLETE, STAGE_FAILED, }: self._last_stage = stage_id - self.spool.append( - self.registration, - event, - resources=self.sampler.sample(), - ) + self._enqueue(event) elif action == ACTION_CLOSE: + # Nothing, not even a heartbeat, is queued behind the build's last + # event. + self.event_queue.close() self._stop.set() elif action == ACTION_PING: return else: raise ValueError(UNSUPPORTED_ACTION_ERROR) + def _enqueue(self, event: Mapping[str, Any], *, close: bool = False) -> None: + """Queue an event with a resource sample taken now.""" + + queued = QueuedEvent.encode(event, self.sampler.sample()) + try: + self.event_queue.offer(queued, close=close) + except QueueFullError: + with self._warning_lock: + first = not self._warned_queue_full + self._warned_queue_full = True + if first: + write_warning( + QUEUE_FULL_WARNING.format( + events=len(self.event_queue), + megabytes=self.event_queue.queued_bytes / (1024 * 1024), + ) + ) + raise + def _worker(self) -> None: # A failed step is skipped for this tick only. The parent check below # must keep running: it is the only thing that stops a service whose @@ -209,36 +298,27 @@ def _worker(self) -> None: # much less likely to disappear between stage and heartbeat events. self._attempt(self.sampler.sample) # A heartbeat that fails stays due, so the next tick retries it. - if now >= next_heartbeat and self._attempt(self._append_heartbeat): + if now >= next_heartbeat and self._attempt(self._queue_heartbeat): next_heartbeat = now + self.heartbeat_seconds self._attempt(self.spool.prune_if_due) self._attempt(self.delivery.flush_once) if not self.sampler.parent_alive(): - self._attempt(self._append_unexpected_exit) + self._attempt(self._queue_unexpected_exit) + # Closed even when the record could not be queued: nothing + # arrives from a dead build. + self.event_queue.close() self._stop.set() break self._drain() - def _append_heartbeat(self) -> None: - self.spool.append( - self.registration, - _heartbeat_event(self._last_stage), - resources=self.sampler.sample(), - ) + def _queue_heartbeat(self) -> None: + self._enqueue(_heartbeat_event(self._last_stage)) - def _append_unexpected_exit(self) -> None: - # This is the only record of a build that was killed, so it waits out - # lock contention for as long as the shutdown drain may take. - retry_spool_contention( - lambda: self.spool.append( - self.registration, - _unexpected_exit_event(self._last_stage), - resources=self.sampler.sample(), - ), - deadline=time.monotonic() + self.drain_seconds, - clock=time.monotonic, - sleep=time.sleep, - ) + def _queue_unexpected_exit(self) -> None: + # This is the only record of a build that was killed. It goes behind + # every event the build sent and closes the queue, so it is the last + # event of the run; as a run event it may use the reserved room. + self._enqueue(_unexpected_exit_event(self._last_stage), close=True) def _attempt(self, step: Callable[[], object]) -> bool: """Run one worker step, reporting rather than raising its failure.""" @@ -252,14 +332,16 @@ def _attempt(self, step: Callable[[], object]) -> bool: def _report(self, error: Exception) -> None: # Lock contention clears by itself and the step runs again on a later - # tick, so it is not worth a line in the build's log. Anything else is - # reported once per error type. - if is_transient_spool_error(error): + # tick, so it is not worth a line in the build's log. Nor is an event + # the queue refused: a full queue has printed its own line. Anything + # else is reported once per error type. + if is_transient_spool_error(error) or isinstance(error, EventRefusedError): return error_type = type(error).__name__ - if error_type in self._reported_error_types: - return - self._reported_error_types.add(error_type) + with self._warning_lock: + if error_type in self._reported_error_types: + return + self._reported_error_types.add(error_type) write_warning( WORKER_STEP_WARNING.format( error_type=error_type, @@ -267,8 +349,48 @@ def _report(self, error: Exception) -> None: ) ) + def _report_dropped(self, error: Exception) -> None: + # An event the spool refused for a reason other than lock contention, + # reported once per error type. + error_type = type(error).__name__ + with self._warning_lock: + if error_type in self._dropped_error_types: + return + self._dropped_error_types.add(error_type) + write_warning( + EVENT_DROPPED_WARNING.format( + error_type=error_type, + error=describe_error(error), + ) + ) + def _drain(self) -> None: + """Write the queue to the spool, then deliver, within ``drain_seconds``. + + Runs once: from the worker on its way out, or from ``run`` when the + worker stopped without draining. + """ + + with self._drain_lock: + if self._drain_started: + return + self._drain_started = True deadline = time.monotonic() + self.drain_seconds + unwritten = self.writer.finish(deadline) + if unwritten: + stopped_by = self.writer.stopped_by + write_warning( + UNWRITTEN_EVENTS_WARNING.format( + count=unwritten, + seconds=self.drain_seconds, + reason=( + describe_error(stopped_by) + if stopped_by is not None + and not isinstance(stopped_by, DrainDeadlineError) + else DRAIN_TIME_REASON + ), + ) + ) while time.monotonic() < deadline: progressed = False try: diff --git a/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/spool.py b/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/spool.py index 20c0b44e23..869fc0da31 100644 --- a/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/spool.py +++ b/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/spool.py @@ -5,7 +5,7 @@ import threading import time import uuid -from collections.abc import Mapping +from collections.abc import Mapping, Sequence from datetime import UTC, datetime, timedelta from pathlib import Path from typing import Any @@ -56,9 +56,12 @@ def __init__( # lowers it so its own retry loop decides when to give up; it applies # from each connection's next checkout. self.busy_timeout_seconds = busy_timeout_seconds + # One call's own wait, set by append_many while it holds _lock, which + # every connection checkout here happens under. + self._busy_timeout_override: float | None = None self._engine = create_spool_engine( self.path, - busy_timeout_seconds=lambda: self.busy_timeout_seconds, + busy_timeout_seconds=self._statement_busy_timeout, ) try: upgrade_spool_database( @@ -112,41 +115,72 @@ def append( ) -> dict[str, Any]: """Append an event and assign its stable producer sequence.""" + return self.append_many(registration, [(event, resources)])[0] + + def append_many( + self, + registration: Mapping[str, Any], + events: Sequence[tuple[Mapping[str, Any], Mapping[str, Any] | None]], + *, + busy_timeout_seconds: float | None = None, + ) -> list[dict[str, Any]]: + """Append ``(event, resources)`` pairs in one transaction, in order. + + The events take consecutive producer sequences in the order given, or, + when the transaction fails, none of them is stored and the producer's + next sequence is unchanged, so a failed call can simply be repeated. + ``busy_timeout_seconds``, when given, caps how long this call's + statements wait for another process's lock instead of + ``busy_timeout_seconds`` on the spool. + """ + run_id = str(registration["run_id"]) producer_id = str(registration["producer_id"]) - with self._lock, self._session_factory.begin() as session: - run = session.get(TelemetryRunRecord, (run_id, producer_id)) - if run is None: - raise KeyError(run_id) - sequence = run.next_sequence - event_id = uuid.uuid4().hex - payload = { - "schema_version": TELEMETRY_SCHEMA_VERSION, - "event_id": event_id, - "run_id": run_id, - "producer_id": producer_id, - "sequence": sequence, - "timestamp": event.get("timestamp") or utc_now(), - "event_type": event["event_type"], - "stage_id": event.get("stage_id"), - "status": event["status"], - "message": event.get("message"), - "details": event.get("details") or {}, - "resources": resources, - } - session.add( - TelemetryEventRecord( - event_id=event_id, - run_id=run_id, - producer_id=producer_id, - sequence=sequence, - payload=payload, - created_at=utc_now(), - ) - ) - run.next_sequence = sequence + 1 - run.updated_at = utc_now() - return payload + payloads: list[dict[str, Any]] = [] + with self._lock: + self._busy_timeout_override = busy_timeout_seconds + try: + with self._session_factory.begin() as session: + run = session.get(TelemetryRunRecord, (run_id, producer_id)) + if run is None: + raise KeyError(run_id) + for event, resources in events: + sequence = run.next_sequence + event_id = uuid.uuid4().hex + payload = { + "schema_version": TELEMETRY_SCHEMA_VERSION, + "event_id": event_id, + "run_id": run_id, + "producer_id": producer_id, + "sequence": sequence, + "timestamp": event.get("timestamp") or utc_now(), + "event_type": event["event_type"], + "stage_id": event.get("stage_id"), + "status": event["status"], + "message": event.get("message"), + "details": event.get("details") or {}, + "resources": resources, + } + session.add( + TelemetryEventRecord( + event_id=event_id, + run_id=run_id, + producer_id=producer_id, + sequence=sequence, + payload=payload, + created_at=utc_now(), + ) + ) + run.next_sequence = sequence + 1 + payloads.append(payload) + run.updated_at = utc_now() + finally: + self._busy_timeout_override = None + return payloads + + def _statement_busy_timeout(self) -> float: + override = self._busy_timeout_override + return self.busy_timeout_seconds if override is None else override def pending_runs(self) -> list[dict[str, Any]]: """Return registrations that have events eligible for delivery.""" diff --git a/packages/microcosm-build/tests/engine_free/shared/test_telemetry_emitter.py b/packages/microcosm-build/tests/engine_free/shared/test_telemetry_emitter.py index 10411461df..68e073236a 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_telemetry_emitter.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_telemetry_emitter.py @@ -587,7 +587,7 @@ def flush_once(self): return False -def test_local_socket_acknowledges_after_durable_queue( +def test_local_socket_acknowledges_a_queued_event_that_then_reaches_the_spool( tmp_path, real_local_telemetry ) -> None: socket_path = ( @@ -618,7 +618,12 @@ def test_local_socket_acknowledges_after_durable_queue( ) assert client.recv(16) == b"ok\n" - assert spool.batch("run-a", "producer-a")[0]["resources"]["rss_bytes"] == 100 + # "ok" means queued in memory; the writer thread stores it moments later. + deadline = time.monotonic() + 5 + while not (stored := spool.batch("run-a", "producer-a")): + assert time.monotonic() < deadline, "the acknowledged event was not stored" + time.sleep(0.01) + assert stored[0]["resources"]["rss_bytes"] == 100 with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as client: client.connect(str(socket_path)) diff --git a/packages/microcosm-build/tests/engine_free/shared/test_telemetry_event_queue.py b/packages/microcosm-build/tests/engine_free/shared/test_telemetry_event_queue.py new file mode 100644 index 0000000000..0985a461ec --- /dev/null +++ b/packages/microcosm-build/tests/engine_free/shared/test_telemetry_event_queue.py @@ -0,0 +1,1045 @@ +"""The emitter service's event path, decoupled from writes to its spool. + +The service acknowledges an event once it is queued in memory; one writer +thread appends queued events to the shared spool, oldest first, retrying lock +contention. These tests state the queue's and the writer's invariants as +properties, then hold the spool's write lock from a second connection +(``BEGIN IMMEDIATE``), the way another build's service does while it writes, +while a build sends events. +""" + +from __future__ import annotations + +import contextlib +import io +import json +import math +import socket +import sqlite3 +import tempfile +import threading +import time +from pathlib import Path +from types import SimpleNamespace + +import pytest +from hypothesis import HealthCheck, example, given, settings +from hypothesis import strategies as st +from sqlalchemy.exc import OperationalError + +from microcosm.build.telemetry_emitter import LocalTelemetryEmitter, TelemetryRun +from microcosm.build.telemetry_emitter_constants import DEFAULT_SEND_TIMEOUT_SECONDS +from microcosm.build.telemetry_emitter_service import runtime as runtime_module +from microcosm.build.telemetry_emitter_service.constants import ( + DATABASE_TIMEOUT_SECONDS, + QUEUE_MAX_BYTES, + QUEUE_MAX_EVENTS, + QUEUE_RESERVED_BYTES, + QUEUE_RESERVED_EVENTS, + SPOOL_RETRY_MAX_SECONDS, + WRITER_BATCH_EVENTS, + WRITER_BUSY_TIMEOUT_SECONDS, +) +from microcosm.build.telemetry_emitter_service.contention import ( + is_transient_spool_error, + retry_spool_contention, +) +from microcosm.build.telemetry_emitter_service.event_queue import ( + EventQueue, + QueueClosedError, + QueuedEvent, + QueueFullError, + SpoolWriter, +) +from microcosm.build.telemetry_emitter_service.runtime import EmitterService +from microcosm.build.telemetry_emitter_service.spool import EventSpool +from microcosm.build.telemetry_protocol import ( + BUILD_COMPLETED_MESSAGE, + BUILD_STARTED_MESSAGE, + LOCAL_ACKNOWLEDGEMENT_ERROR, + LOCAL_ACKNOWLEDGEMENT_OK, + MAX_TELEMETRY_DETAILS_BYTES, + MAX_TELEMETRY_MESSAGE_CHARS, + UNEXPECTED_PROCESS_EXIT_MESSAGE, +) +from microcosm.build.telemetry_sanitization import sanitize_details, sanitize_text + +_LOOPBACK_COLLECTOR = "http://127.0.0.1:9" + + +def _registration(run_id: str = "run-a") -> dict[str, object]: + return TelemetryRun( + run_id=run_id, + country_code="US", + pipeline="test-pipeline", + producer_id="producer-a", + ).as_registration() + + +def _event(stage_id: str, **details: object) -> dict[str, object]: + return { + "event_type": "stage", + "stage_id": stage_id, + "status": "started", + "details": details, + } + + +def _lock_error() -> OperationalError: + driver_error = sqlite3.OperationalError("database is locked") + driver_error.sqlite_errorcode = sqlite3.SQLITE_BUSY + return OperationalError("INSERT INTO telemetry_events ...", {}, driver_error) + + +@contextlib.contextmanager +def _write_lock(path: Path): + """Hold the spool's write lock from another connection, as a writer would.""" + + connection = sqlite3.connect(path, isolation_level=None) + try: + connection.execute("BEGIN IMMEDIATE") + yield connection + finally: + if connection.in_transaction: + connection.rollback() + connection.close() + + +def _short_socket_path() -> Path: + # macOS limits AF_UNIX paths to roughly 100 bytes. + return Path(tempfile.mkdtemp(prefix="microcosm-test-", dir="/tmp")) / "e.sock" + + +def _stored_stages(path: Path) -> list[str]: + """Stage ids on disk in sequence order, read without the service's spool. + + A reader is not blocked by another connection's ``BEGIN IMMEDIATE``. + """ + + connection = sqlite3.connect(path) + try: + rows = connection.execute( + "SELECT payload_json FROM telemetry_events ORDER BY sequence" + ).fetchall() + finally: + connection.close() + return [json.loads(payload)["stage_id"] for (payload,) in rows] + + +def _stored_events(path: Path, run_id: str, producer_id: str) -> list[dict]: + spool = EventSpool(path) + try: + return spool.batch(run_id, producer_id, limit=100_000) + finally: + spool._engine.dispose() + + +class _Clock: + """A fake monotonic clock; sleeping advances it.""" + + def __init__(self, now: float = 1_000.0) -> None: + self.now = now + self.sleeps: list[tuple[float, float]] = [] + + def monotonic(self) -> float: + return self.now + + def sleep(self, seconds: float) -> None: + self.sleeps.append((self.now, seconds)) + self.now += seconds + + +def _service(spool, **overrides) -> EmitterService: + arguments = { + "socket_path": Path("/unused"), + "registration": _registration(), + "spool": spool, + "delivery": SimpleNamespace(flush_once=lambda: False), + "sampler": SimpleNamespace(sample=dict, parent_alive=lambda: True), + "heartbeat_seconds": 60, + } + return EmitterService(**(arguments | overrides)) + + +def _send(service: EmitterService, event: dict[str, object]) -> bool: + """Hand one event to the service as the socket thread would; True for ``ok``.""" + + try: + service._handle({"action": "event", "event": event}) + except Exception: + return False + return True + + +# --- The queue's bound ---------------------------------------------------------- + + +@settings(max_examples=300) +@given( + max_events=st.integers(1, 12), + reserved_events=st.integers(0, 6), + max_bytes=st.integers(1, 400), + reserved_bytes=st.integers(0, 200), + operations=st.lists( + st.one_of( + st.tuples( + st.just("offer"), + st.integers(1, 80), # encoded size + st.booleans(), # run event + st.sampled_from([False, False, False, True]), # close behind it + ), + st.tuples(st.just("remove"), st.integers(0, 4)), + ), + max_size=80, + ), +) +def test_queue_takes_exactly_what_fits_keeps_order_and_never_exceeds_its_bound( + max_events, reserved_events, max_bytes, reserved_bytes, operations +) -> None: + queue = EventQueue( + max_events=max_events, + max_bytes=max_bytes, + reserved_events=reserved_events, + reserved_bytes=reserved_bytes, + ) + model: list[QueuedEvent] = [] + closed = False + for operation in operations: + if operation[0] == "remove": + count = min(operation[1], len(model)) + queue.remove(count) + del model[:count] + else: + _, size, run_event, close = operation + event = QueuedEvent(b"x" * size, run_event) + # Reference rule: a run event may use the reserve; nothing else may. + limit_events = max_events - (0 if run_event else reserved_events) + limit_bytes = max_bytes - (0 if run_event else reserved_bytes) + fits = ( + len(model) < limit_events + and sum(e.size for e in model) + size <= limit_bytes + ) + expected = ( + QueueClosedError if closed else (None if fits else QueueFullError) + ) + try: + queue.offer(event, close=close) + except (QueueClosedError, QueueFullError) as error: + assert type(error) is expected + else: + assert expected is None + model.append(event) + if not run_event: + # An ordinary event never takes room held for run events. + assert len(model) <= max_events - reserved_events + assert sum(e.size for e in model) <= max_bytes - reserved_bytes + closed = closed or close + # The bound holds after every step, and the queue is exactly the model. + assert len(queue) == len(model) <= max_events + assert queue.queued_bytes == sum(e.size for e in model) <= max_bytes + oldest = queue.oldest(len(model) + 1) + assert len(oldest) == len(model) + assert all(a is b for a, b in zip(oldest, model, strict=True)) + assert queue.closed is closed + + +def test_the_reserve_holds_every_run_event_of_a_build_at_the_clients_limits() -> None: + """Started, completed or failed, and the service's record of a killed build.""" + + largest_message = sanitize_text( + "\U0001f600" * 10_000, limit=MAX_TELEMETRY_MESSAGE_CHARS + ) + largest_details = sanitize_details( + {f"key-{n}": "\U0001f600" * 50 for n in range(500)} + ) + run_event = { + "timestamp": "2026-10-09T12:00:00.000000+00:00", + "event_type": "run", + "stage_id": "complete", + "status": "completed", + "message": largest_message, + "details": largest_details, + } + sample = { + "cpu_user_seconds": 1e12, + "cpu_system_seconds": 1e12, + "rss_bytes": 2**62, + "peak_rss_bytes": 2**62, + } + largest = QueuedEvent.encode(run_event, sample).size + assert largest > MAX_TELEMETRY_DETAILS_BYTES + assert QUEUE_RESERVED_EVENTS >= 3 + assert QUEUE_RESERVED_BYTES >= 3 * largest + assert QUEUE_RESERVED_EVENTS < QUEUE_MAX_EVENTS + assert QUEUE_RESERVED_BYTES < QUEUE_MAX_BYTES + + +def test_queued_bytes_count_the_encoded_event_and_its_sample() -> None: + sampler_reading = {"cpu_user_seconds": 1.5, "rss_bytes": 10} + service = _service( + SimpleNamespace(), + sampler=SimpleNamespace(sample=lambda: sampler_reading), + ) + events = [_event(f"stage-{index}", note="x" * index) for index in range(5)] + for event in events: + assert _send(service, event) + + queued = service.event_queue.oldest(10) + assert [item.decode() for item in queued] == [(e, sampler_reading) for e in events] + assert service.event_queue.queued_bytes == sum( + len(json.dumps([e, sampler_reading], separators=(",", ":")).encode()) + for e in events + ) + + +# --- What the socket acknowledges --------------------------------------------- + + +def _reply(service: EmitterService, message: bytes) -> bytes: + """Run one connection through the service's socket handling.""" + + ours, theirs = socket.socketpair() + with ours, theirs: + ours.sendall(message) + theirs.settimeout(1) + return service._serve_connection(theirs) + + +def test_acknowledgement_means_queued_and_error_means_not_queued(capsys) -> None: + service = _service( + SimpleNamespace(), + event_queue=EventQueue( + max_events=3, max_bytes=10_000, reserved_events=1, reserved_bytes=500 + ), + ) + + def message(event) -> bytes: + return json.dumps({"action": "event", "event": event}).encode() + b"\n" + + # Malformed events are refused and never queued. + assert _reply(service, message({"stage_id": "x"})) == LOCAL_ACKNOWLEDGEMENT_ERROR + assert _reply(service, message([1, 2])) == LOCAL_ACKNOWLEDGEMENT_ERROR + nan = b'{"action":"event","event":{"event_type":"stage","status":"started",' + nan += b'"details":{"x":NaN}}}\n' + assert _reply(service, nan) == LOCAL_ACKNOWLEDGEMENT_ERROR + assert len(service.event_queue) == 0 + + # Two ordinary events fill the room outside the reserve. + assert _reply(service, message(_event("a"))) == LOCAL_ACKNOWLEDGEMENT_OK + assert _reply(service, message(_event("b"))) == LOCAL_ACKNOWLEDGEMENT_OK + for _ in range(3): + assert _reply(service, message(_event("c"))) == LOCAL_ACKNOWLEDGEMENT_ERROR + # A run event still fits: the build's outcome is not crowded out. + completed = {"event_type": "run", "stage_id": "complete", "status": "completed"} + assert _reply(service, message(completed)) == LOCAL_ACKNOWLEDGEMENT_OK + assert [item.decode()[0]["stage_id"] for item in service.event_queue.oldest(9)] == [ + "a", + "b", + "complete", + ] + # One warning line however many updates were refused. + lines = capsys.readouterr().err.splitlines() + assert len(lines) == 1 + assert lines[0].startswith("warning: the local telemetry emitter service's queue") + + # Close refuses everything after it, heartbeats included. + assert _reply(service, b'{"action":"close"}\n') == LOCAL_ACKNOWLEDGEMENT_OK + service.event_queue.remove(3) + assert _reply(service, message(_event("late"))) == LOCAL_ACKNOWLEDGEMENT_ERROR + assert not service._attempt(service._queue_heartbeat) + assert len(service.event_queue) == 0 + assert capsys.readouterr().err == "" + + +# --- The writer: order, no loss under contention, a bounded drain ------------- + + +class _ScriptedSpool: + """Appends like the real spool: in order, all or nothing, numbered. + + Each attempt first takes the next scripted lock outcome; an event whose + details carry ``refused`` makes its whole transaction fail for a reason + other than contention. + """ + + def __init__(self, lock_errors) -> None: + self._lock_errors = iter(lock_errors) + self.stored: list[tuple[str, int]] = [] + self.locked_attempts = 0 + self.busy_timeouts: set[float | None] = set() + + def append_many(self, registration, pairs, *, busy_timeout_seconds=None): + self.busy_timeouts.add(busy_timeout_seconds) + if next(self._lock_errors, False): + self.locked_attempts += 1 + raise _lock_error() + if any(event["details"].get("refused") for event, _ in pairs): + raise ValueError("the spool refused this event") + for event, _ in pairs: + self.stored.append((event["stage_id"], len(self.stored) + 1)) + + +@settings(max_examples=200, deadline=None) +@given( + operations=st.lists( + st.one_of( + st.tuples( + st.just("send"), + st.booleans(), # a run event + st.sampled_from([False] * 5 + [True]), # the spool refuses it + ), + st.just(("write",)), + ), + max_size=60, + ), + lock_errors=st.lists(st.booleans(), max_size=150), + max_events=st.integers(1, 15), + reserved_events=st.integers(0, 4), +) +@example( + operations=[("send", False, False)] * 3 + [("write",)], + lock_errors=[True] * 40, + max_events=5, + reserved_events=0, +) +def test_every_acknowledged_event_is_stored_once_in_acknowledgement_order( + operations, lock_errors, max_events, reserved_events +) -> None: + clock = _Clock() + spool = _ScriptedSpool(lock_errors) + service = _service( + spool, + event_queue=EventQueue( + max_events=max_events, + max_bytes=10**9, + reserved_events=reserved_events, + reserved_bytes=0, + ), + ) + service.writer = SpoolWriter( + spool, + service.registration, + service.event_queue, + report_dropped=service._report_dropped, + clock=clock.monotonic, + sleep=clock.sleep, + ) + acknowledged: list[tuple[str, bool]] = [] + queue_was_full = False + error_output = io.StringIO() + with contextlib.redirect_stderr(error_output): + for index, operation in enumerate(operations): + if operation[0] == "write": + # While serving, the writer outlasts any run of lock errors. + assert service.writer.write_queued() + assert len(service.event_queue) == 0 + continue + _, run_event, refused = operation + stage = f"event-{index}" + event = _event(stage, refused=refused) + if run_event: + event["event_type"] = "run" + if _send(service, event): + acknowledged.append((stage, refused)) + else: + queue_was_full = True + unwritten = service.writer.finish(math.inf) + + kept = [stage for stage, refused in acknowledged if not refused] + # Sequence order is acknowledgement order; nothing acknowledged is lost or + # stored twice, whatever lock errors the spool raised. + assert [stage for stage, _ in spool.stored] == kept + assert [sequence for _, sequence in spool.stored] == list(range(1, len(kept) + 1)) + assert unwritten == 0 + # Only events the spool itself refused are dropped, each once. Each kind of + # warning is one line however often it happens. + assert service.writer.dropped == len(acknowledged) - len(kept) + lines = error_output.getvalue().splitlines() + assert sorted(line.split(" ")[6] for line in lines) == sorted( + (["spool"] if service.writer.dropped else []) + + (["queue"] if queue_was_full else []) + ) + # Lock errors were retried with backoff, never reported. + assert all(seconds <= SPOOL_RETRY_MAX_SECONDS for _, seconds in clock.sleeps) + assert len(clock.sleeps) == spool.locked_attempts + assert spool.busy_timeouts <= {WRITER_BUSY_TIMEOUT_SECONDS} + + +class _LockedUntil: + """A spool another process keeps locked until ``lock_until`` on a fake clock. + + An attempt waits in SQLite's busy handler for up to its statement wait and + succeeds as soon as the lock is released within it. + """ + + def __init__(self, clock: _Clock, lock_until: float, insert_seconds: float): + self.clock = clock + self.lock_until = lock_until + self.insert_seconds = insert_seconds + self.attempt_starts: list[float] = [] + self.stored: list[str] = [] + + def append_many(self, registration, pairs, *, busy_timeout_seconds=None): + started = self.clock.now + self.attempt_starts.append(started) + if self.lock_until > started + busy_timeout_seconds: + self.clock.now = started + busy_timeout_seconds + raise _lock_error() + self.clock.now = max(started, self.lock_until) + self.insert_seconds + self.stored.extend(event["stage_id"] for event, _ in pairs) + + def has_deliverable(self) -> bool: + return False + + +@settings( + max_examples=200, + deadline=None, + suppress_health_check=[HealthCheck.function_scoped_fixture], +) +@given( + events=st.integers(1, 450), + locked_for=st.one_of(st.just(0.0), st.floats(0, 25)), + drain_seconds=st.one_of(st.just(0.0), st.floats(0, 20)), + insert_seconds=st.floats(0, 0.05), +) +@example(events=450, locked_for=3.0, drain_seconds=15.0, insert_seconds=0.05) +@example(events=10, locked_for=30.0, drain_seconds=15.0, insert_seconds=0.0) +def test_the_drain_writes_in_order_until_its_deadline_and_reports_the_rest( + monkeypatch, events, locked_for, drain_seconds, insert_seconds +) -> None: + clock = _Clock() + monkeypatch.setattr(runtime_module, "time", clock) + spool = _LockedUntil(clock, clock.now + locked_for, insert_seconds) + service = _service(spool, drain_seconds=drain_seconds) + sent = [f"event-{index}" for index in range(events)] + for stage in sent: + assert _send(service, _event(stage)) + service._stop.set() + started = clock.now + deadline = started + drain_seconds + error_output = io.StringIO() + + with contextlib.redirect_stderr(error_output): + service._drain() + + last_start = deadline - WRITER_BUSY_TIMEOUT_SECONDS + # No append starts within one statement wait of the deadline, so the last + # ends by the deadline plus its own inserts. + assert all(start < last_start for start in spool.attempt_starts) + assert clock.now <= deadline + insert_seconds + 1e-9 + # What was written is the oldest events, in order. + assert spool.stored == sent[: len(spool.stored)] + unwritten = events - len(spool.stored) + lines = error_output.getvalue().splitlines() + if unwritten: + assert len(lines) == 1 + assert f"could not write {unwritten} queued update(s)" in lines[0] + else: + assert lines == [] + # Contention that clears with room to spare loses nothing: an attempt in + # its busy wait succeeds at the release, a sleeping one within a backoff. + batches = math.ceil(events / WRITER_BATCH_EVENTS) + release = started + locked_for + if release + SPOOL_RETRY_MAX_SECONDS + batches * insert_seconds < last_start: + assert unwritten == 0 + # A lock held through the whole drain writes nothing and loses all. + if release >= deadline: + assert spool.stored == [] + + +def test_a_writer_stops_at_the_drain_deadline_even_mid_retry() -> None: + """The deadline is read after every lock error, so finish reaches a retry.""" + + clock = _Clock() + queue = EventQueue() + queue.offer(QueuedEvent.encode(_event("a"), None)) + spool = _LockedUntil(clock, math.inf, 0.0) + writer = SpoolWriter( + spool, + _registration(), + queue, + report_dropped=pytest.fail, + clock=clock.monotonic, + sleep=clock.sleep, + ) + finish_at = clock.now + 30.0 + real_sleep = clock.sleep + + def sleep(seconds): + real_sleep(seconds) + # The service stops while the writer is retrying an unbounded wait. + if clock.now >= finish_at and writer._drain_deadline == math.inf: + writer._drain_deadline = clock.now + 2.0 + assert clock.now < finish_at + 60, "the writer ignored the drain deadline" + + writer._sleep = sleep + assert not writer.write_queued() + assert is_transient_spool_error(writer.stopped_by) + assert len(queue) == 1 + assert ( + spool.attempt_starts[-1] < writer._drain_deadline - WRITER_BUSY_TIMEOUT_SECONDS + ) + + +# --- The retry helper with a deadline that moves ------------------------------ + + +@given( + outcomes=st.lists( + st.sampled_from(["locked", "locked", "locked", "ok", "fatal"]), + min_size=1, + max_size=40, + ), + finite_from=st.integers(0, 40), + window=st.floats(-1, 5), + jitter_fraction=st.floats(0, 1), +) +def test_retry_reads_a_moving_deadline_after_every_lock_error( + outcomes, finite_from, window, jitter_fraction +) -> None: + clock = SimpleNamespace(now=100.0) + reads: list[float] = [] + decisions: list[tuple[float, float, float]] = [] # (at, wait, deadline read) + attempts: list[float] = [] + brought_forward: list[float] = [] + + def deadline() -> float: + # Infinite until the caller brings it forward, then fixed. + if len(reads) >= finite_from: + if not brought_forward: + brought_forward.append(clock.now + window) + value = brought_forward[0] + else: + value = math.inf + reads.append(value) + return value + + def operation(): + attempts.append(clock.now) + kind = outcomes[min(len(attempts) - 1, len(outcomes) - 1)] + clock.now += 0.01 + if kind == "locked": + raise _lock_error() + if kind == "fatal": + raise ValueError("schema error") + return len(attempts) + + def jitter(low, high): + return low + jitter_fraction * (high - low) + + def sleep(seconds): + decisions.append((clock.now, seconds, reads[-1])) + clock.now += seconds + + try: + retry_spool_contention( + operation, + deadline=deadline, + clock=lambda: clock.now, + sleep=sleep, + jitter=jitter, + ) + except ValueError: + outcome = "fatal" + except OperationalError: + outcome = "locked" + else: + outcome = "ok" + + kinds = [outcomes[min(i, len(outcomes) - 1)] for i in range(len(attempts))] + assert kinds[-1] == outcome + # The deadline is read once per lock error and never before the first try. + assert len(reads) == kinds.count("locked") + # No wait reaches the deadline read just before it. + assert all(at + wait < read for at, wait, read in decisions) + # While it reads infinity the helper never gives up on contention. + if outcome == "locked": + assert reads[-1] < math.inf + if all(read == math.inf for read in reads): + assert outcome != "locked" + + +# --- Batched appends ------------------------------------------------------------ + + +@settings(max_examples=25, deadline=None) +@given( + stages=st.lists(st.text("abc", min_size=1, max_size=3), min_size=1, max_size=12), + cuts=st.lists(st.integers(0, 12), max_size=6), +) +def test_appending_in_batches_matches_appending_one_at_a_time(stages, cuts) -> None: + """Differential: batching changes how many transactions, not what is stored.""" + + boundaries = sorted({0, len(stages), *(min(cut, len(stages)) for cut in cuts)}) + batches = [ + stages[start:end] + for start, end in zip(boundaries, boundaries[1:], strict=False) + ] + registration = _registration() + stored = [] + with tempfile.TemporaryDirectory() as directory: + for name, write in ( + ( + "one-at-a-time", + lambda spool: [ + spool.append(registration, _event(s), resources={"n": 1}) + for s in stages + ], + ), + ( + "batched", + lambda spool: [ + spool.append_many( + registration, [(_event(s), {"n": 1}) for s in batch] + ) + for batch in batches + ], + ), + ): + spool = EventSpool(Path(directory) / f"{name}.sqlite3") + try: + spool.register(registration) + write(spool) + stored.append( + [ + { + key: value + for key, value in payload.items() + if key not in {"event_id", "timestamp"} + } + for payload in spool.batch("run-a", "producer-a", limit=100) + ] + ) + finally: + spool._engine.dispose() + one_at_a_time, batched = stored + assert batched == one_at_a_time + assert [p["sequence"] for p in batched] == list(range(1, len(stages) + 1)) + assert [p["stage_id"] for p in batched] == stages + + +def test_a_batch_is_stored_whole_or_not_at_all_and_waits_only_its_own_wait( + tmp_path, +) -> None: + spool_path = tmp_path / "events.sqlite3" + spool = EventSpool(spool_path) + registration = _registration() + spool.register(registration) + pairs = [(_event(f"s{index}"), None) for index in range(5)] + try: + with _write_lock(spool_path): + started = time.monotonic() + with pytest.raises(OperationalError) as locked: + spool.append_many( + registration, + pairs, + busy_timeout_seconds=WRITER_BUSY_TIMEOUT_SECONDS, + ) + waited = time.monotonic() - started + assert is_transient_spool_error(locked.value) + assert waited < DATABASE_TIMEOUT_SECONDS / 2 + assert _stored_stages(spool_path) == [] + # The failed call changed nothing, so repeating it numbers from 1. + payloads = spool.append_many(registration, pairs) + assert [p["sequence"] for p in payloads] == [1, 2, 3, 4, 5] + assert _stored_stages(spool_path) == [f"s{index}" for index in range(5)] + # The shorter wait applied to that call only. + with spool._lock, spool._engine.connect() as connection: + milliseconds = connection.exec_driver_sql("PRAGMA busy_timeout").scalar() + assert milliseconds == DATABASE_TIMEOUT_SECONDS * 1000 + finally: + spool._engine.dispose() + + +def test_a_commit_a_reader_blocks_stores_nothing_so_a_retry_cannot_duplicate( + tmp_path, +) -> None: + """The lock error can come at COMMIT, after the inserts, and still undoes them.""" + + spool_path = tmp_path / "events.sqlite3" + spool = EventSpool(spool_path) + registration = _registration() + spool.register(registration) + pairs = [(_event(f"s{index}"), None) for index in range(3)] + reader = sqlite3.connect(spool_path, isolation_level=None) + try: + # A read transaction holds a SHARED lock, which a commit must wait out. + reader.execute("BEGIN") + reader.execute("SELECT count(*) FROM telemetry_events").fetchone() + with pytest.raises(OperationalError) as locked: + spool.append_many( + registration, pairs, busy_timeout_seconds=WRITER_BUSY_TIMEOUT_SECONDS + ) + assert is_transient_spool_error(locked.value) + reader.execute("COMMIT") + assert _stored_stages(spool_path) == [] + payloads = spool.append_many(registration, pairs) + assert [p["sequence"] for p in payloads] == [1, 2, 3] + assert _stored_stages(spool_path) == ["s0", "s1", "s2"] + finally: + reader.close() + spool._engine.dispose() + + +# --- Real lock contention ----------------------------------------------------- + + +class _Sampler: + def __init__(self) -> None: + self.alive = True + + def sample(self) -> dict[str, object]: + return {"rss_bytes": 100} + + def parent_alive(self) -> bool: + return self.alive + + +def _start_service(spool_path: Path, **overrides): + spool = EventSpool(spool_path) + registration = _registration() + spool.register(registration) + # No collector here, so the shutdown drain stops once the spool is written. + spool.has_deliverable = lambda: False + socket_path = _short_socket_path() + service = EmitterService( + **( + { + "socket_path": socket_path, + "registration": registration, + "spool": spool, + "delivery": SimpleNamespace(flush_once=lambda: False), + "sampler": _Sampler(), + "heartbeat_seconds": 60, + "drain_seconds": 5, + } + | overrides + ) + ) + serving = threading.Thread(target=service.run, daemon=True) + serving.start() + deadline = time.monotonic() + 10 + while not socket_path.exists(): + assert time.monotonic() < deadline, "service never bound its socket" + time.sleep(0.01) + client = LocalTelemetryEmitter( + run=TelemetryRun("run-a", "US", "test-pipeline", producer_id="producer-a"), + process=None, + socket_path=socket_path, + runtime_dir=None, + ) + return service, serving, client + + +def _stop(service: EmitterService, serving: threading.Thread, client) -> None: + client.close() + service._stop.set() + serving.join(timeout=30) + service.spool._engine.dispose() + + +def test_a_held_spool_lock_neither_delays_nor_loses_a_builds_events( + tmp_path, capsys +) -> None: + """The lock outlasts SQLite's whole 5 s busy wait, which used to drop events.""" + + spool_path = tmp_path / "events.sqlite3" + service, serving, client = _start_service(spool_path) + spool = service.spool + lock_errors: list[float] = [] + append_many = spool.append_many + + def recording(*args, **kwargs): + try: + return append_many(*args, **kwargs) + except Exception as error: + if is_transient_spool_error(error): + lock_errors.append(time.monotonic()) + raise + + spool.append_many = recording + sent: list[str] = [] + send_seconds: list[float] = [] + try: + with _write_lock(spool_path): + held_from = time.monotonic() + while time.monotonic() - held_from < DATABASE_TIMEOUT_SECONDS + 1.0: + stage = f"stage-{len(sent)}" + started = time.monotonic() + client.stage(stage, message="Working.") + send_seconds.append(time.monotonic() - started) + sent.append(stage) + time.sleep(0.02) + # A burst, as a calibration loop sends them. + for _ in range(200): + stage = f"stage-{len(sent)}" + started = time.monotonic() + client.stage(stage) + send_seconds.append(time.monotonic() - started) + sent.append(stage) + # Every reply came while the spool was locked, before any write. + assert _stored_stages(spool_path) == [] + released_at = time.monotonic() + deadline = time.monotonic() + 20 + while len(_stored_stages(spool_path)) < len(sent): + assert time.monotonic() < deadline, "queued events never reached the spool" + time.sleep(0.05) + finally: + _stop(service, serving, client) + + # Not lost, not reordered, not duplicated. + assert _stored_stages(spool_path) == sent + events = _stored_events(spool_path, "run-a", "producer-a") + assert [event["sequence"] for event in events] == list(range(1, len(sent) + 1)) + assert all(event["resources"] == {"rss_bytes": 100} for event in events) + # Not delayed: the client never hit its own timeout or printed its warning. + assert not client._warned + assert max(send_seconds) < DEFAULT_SEND_TIMEOUT_SECONDS + assert "could not queue" not in capsys.readouterr().err + # The writer met the lock repeatedly while it was held. + assert len([at for at in lock_errors if at < released_at]) >= 2 + assert not serving.is_alive() + + +def test_close_drains_events_queued_behind_a_held_lock(tmp_path, capsys) -> None: + spool_path = tmp_path / "events.sqlite3" + service, serving, client = _start_service(spool_path, drain_seconds=10) + sent = [f"stage-{index}" for index in range(30)] + try: + with _write_lock(spool_path): + for stage in sent: + client.stage(stage) + client.close() + closed_at = time.monotonic() + time.sleep(1.0) + # Still locked, still serving its drain. + assert serving.is_alive() + serving.join(timeout=20) + stopped_after = time.monotonic() - closed_at + finally: + _stop(service, serving, client) + + assert not serving.is_alive() + assert stopped_after < 10 + 1 + assert _stored_stages(spool_path) == sent + assert capsys.readouterr().err == "" + + +def test_a_lock_held_through_the_drain_loses_only_the_queue_and_says_so( + tmp_path, capsys +) -> None: + spool_path = tmp_path / "events.sqlite3" + drain_seconds = 1.0 + service, serving, client = _start_service(spool_path, drain_seconds=drain_seconds) + try: + with _write_lock(spool_path): + for index in range(5): + client.stage(f"stage-{index}") + client.close() + closed_at = time.monotonic() + serving.join(timeout=drain_seconds + 5) + stopped_after = time.monotonic() - closed_at + finally: + _stop(service, serving, client) + + assert not serving.is_alive() + # The drain gives up on time: its last attempt starts a statement wait + # before the deadline, and the socket thread's accept wait adds at most + # its own timeout. + assert stopped_after < drain_seconds + 1.5 + assert _stored_stages(spool_path) == [] + lines = capsys.readouterr().err.splitlines() + assert len(lines) == 1 + assert "could not write 5 queued update(s)" in lines[0] + assert "(database is locked)" in lines[0] + + +def test_a_killed_build_is_recorded_after_every_event_it_sent(tmp_path) -> None: + spool_path = tmp_path / "events.sqlite3" + service, serving, client = _start_service(spool_path, drain_seconds=10) + sent = [f"stage-{index}" for index in range(10)] + try: + with _write_lock(spool_path): + for stage in sent: + client.stage(stage) + # The build dies without closing; the worker notices on its tick. + service.sampler.alive = False + deadline = time.monotonic() + 10 + while not service.event_queue.closed: + assert time.monotonic() < deadline, "worker never noticed" + time.sleep(0.05) + time.sleep(0.5) + serving.join(timeout=20) + finally: + _stop(service, serving, client) + + events = _stored_events(spool_path, "run-a", "producer-a") + assert [event["stage_id"] for event in events[:-1]] == sent + assert events[-1]["message"] == UNEXPECTED_PROCESS_EXIT_MESSAGE + assert events[-1]["details"]["failed_during"] == sent[-1] + assert [event["sequence"] for event in events] == list(range(1, len(sent) + 2)) + + +def test_service_stops_serving_when_its_writer_dies(tmp_path, capsys) -> None: + """Acknowledged events would never reach the spool without the writer.""" + + spool_path = tmp_path / "events.sqlite3" + service, serving, client = _start_service(spool_path, drain_seconds=0) + + def bug() -> bool: + raise RuntimeError("writer bug") + + service.writer.write_queued = bug + try: + client.stage("wakes the writer") + serving.join(timeout=10) + assert not serving.is_alive() + finally: + _stop(service, serving, client) + + err = capsys.readouterr().err + assert "spool writer stopped: RuntimeError: writer bug" in err + assert "Traceback" not in err + + +def test_build_sends_through_a_held_spool_lock_end_to_end( + tmp_path, capfd, real_local_telemetry +) -> None: + """The real client and the real service process.""" + + spool_path = tmp_path / "spool" / "events.sqlite3" + emitter = LocalTelemetryEmitter.start( + run_id="held-lock-run", + country_code="US", + pipeline="test-pipeline", + development_collector_url=_LOOPBACK_COLLECTOR, + spool_path=spool_path, + heartbeat_seconds=60, + startup_timeout_seconds=60, + ) + assert emitter.available + send_seconds: list[float] = [] + with _write_lock(spool_path): + held_from = time.monotonic() + done = 0 + while time.monotonic() - held_from < DATABASE_TIMEOUT_SECONDS + 1.0: + started = time.monotonic() + emitter.progress("calibrating", done=done, total=1000) + send_seconds.append(time.monotonic() - started) + done += 1 + time.sleep(0.05) + emitter.complete() + assert emitter._process is not None + assert emitter._process.wait(timeout=60) == 0 + + events = _stored_events(spool_path, "held-lock-run", emitter.run.producer_id) + assert [event["sequence"] for event in events] == list(range(1, len(events) + 1)) + assert events[0]["message"] == BUILD_STARTED_MESSAGE + assert [event["details"]["done"] for event in events[1:-1]] == list(range(done)) + assert events[-1]["message"] == BUILD_COMPLETED_MESSAGE + assert not emitter._warned + assert max(send_seconds) < DEFAULT_SEND_TIMEOUT_SECONDS + err = capfd.readouterr().err + assert "could not queue" not in err + assert "Traceback" not in err diff --git a/packages/microcosm-build/tests/engine_free/shared/test_telemetry_spool_contention.py b/packages/microcosm-build/tests/engine_free/shared/test_telemetry_spool_contention.py index e8feac6f5e..362a5e8c37 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_telemetry_spool_contention.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_telemetry_spool_contention.py @@ -937,7 +937,7 @@ def test_worker_survives_any_step_failure( ), ) outcomes = _Outcomes(script) - appended: list[tuple[int, str]] = [] + appended: list[str] = [] parent_checks: list[int] = [] class Stop: @@ -955,9 +955,11 @@ def set(self): def is_set(self): return self.flag - def append(registration, event, *, resources=None): - outcomes.next() - appended.append((clock.tick, event["event_type"] + ":" + event["status"])) + def append_many(registration, pairs, *, busy_timeout_seconds=None): + # The worker only queues events; the drain's writer stores them. + appended.extend( + event["event_type"] + ":" + event["status"] for event, _ in pairs + ) def parent_alive(): parent_checks.append(clock.tick) @@ -967,7 +969,7 @@ def parent_alive(): socket_path=Path("/unused"), registration=_registration(), spool=SimpleNamespace( - append=append, + append_many=append_many, prune_if_due=outcomes.next, has_deliverable=lambda: outcomes.next(False), ), @@ -980,25 +982,29 @@ def parent_alive(): ) service._stop = Stop() heartbeat_attempts: list[tuple[int, float, bool]] = [] - append_heartbeat = service._append_heartbeat + queue_heartbeat = service._queue_heartbeat def recorded_heartbeat() -> None: try: - append_heartbeat() + queue_heartbeat() except Exception: heartbeat_attempts.append((clock.tick, clock.now, False)) raise heartbeat_attempts.append((clock.tick, clock.now, True)) - service._append_heartbeat = recorded_heartbeat - exit_records: list[int] = [] - append_unexpected_exit = service._append_unexpected_exit + service._queue_heartbeat = recorded_heartbeat + exit_records: list[tuple[int, bool]] = [] + queue_unexpected_exit = service._queue_unexpected_exit def recorded_unexpected_exit() -> None: - exit_records.append(clock.tick) - append_unexpected_exit() + try: + queue_unexpected_exit() + except Exception: + exit_records.append((clock.tick, False)) + raise + exit_records.append((clock.tick, True)) - service._append_unexpected_exit = recorded_unexpected_exit + service._queue_unexpected_exit = recorded_unexpected_exit error_output = io.StringIO() with contextlib.redirect_stderr(error_output): @@ -1010,7 +1016,9 @@ def recorded_unexpected_exit() -> None: # A dead parent is recorded once, on the tick it is found, and stops the # loop. parent_died = parent_dies_at is not None and parent_dies_at <= ticks - assert exit_records == ([parent_dies_at] if parent_died else []) + assert [tick for tick, _ in exit_records] == ( + [parent_dies_at] if parent_died else [] + ) assert service._stop.is_set() is parent_died # Reference heartbeat schedule: due one interval after the start or the # last success; a failed heartbeat stays due and is retried next tick. @@ -1026,6 +1034,13 @@ def recorded_unexpected_exit() -> None: if succeeded: due = now + heartbeat_seconds assert [tick for tick, _, _ in heartbeat_attempts] == expected_ticks + # The drain stores what the worker queued, in order: every heartbeat that + # was queued, then the record of a dead build. + queued_heartbeats = sum(succeeded for _, _, succeeded in heartbeat_attempts) + recorded_exit = any(succeeded for _, succeeded in exit_records) + assert appended == ["heartbeat:progress"] * queued_heartbeats + ( + ["run:failed"] if recorded_exit else [] + ) # One line per error type other than lock contention, which is silent. reported = { name for name in outcomes.raised if name in {"RuntimeError", "ValueError"} @@ -1037,6 +1052,8 @@ def recorded_unexpected_exit() -> None: def test_unexpected_exit_waits_out_lock_contention(monkeypatch) -> None: + """The drain's writer stores a killed build's record once the lock clears.""" + clock = SimpleNamespace(now=0.0) monkeypatch.setattr( runtime_module, @@ -1047,26 +1064,34 @@ def test_unexpected_exit_waits_out_lock_contention(monkeypatch) -> None: ), ) attempts: list[float] = [] + stored: list[str] = [] - def append(registration, event, *, resources=None): + def append_many(registration, pairs, *, busy_timeout_seconds=None): attempts.append(clock.now) clock.now += 0.2 if len(attempts) < 4: raise _lock_error() + stored.extend(event["stage_id"] for event, _ in pairs) service = EmitterService( socket_path=Path("/unused"), registration=_registration(), - spool=SimpleNamespace(append=append), + spool=SimpleNamespace(append_many=append_many, has_deliverable=lambda: False), delivery=SimpleNamespace(), sampler=SimpleNamespace(sample=dict), heartbeat_seconds=60, drain_seconds=15, ) + service._handle({"action": "event", "event": _event("compile")}) + + assert service._attempt(service._queue_unexpected_exit) + service._drain() - assert service._attempt(service._append_unexpected_exit) assert len(attempts) == 4 assert attempts[-1] < 15 + # Behind every event the build sent, and last. + assert stored == ["compile", "failed"] + assert service.event_queue.closed def test_service_stops_serving_when_its_worker_dies(tmp_path) -> None: @@ -1091,6 +1116,8 @@ def test_service_stops_serving_when_its_worker_dies(tmp_path) -> None: serving.join(timeout=10) assert not serving.is_alive() assert not socket_path.exists() + # The service drained in the worker's place and stopped its writer. + assert not service.writer.is_alive() finally: # A failing run must not leave a server thread behind. service._stop.set() From 5e36e6fa86f3461e8c484f186e5ff0b1868c2202 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 9 Oct 2026 12:31:43 -0400 Subject: [PATCH 08/13] Let other writers in while a prune drains, and keep an idle prune read-only From the in-session review: back-to-back prune batches still kept other processes off the spool for the whole prune, because SQLite's busy handler polls rather than queueing and rarely lands in the gap between two transactions; another service's appends waited up to 5.2 s during a 100k-row prune. And the batched DELETEs took the write lock even when nothing was due, where the old prune only read. - prune pauses 25 ms between batches, outside the process lock, and works at most 1 s per worker tick; an unfinished prune resumes on the next tick instead of waiting the 60 s interval. - prune checks read-only for expired events and expired runs before deleting, so an idle prune takes no write lock. Co-Authored-By: Claude Opus 5.5 --- .../telemetry_emitter_service/constants.py | 5 + .../build/telemetry_emitter_service/spool.py | 76 ++++++++--- .../shared/test_telemetry_spool_contention.py | 127 +++++++++++++++--- 3 files changed, 172 insertions(+), 36 deletions(-) diff --git a/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/constants.py b/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/constants.py index f54620415c..d63ba4d87b 100644 --- a/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/constants.py +++ b/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/constants.py @@ -54,6 +54,11 @@ DATABASE_TIMEOUT_SECONDS: Final = 5 PRUNE_INTERVAL_SECONDS: Final = 60.0 PRUNE_BATCH_ROWS: Final = 500 +# SQLite's busy handler polls rather than queueing, so another process rarely +# gets the lock between two back-to-back transactions. Prune pauses between +# batches and works at most PRUNE_STEP_SECONDS per worker tick. +PRUNE_BATCH_PAUSE_SECONDS: Final = 0.025 +PRUNE_STEP_SECONDS: Final = 1.0 # Every concurrent build on a host shares one spool, so a write can find it # locked for longer than SQLite's own busy wait. Startup retries such errors # with jittered, doubling waits until shortly before the build stops waiting diff --git a/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/spool.py b/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/spool.py index 20c0b44e23..1b12f9d712 100644 --- a/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/spool.py +++ b/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/spool.py @@ -2,6 +2,7 @@ from __future__ import annotations +import math import threading import time import uuid @@ -16,8 +17,10 @@ BATCH_SIZE, DATABASE_TIMEOUT_SECONDS, MAX_QUEUED_BYTES, + PRUNE_BATCH_PAUSE_SECONDS, PRUNE_BATCH_ROWS, PRUNE_INTERVAL_SECONDS, + PRUNE_STEP_SECONDS, RETENTION_DAYS, UPLOAD_STATE_LOCAL_ONLY, UPLOAD_STATE_PENDING, @@ -231,10 +234,13 @@ def has_pending(self) -> bool: return session.scalar(statement) is not None def prune_if_due(self) -> None: - """Prune unless an attempt began less than ``PRUNE_INTERVAL_SECONDS`` ago. + """Prune when due, at most ``PRUNE_STEP_SECONDS`` of work per call. - The first call always prunes. A failed attempt still counts, so a spool + The first call prunes. After a prune finishes, or fails, the next waits + ``PRUNE_INTERVAL_SECONDS``: a failed attempt still counts, so a spool that another process keeps locked is not rescanned every worker tick. + A prune that runs out of its step continues on the next call, so a + large backlog drains over several ticks. """ now = time.monotonic() @@ -244,36 +250,64 @@ def prune_if_due(self) -> None: ): return self._last_prune_at = now - self.prune() + if not self.prune(time_budget_seconds=PRUNE_STEP_SECONDS): + self._last_prune_at = None - def prune(self) -> None: + def prune(self, *, time_budget_seconds: float = math.inf) -> bool: """Enforce the age and total-size retention limits. Rows go in batches of ``PRUNE_BATCH_ROWS``, each its own short - transaction, so a large backlog never holds the spool's write lock, or - the lock this process's event path shares, for long. + transaction, with a ``PRUNE_BATCH_PAUSE_SECONDS`` pause between batches + outside this process's lock, so other processes' writers and this + build's events get in while a backlog drains. A spool with nothing to + remove is only read. Returns whether the prune finished; it stops + between batches once ``time_budget_seconds`` have passed. """ + deadline = time.monotonic() + time_budget_seconds cutoff = (datetime.now(UTC) - timedelta(days=RETENTION_DAYS)).isoformat() - expired = ( - select(TelemetryEventRecord.event_id) - .where(TelemetryEventRecord.created_at < cutoff) - .limit(PRUNE_BATCH_ROWS) - ) - while self._delete_events(expired) == PRUNE_BATCH_ROWS: - pass + is_expired = TelemetryEventRecord.created_at < cutoff + if self._any(select(TelemetryEventRecord.event_id).where(is_expired)): + expired = ( + select(TelemetryEventRecord.event_id) + .where(is_expired) + .limit(PRUNE_BATCH_ROWS) + ) + while self._delete_events(expired) == PRUNE_BATCH_ROWS: + if not self._pause_between_batches(deadline): + return False oldest_first = self._oldest_events_over_size_limit() for start in range(0, len(oldest_first), PRUNE_BATCH_ROWS): + if start and not self._pause_between_batches(deadline): + return False self._delete_events(oldest_first[start : start + PRUNE_BATCH_ROWS]) - with self._lock, self._session_factory.begin() as session: - session.execute( - delete(TelemetryRunRecord) - .where( - TelemetryRunRecord.updated_at < cutoff, - ~TelemetryRunRecord.events.any(), + is_expired_run = ( + TelemetryRunRecord.updated_at < cutoff, + ~TelemetryRunRecord.events.any(), + ) + if self._any(select(TelemetryRunRecord.run_id).where(*is_expired_run)): + with self._lock, self._session_factory.begin() as session: + session.execute( + delete(TelemetryRunRecord) + .where(*is_expired_run) + .execution_options(synchronize_session=False) ) - .execution_options(synchronize_session=False) - ) + return True + + def _any(self, statement) -> bool: + """Return whether a read-only query matches any row.""" + + with self._lock, self._session_factory() as session: + return session.scalar(statement.limit(1)) is not None + + @staticmethod + def _pause_between_batches(deadline: float) -> bool: + """Pause before the next batch, or return False when out of time.""" + + if time.monotonic() + PRUNE_BATCH_PAUSE_SECONDS >= deadline: + return False + time.sleep(PRUNE_BATCH_PAUSE_SECONDS) + return True def _delete_events(self, event_ids) -> int: """Delete the selected or listed events in one transaction.""" diff --git a/packages/microcosm-build/tests/engine_free/shared/test_telemetry_spool_contention.py b/packages/microcosm-build/tests/engine_free/shared/test_telemetry_spool_contention.py index 0c06f7773b..9ac70b47f9 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_telemetry_spool_contention.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_telemetry_spool_contention.py @@ -44,6 +44,7 @@ from microcosm.build.telemetry_emitter_service.constants import ( DATABASE_TIMEOUT_SECONDS, PRUNE_INTERVAL_SECONDS, + PRUNE_STEP_SECONDS, READY_DEADLINE_MARGIN_SECONDS, RETENTION_DAYS, SPOOL_LOCKED_EXIT_STATUS, @@ -744,23 +745,28 @@ def test_appends_never_prune(tmp_path, monkeypatch) -> None: @given( gaps=st.lists(st.floats(0, 3 * PRUNE_INTERVAL_SECONDS), max_size=30), - failures=st.lists(st.booleans(), max_size=30), + results=st.lists( + st.sampled_from(["finished", "unfinished", "failed"]), max_size=30 + ), ) @settings(suppress_health_check=[HealthCheck.function_scoped_fixture]) -def test_prune_runs_at_most_once_per_interval_counting_failures( - monkeypatch, gaps, failures +def test_prune_runs_once_per_interval_until_a_backlog_is_drained( + monkeypatch, gaps, results ) -> None: clock = SimpleNamespace(now=1_000.0) monkeypatch.setattr( spool_module, "time", SimpleNamespace(monotonic=lambda: clock.now) ) - attempts: list[float] = [] - outcomes = iter(failures) - - def prune() -> None: - attempts.append(clock.now) - if next(outcomes, False): + attempts: list[tuple[float, str]] = [] + outcomes = iter(results) + + def prune(*, time_budget_seconds): + assert time_budget_seconds == PRUNE_STEP_SECONDS + result = next(outcomes, "finished") + attempts.append((clock.now, result)) + if result == "failed": raise _lock_error() + return result == "finished" # prune_if_due reads only the schedule and prune, so no database is needed. spool = EventSpool.__new__(EventSpool) @@ -774,13 +780,19 @@ def prune() -> None: with contextlib.suppress(OperationalError): spool.prune_if_due() - # Reference schedule: the first call prunes; later calls prune once the - # interval has passed since the previous attempt, whether or not it failed. + # Reference schedule: the first call prunes. A later call prunes at once + # when the previous prune ran out of its step; otherwise once the interval + # has passed since the previous attempt, whether it finished or failed. expected: list[float] = [] for now in calls: - if not expected or now - expected[-1] >= PRUNE_INTERVAL_SECONDS: + previous = attempts[len(expected) - 1] if expected else None + if ( + previous is None + or previous[1] == "unfinished" + or now - previous[0] >= PRUNE_INTERVAL_SECONDS + ): expected.append(now) - assert attempts == expected + assert [at for at, _ in attempts] == expected # --- Lock-error classification ------------------------------------------------ @@ -1363,9 +1375,10 @@ def _stamp(age_days: float, index: int) -> str: run_ages=st.lists(st.floats(0, 2 * RETENTION_DAYS), min_size=3, max_size=3), cap_fraction=st.floats(0, 1.2), batch_rows=st.integers(1, 4), + step_budget=st.sampled_from([0.0, math.inf]), ) def test_batched_prune_matches_the_single_pass_retention_rule( - tmp_path_factory, events, run_ages, cap_fraction, batch_rows + tmp_path_factory, events, run_ages, cap_fraction, batch_rows, step_budget ) -> None: """Differential: the batched prune keeps exactly what one pass would. @@ -1434,9 +1447,15 @@ def recording_delete(event_ids) -> int: with ( mock.patch.object(spool_module, "MAX_QUEUED_BYTES", cap), mock.patch.object(spool_module, "PRUNE_BATCH_ROWS", batch_rows), + mock.patch.object(spool_module, "PRUNE_BATCH_PAUSE_SECONDS", 0.0), mock.patch.object(spool_module, "datetime", frozen_now), ): - spool.prune() + # A zero budget stops after every batch, so the prune is resumed + # call by call until it reports that it finished. + calls = 1 + while not spool.prune(time_budget_seconds=step_budget): + calls += 1 + assert calls <= len(events) + 3 with spool._session_factory() as session: kept_events = set( session.scalars(select(TelemetryEventRecord.event_id)).all() @@ -1469,3 +1488,81 @@ def recording_delete(event_ids) -> int: assert kept_runs == expected_runs # Each transaction deleted at most one batch. assert all(deleted <= batch_rows for deleted in batches) + + +def test_a_prune_with_nothing_to_remove_only_reads(tmp_path) -> None: + """An idle prune runs every minute in every service; it takes no lock.""" + + spool_path = tmp_path / "events.sqlite3" + spool = EventSpool(spool_path) + try: + registration = _registration() + spool.register(registration) + spool.append(registration, _event()) + with _write_lock(spool_path): + started = time.monotonic() + assert spool.prune() is True + assert time.monotonic() - started < 1.0 + assert spool.has_pending() + finally: + _close(spool) + + +def test_prune_pauses_between_batches_outside_the_process_lock( + tmp_path, monkeypatch +) -> None: + spool = EventSpool(tmp_path / "events.sqlite3") + pauses: list[bool] = [] + try: + registration = _registration() + spool.register(registration) + for _ in range(5): + spool.append(registration, _event()) + expired = (datetime.now(UTC) - timedelta(days=RETENTION_DAYS + 1)).isoformat() + with spool._session_factory.begin() as session: + session.query(TelemetryEventRecord).update({"created_at": expired}) + + def pause(seconds: float) -> None: + # This build's event path takes the same lock, so it must be free. + pauses.append(spool._lock._is_owned()) + + monkeypatch.setattr(spool_module, "PRUNE_BATCH_ROWS", 2) + monkeypatch.setattr( + spool_module, + "time", + SimpleNamespace(monotonic=time.monotonic, sleep=pause), + ) + assert spool.prune() is True + assert not spool.has_pending() + finally: + _close(spool) + + # Five rows in batches of two: a pause after each full batch, never while + # this process holds its lock. + assert pauses == [False, False] + + +def test_a_prune_out_of_time_stops_between_batches_and_resumes( + tmp_path, monkeypatch +) -> None: + spool = EventSpool(tmp_path / "events.sqlite3") + try: + registration = _registration() + spool.register(registration) + for _ in range(5): + spool.append(registration, _event()) + expired = (datetime.now(UTC) - timedelta(days=RETENTION_DAYS + 1)).isoformat() + with spool._session_factory.begin() as session: + session.query(TelemetryEventRecord).update({"created_at": expired}) + monkeypatch.setattr(spool_module, "PRUNE_BATCH_ROWS", 2) + monkeypatch.setattr(spool_module, "PRUNE_BATCH_PAUSE_SECONDS", 0.0) + + remaining = [] + while not spool.prune(time_budget_seconds=0.0): + remaining.append(len(spool.batch("run-a", "producer-a", limit=10))) + remaining.append(len(spool.batch("run-a", "producer-a", limit=10))) + finally: + _close(spool) + + # One batch per call: five rows go two, two, then one. + assert remaining == [3, 1, 0] From 7375c18e88a3ad7891e0775aa4757d4b4f93a3fa Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 9 Oct 2026 12:35:40 -0400 Subject: [PATCH 09/13] State every way an acknowledged event can still be lost Co-Authored-By: Claude Opus 5.5 --- .../build/telemetry_emitter_service/runtime.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/runtime.py b/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/runtime.py index 6b7a26af5e..2c1670141f 100644 --- a/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/runtime.py +++ b/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/runtime.py @@ -105,17 +105,20 @@ class EmitterService: The reply is the acknowledgement contract with the build: - ``ok`` to an event means the event was valid and is queued in memory - behind every event acknowledged before it. The writer appends queued - events to the spool in that order, and the spool assigns their producer - sequences as it does, so sequence order is acknowledgement order. + behind every event queued before it, this service's heartbeats + included. The writer appends queued events to the spool in that order, + and the spool assigns their producer sequences as it does, so the + sequence order of stored events is acknowledgement order. - ``ok`` does not mean the event is on disk, and the reply never waits for the spool. When the spool is free the event is written moments later. While another process holds the spool's lock the event waits in the queue, and the writer retries for as long as the contention lasts. - An acknowledged event is lost only if this process is killed before - writing it, if the spool refuses it for a reason other than lock - contention (reported as one warning line), or if the spool stays locked - through the whole shutdown drain (reported with a count). + writing it; if the spool refuses it for a reason other than lock + contention (one warning line per error type); if the writer thread + fails (one line, and the service stops serving); or if the shutdown + drain ends before the writer reaches it, as when the spool stays locked + throughout the drain (one line with the count). - ``error`` means the message was malformed, or the queue was full or closed. The event was not queued. """ From d7fda0def47722aa85c37ee119c51526b6be88e8 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 9 Oct 2026 12:35:41 -0400 Subject: [PATCH 10/13] Describe the review fixes in the changelog Co-Authored-By: Claude Opus 5.5 --- changelog.d/telemetry-spool-lock-startup.fixed.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/changelog.d/telemetry-spool-lock-startup.fixed.md b/changelog.d/telemetry-spool-lock-startup.fixed.md index 31c65a653a..cf76b5e964 100644 --- a/changelog.d/telemetry-spool-lock-startup.fixed.md +++ b/changelog.d/telemetry-spool-lock-startup.fixed.md @@ -1,5 +1,5 @@ -A build on a busy host no longer loses its hosted telemetry when another build holds the shared local spool (`~/.cache/microcosm/telemetry/events.sqlite3`) while its emitter service starts. Before, the service's registration gave up after SQLite's 5 s busy wait, the service exited, and a SQLAlchemy traceback landed on the build's stderr. Now the build passes the time it will stop waiting (`--ready-deadline`). The service retries lock errors with jittered backoff until 2 s before then, with each SQLite statement waiting at most 0.25 s, and never for more than 60 s in all. If the spool is still locked, it prints one warning line and exits with status 75 before the build gives up. Any other startup failure is also one line, with status 1. +A build on a busy host no longer loses its hosted telemetry when another build holds the shared local spool (`~/.cache/microcosm/telemetry/events.sqlite3`) while its emitter service starts. Before, the service's registration gave up after SQLite's 5 s busy wait, the service exited, and a SQLAlchemy traceback landed on the build's stderr. Now the build passes the time it will stop waiting (`--ready-deadline`). The service retries lock errors with jittered backoff until 2 s before then, with each SQLite statement waiting at most 0.25 s, and never for more than 60 s in all. If the spool is still locked, it prints one warning line and exits with status 75 before the build gives up. Any other startup failure is also one line: status 2 for invalid arguments, 1 otherwise. SIGINT ends the service quietly, as SIGTERM does. -Opening a spool already at the current schema takes no write lock. Retention pruning moved from spool open and every append to the delivery worker, and it deletes in batches of 500 rows, each in its own short transaction. Migrations run in one `BEGIN IMMEDIATE` transaction. Concurrent first opens of a new spool no longer fail with "table already exists"; about half of eight simultaneous opens did before. An interrupted migration leaves no partial schema, and a spool stamped by a migration this checkout does not know is refused without taking the lock. +Opening a spool already at the current schema takes no write lock. Retention pruning moved from spool open and every append to the delivery worker. It deletes in batches of 500 rows, each in its own short transaction, pauses 25 ms between batches so other builds' writers get the lock, and works at most 1 s per worker tick. A prune with nothing to remove only reads. Migrations run in one `BEGIN IMMEDIATE` transaction. Concurrent first opens of a new spool no longer fail with "table already exists"; about half of eight simultaneous opens did before. An interrupted migration leaves no partial schema, and a spool stamped by a migration this checkout does not know is refused without taking the lock. -The delivery worker skips a failed step for one tick instead of dying. Heartbeats, delivery, and the check that records a killed build keep running. Lock contention is retried silently, and any other error prints one line per error type. An unexpected delivery error backs off like a collector failure. Warnings never raise on a closed stderr, and the service stops serving if its worker ever dies. +The delivery worker skips a failed step for one tick instead of dying. Heartbeats, delivery, and the check that records a killed build keep running. Lock contention is retried silently, and any other error prints one line per error type. An unexpected delivery error backs off like a collector failure. Warnings never raise on a closed stderr, and the service stops serving if its worker ever dies. A build that closes and exits while a worker step is still running is no longer recorded as an unexpected exit after its completion. From 8e652a9ea5a7e4ab3b2bfea40430eef6b26c50a5 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 9 Oct 2026 19:54:01 -0400 Subject: [PATCH 11/13] Wait for the lock as long as the build does, and close the reviews' gaps Startup, from the in-session review: with main's 3 s client budget the 2 s margin and 0.25 s SQLite waits made the service give up after about 1 s, where the old single 5 s busy wait rode out a 1.5 s hold. Now every SQLite wait during startup ends by the startup deadline (0.5 s before the build stops waiting) and lasts at most the normal 5 s; opening and registering share one retry loop; and without --ready-deadline startup makes one attempt with the normal wait, as before. From the GPT-6.1 Sol review of bf2ec8e57: - A retry no longer starts after its deadline when the backoff sleep overruns. - A build that closes within the worker's first tick still gets one bounded prune step, so short builds cannot skip retention forever. - Expired runs are deleted in batches like events, and the size check reads the file's page counts before it sums every payload, so a spool under the cap is not scanned. - The contention tests wait on observed attempts and thread state instead of fixed sleeps. Tests, from the tests-and-claims lens: the classification check is exhaustive with literal codes, the deadline property states behaviour rather than the formula, the prune differential covers more expired rows than one batch with no size pressure and an exact-fit cap, main() runs under a timeout, the opener test waits for BEGIN IMMEDIATE, and socket directories and service processes are cleaned up. Co-Authored-By: Claude Opus 5.5 --- README.md | 8 +- .../telemetry-spool-lock-startup.fixed.md | 2 +- .../telemetry_emitter_service/constants.py | 12 +- .../telemetry_emitter_service/contention.py | 6 +- .../build/telemetry_emitter_service/main.py | 36 +- .../telemetry_emitter_service/migrations.py | 10 +- .../telemetry_emitter_service/runtime.py | 3 + .../build/telemetry_emitter_service/spool.py | 73 +++- .../shared/test_telemetry_spool_contention.py | 406 ++++++++++++++---- 9 files changed, 427 insertions(+), 129 deletions(-) diff --git a/README.md b/README.md index 4dd5b985da..996ff49a56 100644 --- a/README.md +++ b/README.md @@ -50,10 +50,10 @@ Supported US and UK build commands always start the local telemetry emitter service. It reports live progress to the hosted collector when the operator's existing Hugging Face login is accepted; otherwise it retains the events locally and the build continues. Every build on a host shares one local -event spool; a service that finds it locked by another process keeps retrying -until shortly before the build stops waiting for it, and one that still cannot -register prints a single warning line, without a traceback, before the build -continues without hosted telemetry. This live event path is independent +event spool; a service that finds it locked by another process waits for it +until shortly before the build stops waiting. One that still cannot register +prints one warning line, without a traceback, and the build adds its own +warning and runs without telemetry, hosted or local. This live event path is independent of the staging run files described below. US fiscal refresh builds also write pre-release staging run files **by diff --git a/changelog.d/telemetry-spool-lock-startup.fixed.md b/changelog.d/telemetry-spool-lock-startup.fixed.md index cf76b5e964..387d66cccc 100644 --- a/changelog.d/telemetry-spool-lock-startup.fixed.md +++ b/changelog.d/telemetry-spool-lock-startup.fixed.md @@ -1,4 +1,4 @@ -A build on a busy host no longer loses its hosted telemetry when another build holds the shared local spool (`~/.cache/microcosm/telemetry/events.sqlite3`) while its emitter service starts. Before, the service's registration gave up after SQLite's 5 s busy wait, the service exited, and a SQLAlchemy traceback landed on the build's stderr. Now the build passes the time it will stop waiting (`--ready-deadline`). The service retries lock errors with jittered backoff until 2 s before then, with each SQLite statement waiting at most 0.25 s, and never for more than 60 s in all. If the spool is still locked, it prints one warning line and exits with status 75 before the build gives up. Any other startup failure is also one line: status 2 for invalid arguments, 1 otherwise. SIGINT ends the service quietly, as SIGTERM does. +A build on a busy host no longer loses its hosted telemetry when another build holds the shared local spool (`~/.cache/microcosm/telemetry/events.sqlite3`) while its emitter service starts. Before, the service's registration gave up after SQLite's 5 s busy wait, the service exited, and a SQLAlchemy traceback landed on the build's stderr. Now the build passes the time it will stop waiting (`--ready-deadline`). The service waits for the lock until 0.5 s before then: each SQLite wait ends by that point and lasts at most 5 s, a failed attempt is retried with jittered backoff, and the whole wait is capped at 60 s. Without a deadline it makes one 5 s attempt, as before. If the spool is still locked, it prints one warning line and exits with status 75 before the build gives up. Any other startup failure is also one line: status 2 for invalid arguments, 1 otherwise. SIGINT ends the service quietly, as SIGTERM does. Opening a spool already at the current schema takes no write lock. Retention pruning moved from spool open and every append to the delivery worker. It deletes in batches of 500 rows, each in its own short transaction, pauses 25 ms between batches so other builds' writers get the lock, and works at most 1 s per worker tick. A prune with nothing to remove only reads. Migrations run in one `BEGIN IMMEDIATE` transaction. Concurrent first opens of a new spool no longer fail with "table already exists"; about half of eight simultaneous opens did before. An interrupted migration leaves no partial schema, and a spool stamped by a migration this checkout does not know is refused without taking the lock. diff --git a/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/constants.py b/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/constants.py index dd6a74519b..381db5ea64 100644 --- a/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/constants.py +++ b/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/constants.py @@ -66,15 +66,13 @@ PRUNE_BATCH_PAUSE_SECONDS: Final = 0.025 PRUNE_STEP_SECONDS: Final = 1.0 # Every concurrent build on a host shares one spool, so a write can find it -# locked for longer than SQLite's own busy wait. Startup retries such errors -# with jittered, doubling waits until shortly before the build stops waiting -# for readiness. While it does, each SQLite statement waits at most -# STARTUP_BUSY_TIMEOUT_SECONDS, so an attempt begun before that deadline ends -# within the margin, which also covers binding the socket and the build's ping. -STARTUP_BUSY_TIMEOUT_SECONDS: Final = 0.25 +# locked for longer than SQLite's own busy wait. Startup waits for the lock, +# and retries lock errors with jittered, doubling pauses, until the margin +# before the build stops waiting for readiness; every SQLite wait in that time +# also ends by then. The margin covers binding the socket and the build's ping. SPOOL_RETRY_INITIAL_SECONDS: Final = 0.05 SPOOL_RETRY_MAX_SECONDS: Final = 0.25 -READY_DEADLINE_MARGIN_SECONDS: Final = 2.0 +READY_DEADLINE_MARGIN_SECONDS: Final = 0.5 STARTUP_RETRY_LIMIT_SECONDS: Final = 60.0 SPOOL_LOCKED_EXIT_STATUS: Final = 75 # EX_TEMPFAIL from sysexits.h diff --git a/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/contention.py b/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/contention.py index 06dc14d700..de0655b4e4 100644 --- a/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/contention.py +++ b/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/contention.py @@ -52,7 +52,7 @@ def retry_spool_contention[T]( transient lock error the next attempt waits a jittered backoff that starts at ``SPOOL_RETRY_INITIAL_SECONDS`` and doubles up to ``SPOOL_RETRY_MAX_SECONDS``. When that attempt would not start before - ``deadline``, the last lock error is raised instead. Any other error is + ``deadline``, or the wait overran it, the last lock error is raised instead. Any other error is raised at once. Each attempt may itself wait in SQLite's busy handler, so this bounds when attempts start, not when the last one ends. """ @@ -67,5 +67,9 @@ def retry_spool_contention[T]( wait = jitter(delay / 2, delay) if clock() + wait >= deadline: raise + contended = error sleep(wait) + if clock() >= deadline: + # The sleep overran, as it can on a loaded host. + raise contended delay = min(SPOOL_RETRY_MAX_SECONDS, delay * 2) diff --git a/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/main.py b/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/main.py index 068180bd27..e377f457dc 100644 --- a/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/main.py +++ b/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/main.py @@ -11,7 +11,6 @@ from microcosm.build.telemetry_emitter_service.collector import CollectorDelivery from microcosm.build.telemetry_emitter_service.constants import ( - DATABASE_TIMEOUT_SECONDS, DEFAULT_HEARTBEAT_SECONDS, READY_DEADLINE_ERROR, READY_DEADLINE_MARGIN_SECONDS, @@ -19,7 +18,6 @@ SERVICE_FAILED_WARNING, SPOOL_LOCKED_EXIT_STATUS, SPOOL_LOCKED_WARNING, - STARTUP_BUSY_TIMEOUT_SECONDS, STARTUP_RETRY_LIMIT_SECONDS, ) from microcosm.build.telemetry_emitter_service.contention import ( @@ -81,18 +79,18 @@ def startup_deadline( *, now_unix: float, now_monotonic: float, -) -> float: - """Return the ``time.monotonic()`` reading at which startup stops retrying. +) -> float | None: + """Return the ``time.monotonic()`` reading at which startup gives up. The build passes its own give-up time as Unix time, the clock both processes share. The margin leaves time to bind the socket and answer the build's ping before the build gives up. The limit bounds the wait however far the deadline or the wall clock moves, so a service whose build has died - still gives up. + still gives up. Without a deadline from the build, there is none here. """ if ready_deadline is None: - return -math.inf + return None remaining = min(ready_deadline - now_unix, STARTUP_RETRY_LIMIT_SECONDS) return now_monotonic + remaining - READY_DEADLINE_MARGIN_SECONDS @@ -101,22 +99,30 @@ def open_registered_spool( path: Path, registration: dict[str, object], *, - deadline: float, + deadline: float | None, ) -> EventSpool: """Open the spool and register this producer, waiting out lock contention. - Each SQLite statement waits at most ``STARTUP_BUSY_TIMEOUT_SECONDS`` for - another process's lock, so this retry loop, bounded by ``deadline``, decides - when to give up. The spool it returns waits the normal - ``DATABASE_TIMEOUT_SECONDS``. + With a ``deadline``, startup waits for another process's lock until then: + each SQLite wait ends by it, and an attempt that fails on the lock is + retried while one can still start before it. Without one, a single attempt + waits as long as any statement does, ``DATABASE_TIMEOUT_SECONDS``. The spool + returned waits normally. """ + opened: list[EventSpool] = [] + + def attempt() -> EventSpool: + if not opened: + opened.append(EventSpool(path, busy_deadline=deadline)) + opened[0].register(registration) + return opened[0] + spool = retry_spool_contention( - lambda: EventSpool(path, busy_timeout_seconds=STARTUP_BUSY_TIMEOUT_SECONDS), - deadline=deadline, + attempt, + deadline=-math.inf if deadline is None else deadline, ) - retry_spool_contention(lambda: spool.register(registration), deadline=deadline) - spool.busy_timeout_seconds = DATABASE_TIMEOUT_SECONDS + spool.busy_deadline = None return spool diff --git a/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/migrations.py b/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/migrations.py index 3550c07149..5535eb799c 100644 --- a/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/migrations.py +++ b/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/migrations.py @@ -2,7 +2,7 @@ from __future__ import annotations -from collections.abc import Iterator +from collections.abc import Callable, Iterator from contextlib import contextmanager from importlib.resources import as_file, files from pathlib import Path @@ -41,7 +41,7 @@ def alembic_config( def upgrade_spool_database( engine: Engine, *, - busy_timeout_seconds: float = DATABASE_TIMEOUT_SECONDS, + busy_timeout_seconds: Callable[[], float] = lambda: DATABASE_TIMEOUT_SECONDS, ) -> None: """Bring a spool to the packaged head, one migrating process at a time. @@ -49,8 +49,8 @@ def upgrade_spool_database( Neither does refusing one stamped with a revision missing from this checkout's history, such as a later migration from a newer checkout that shares the spool. Otherwise the whole upgrade, DDL and version stamp, runs - in one ``BEGIN IMMEDIATE`` transaction whose statements each wait at most - ``busy_timeout_seconds`` for another process's lock. Without that + in one ``BEGIN IMMEDIATE`` transaction whose statements each wait for + another process's lock as long as ``busy_timeout_seconds()`` allows. Without that transaction two services opening a new spool at once interleave their DDL, and one fails with "table already exists". A service that waits for the lock finds the spool at head when it gets it, and Alembic then changes @@ -69,7 +69,7 @@ def upgrade_spool_database( migration_engine = create_spool_engine( engine.url.database, immediate_transactions=True, - busy_timeout_seconds=lambda: busy_timeout_seconds, + busy_timeout_seconds=busy_timeout_seconds, ) try: with migration_engine.begin() as connection: diff --git a/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/runtime.py b/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/runtime.py index fa19729aeb..79197fa028 100644 --- a/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/runtime.py +++ b/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/runtime.py @@ -221,6 +221,9 @@ def _worker(self) -> None: self._attempt(self._append_unexpected_exit) self._stop.set() break + # A build that closes within its first tick never reached the prune + # above, and a host of such builds would never enforce retention. + self._attempt(self.spool.prune_if_due) self._drain() def _parent_alive(self) -> bool: diff --git a/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/spool.py b/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/spool.py index 1b12f9d712..3f31c427d6 100644 --- a/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/spool.py +++ b/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/spool.py @@ -11,7 +11,7 @@ from pathlib import Path from typing import Any -from sqlalchemy import Integer, delete, func, select +from sqlalchemy import Integer, delete, func, select, tuple_ from microcosm.build.telemetry_emitter_service.constants import ( BATCH_SIZE, @@ -47,7 +47,7 @@ def __init__( self, path: Path | str, *, - busy_timeout_seconds: float = DATABASE_TIMEOUT_SECONDS, + busy_deadline: float | None = None, ) -> None: self.path = Path(path) self.path.parent.mkdir(parents=True, exist_ok=True) @@ -55,18 +55,17 @@ def __init__( self.path.parent.chmod(0o700) except OSError: pass - # How long one statement waits for another process's lock. Startup - # lowers it so its own retry loop decides when to give up; it applies - # from each connection's next checkout. - self.busy_timeout_seconds = busy_timeout_seconds + # A time.monotonic() reading that startup sets: until it is cleared, + # no statement waits past it for another process's lock. + self.busy_deadline = busy_deadline self._engine = create_spool_engine( self.path, - busy_timeout_seconds=lambda: self.busy_timeout_seconds, + busy_timeout_seconds=self.busy_timeout_seconds, ) try: upgrade_spool_database( self._engine, - busy_timeout_seconds=busy_timeout_seconds, + busy_timeout_seconds=self.busy_timeout_seconds, ) except BaseException: self._engine.dispose() @@ -79,6 +78,18 @@ def __init__( # ping is answered. self._last_prune_at: float | None = None + def busy_timeout_seconds(self) -> float: + """How long a statement may wait for another process's lock. + + At most ``DATABASE_TIMEOUT_SECONDS``, and never past ``busy_deadline`` + while one is set. Applied each time a connection is checked out. + """ + + if self.busy_deadline is None: + return DATABASE_TIMEOUT_SECONDS + remaining = self.busy_deadline - time.monotonic() + return min(DATABASE_TIMEOUT_SECONDS, max(0.0, remaining)) + def register(self, registration: Mapping[str, Any]) -> None: """Create or refresh a producer registration.""" @@ -273,25 +284,32 @@ def prune(self, *, time_budget_seconds: float = math.inf) -> bool: .where(is_expired) .limit(PRUNE_BATCH_ROWS) ) - while self._delete_events(expired) == PRUNE_BATCH_ROWS: + in_batch = TelemetryEventRecord.event_id.in_(expired) + while self._delete(TelemetryEventRecord, in_batch) == PRUNE_BATCH_ROWS: if not self._pause_between_batches(deadline): return False oldest_first = self._oldest_events_over_size_limit() for start in range(0, len(oldest_first), PRUNE_BATCH_ROWS): if start and not self._pause_between_batches(deadline): return False - self._delete_events(oldest_first[start : start + PRUNE_BATCH_ROWS]) + batch = oldest_first[start : start + PRUNE_BATCH_ROWS] + self._delete(TelemetryEventRecord, TelemetryEventRecord.event_id.in_(batch)) is_expired_run = ( TelemetryRunRecord.updated_at < cutoff, ~TelemetryRunRecord.events.any(), ) if self._any(select(TelemetryRunRecord.run_id).where(*is_expired_run)): - with self._lock, self._session_factory.begin() as session: - session.execute( - delete(TelemetryRunRecord) - .where(*is_expired_run) - .execution_options(synchronize_session=False) - ) + expired_runs = ( + select(TelemetryRunRecord.run_id, TelemetryRunRecord.producer_id) + .where(*is_expired_run) + .limit(PRUNE_BATCH_ROWS) + ) + in_batch = tuple_( + TelemetryRunRecord.run_id, TelemetryRunRecord.producer_id + ).in_(expired_runs) + while self._delete(TelemetryRunRecord, in_batch) == PRUNE_BATCH_ROWS: + if not self._pause_between_batches(deadline): + return False return True def _any(self, statement) -> bool: @@ -309,19 +327,34 @@ def _pause_between_batches(deadline: float) -> bool: time.sleep(PRUNE_BATCH_PAUSE_SECONDS) return True - def _delete_events(self, event_ids) -> int: - """Delete the selected or listed events in one transaction.""" + def _delete(self, model, criterion) -> int: + """Delete one batch of matching rows in its own transaction.""" with self._lock, self._session_factory.begin() as session: return session.execute( - delete(TelemetryEventRecord) - .where(TelemetryEventRecord.event_id.in_(event_ids)) + delete(model) + .where(criterion) .execution_options(synchronize_session=False) ).rowcount + def _used_bytes(self) -> int: + """Return the bytes of the pages the spool file uses, without a scan.""" + + with self._lock, self._engine.connect() as connection: + page_size = connection.exec_driver_sql("PRAGMA page_size").scalar() + pages = connection.exec_driver_sql("PRAGMA page_count").scalar() + free_pages = connection.exec_driver_sql("PRAGMA freelist_count").scalar() + return (pages - free_pages) * page_size + def _oldest_events_over_size_limit(self) -> list[str]: """Return the oldest events whose removal brings storage under the cap.""" + # Every stored payload lies on a used page, so a file that uses no more + # than the cap cannot hold more than the cap. That spares the usual + # prune a read of every payload; other writers cannot commit while + # such a read runs. + if self._used_bytes() <= MAX_QUEUED_BYTES: + return [] stored = func.length(TelemetryEventRecord.payload, type_=Integer) with self._lock, self._session_factory() as session: excess = ( diff --git a/packages/microcosm-build/tests/engine_free/shared/test_telemetry_spool_contention.py b/packages/microcosm-build/tests/engine_free/shared/test_telemetry_spool_contention.py index 9ac70b47f9..eade7baec1 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_telemetry_spool_contention.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_telemetry_spool_contention.py @@ -14,6 +14,7 @@ import json import math import os +import shutil import signal import socket import sqlite3 @@ -25,18 +26,21 @@ from datetime import UTC, datetime, timedelta from pathlib import Path from types import SimpleNamespace +from unittest import mock import alembic.op import pytest from alembic import command from hypothesis import HealthCheck, example, given, settings from hypothesis import strategies as st +from sqlalchemy import event as sqlalchemy_event from sqlalchemy import inspect, select from sqlalchemy.exc import IntegrityError, OperationalError from microcosm.build.telemetry_emitter import LocalTelemetryEmitter, TelemetryRun from microcosm.build.telemetry_emitter_constants import TELEMETRY_SERVICE_MODULE from microcosm.build.telemetry_emitter_service import collector as collector_module +from microcosm.build.telemetry_emitter_service import database as database_module from microcosm.build.telemetry_emitter_service import main as main_module from microcosm.build.telemetry_emitter_service import runtime as runtime_module from microcosm.build.telemetry_emitter_service import spool as spool_module @@ -50,7 +54,6 @@ SPOOL_LOCKED_EXIT_STATUS, SPOOL_RETRY_INITIAL_SECONDS, SPOOL_RETRY_MAX_SECONDS, - STARTUP_BUSY_TIMEOUT_SECONDS, STARTUP_RETRY_LIMIT_SECONDS, ) from microcosm.build.telemetry_emitter_service.contention import ( @@ -132,9 +135,34 @@ def _write_lock(path: Path): connection.close() +_SOCKET_DIRECTORIES: list[Path] = [] + + def _short_socket_path() -> Path: - # macOS limits AF_UNIX paths to roughly 100 bytes. - return Path(tempfile.mkdtemp(prefix="microcosm-test-", dir="/tmp")) / "e.sock" + # macOS limits AF_UNIX paths to roughly 100 bytes, too few for tmp_path. + directory = Path(tempfile.mkdtemp(prefix="microcosm-test-", dir="/tmp")) + _SOCKET_DIRECTORIES.append(directory) + return directory / "e.sock" + + +@pytest.fixture(autouse=True) +def _remove_socket_directories(): + yield + while _SOCKET_DIRECTORIES: + shutil.rmtree(_SOCKET_DIRECTORIES.pop(), ignore_errors=True) + + +def _run_main(arguments: list[str], *, timeout: float) -> int: + """Run the service's main(), failing rather than hanging if it never returns.""" + + status: list[int] = [] + thread = threading.Thread( + target=lambda: status.append(main_module.main(arguments)), daemon=True + ) + thread.start() + thread.join(timeout) + assert not thread.is_alive(), f"main() still running after {timeout} s" + return status[0] def _ping(socket_path: Path) -> bool: @@ -192,8 +220,13 @@ def _lock_error(code: int = sqlite3.SQLITE_BUSY) -> OperationalError: def test_service_registers_once_a_held_spool_lock_is_released( tmp_path, monkeypatch, capsys, state ) -> None: - """In process: the lock outlasts many of startup's short SQLite waits.""" + """In process: the lock outlasts several SQLite waits, so startup retries. + + Each wait is capped at 0.1 s here, and the lock is held until two attempts + have failed on it. + """ + monkeypatch.setattr(spool_module, "DATABASE_TIMEOUT_SECONDS", 0.1) spool_path = tmp_path / "events.sqlite3" _prepare_spool(spool_path, state) socket_path = _short_socket_path() @@ -223,14 +256,22 @@ def wrapped(*args, **kwargs): target=lambda: status.append(main_module.main(arguments)), daemon=True ) + def failures() -> list[BaseException]: + return [error for _, _, error in outcomes if error is not None] + try: with _write_lock(spool_path): service.start() - time.sleep(1.5) + # Hold the lock until startup has failed on it twice, however long + # the host takes, so one SQLite wait cannot explain the success. + deadline = time.monotonic() + 60 + while len(failures()) < 2: + assert time.monotonic() < deadline, "startup never retried" + assert service.is_alive(), f"service exited with {status}" + time.sleep(0.02) assert not socket_path.exists() - assert service.is_alive() released_at = time.monotonic() - deadline = time.monotonic() + 20 + deadline = time.monotonic() + 60 while not _ping(socket_path): assert time.monotonic() < deadline, "service never became ready" assert service.is_alive(), f"service exited with {status}" @@ -240,14 +281,11 @@ def wrapped(*args, **kwargs): # Stop the service even when an assertion above failed. if socket_path.exists(): _close_service(socket_path) - service.join(timeout=10) + service.join(timeout=60) + assert not service.is_alive() assert status == [0] - failures = [error for _, _, error in outcomes if error is not None] - # Six startup waits fit in the hold, so a single SQLite wait cannot explain - # the success: startup retried. - assert len(failures) >= 2 - assert all(is_transient_spool_error(error) for error in failures) + assert all(is_transient_spool_error(error) for error in failures()) name, finished_at, error = outcomes[-1] assert (name, error) == ("register", None) assert finished_at >= released_at @@ -269,7 +307,7 @@ def test_service_exits_75_with_one_line_before_the_build_stops_waiting( with _write_lock(spool_path): started = time.monotonic() - status = main_module.main(arguments) + status = _run_main(arguments, timeout=30) waited = time.monotonic() - started assert status == SPOOL_LOCKED_EXIT_STATUS @@ -287,6 +325,40 @@ def test_service_exits_75_with_one_line_before_the_build_stops_waiting( assert "Traceback" not in error_output +def test_startup_waits_for_the_lock_as_long_as_the_build_waits(tmp_path) -> None: + """Never worse than one long SQLite wait. + + With a 3 s build budget, the client's default before #1166, a lock held for + 1.5 s is waited out, as the old service's single 5 s busy wait did. + """ + + spool_path = tmp_path / "events.sqlite3" + _prepare_spool(spool_path, "at_head") + socket_path = _short_socket_path() + arguments = _service_arguments( + socket_path, spool_path, ready_deadline=time.time() + 3.0 + ) + status: list[int] = [] + service = threading.Thread( + target=lambda: status.append(main_module.main(arguments)), daemon=True + ) + try: + with _write_lock(spool_path): + service.start() + time.sleep(1.5) + deadline = time.monotonic() + 10 + while not _ping(socket_path): + assert time.monotonic() < deadline, f"service never ready: {status}" + assert service.is_alive(), f"service exited with {status}" + time.sleep(0.02) + finally: + if socket_path.exists(): + _close_service(socket_path) + service.join(timeout=60) + assert not service.is_alive() + assert status == [0] + + def test_unexpected_service_failure_is_one_line_with_status_1(tmp_path, capsys) -> None: occupied = tmp_path / "not-a-directory" occupied.write_text("") @@ -296,7 +368,7 @@ def test_unexpected_service_failure_is_one_line_with_status_1(tmp_path, capsys) ready_deadline=time.time() + 60, ) - assert main_module.main(arguments) == 1 + assert _run_main(arguments, timeout=30) == 1 error_output = capsys.readouterr().err assert error_output.count("\n") == 1 @@ -342,19 +414,25 @@ def test_ready_deadline_rejects_nan_and_caps_infinity(capsys) -> None: ) -def test_startup_statements_wait_briefly_and_the_spool_then_waits_normally( +def test_startup_waits_end_by_the_deadline_and_the_spool_then_waits_normally( tmp_path, ) -> None: spool_path = tmp_path / "events.sqlite3" _prepare_spool(spool_path, "at_head") - spool = EventSpool(spool_path, busy_timeout_seconds=STARTUP_BUSY_TIMEOUT_SECONDS) + spool = EventSpool(spool_path, busy_deadline=time.monotonic() + 0.3) try: with _write_lock(spool_path): started = time.monotonic() with pytest.raises(OperationalError): spool.register(_registration()) waited = time.monotonic() - started - assert waited < 4 * STARTUP_BUSY_TIMEOUT_SECONDS + assert waited < 1.0 + # A far deadline still waits no longer than a normal statement, and a + # past one does not wait at all. + spool.busy_deadline = time.monotonic() + 100 + assert spool.busy_timeout_seconds() == DATABASE_TIMEOUT_SECONDS + spool.busy_deadline = time.monotonic() - 1 + assert spool.busy_timeout_seconds() == 0 finally: _close(spool) @@ -362,7 +440,7 @@ def test_startup_statements_wait_briefly_and_the_spool_then_waits_normally( spool_path, _registration(), deadline=time.monotonic() + 10 ) try: - assert registered.busy_timeout_seconds == DATABASE_TIMEOUT_SECONDS + assert registered.busy_deadline is None with registered._engine.connect() as connection: milliseconds = connection.exec_driver_sql("PRAGMA busy_timeout").scalar() assert milliseconds == DATABASE_TIMEOUT_SECONDS * 1000 @@ -370,6 +448,29 @@ def test_startup_statements_wait_briefly_and_the_spool_then_waits_normally( _close(registered) +def test_without_a_deadline_startup_makes_one_normal_attempt(tmp_path) -> None: + """A build that passes no deadline gets the old behaviour: one 5 s wait.""" + + spool_path = tmp_path / "events.sqlite3" + _prepare_spool(spool_path, "at_head") + opened: list[EventSpool] = [] + original_init = EventSpool.__init__ + + def recording_init(self, *args, **kwargs): + original_init(self, *args, **kwargs) + opened.append(self) + + with mock.patch.object(EventSpool, "__init__", recording_init): + spool = main_module.open_registered_spool( + spool_path, _registration(), deadline=None + ) + try: + assert [item.busy_deadline for item in opened] == [None] + assert spool.busy_timeout_seconds() == DATABASE_TIMEOUT_SECONDS + finally: + _close(spool) + + def test_build_keeps_telemetry_when_the_spool_is_locked_past_a_busy_wait( tmp_path, capfd, real_local_telemetry ) -> None: @@ -411,12 +512,18 @@ def start() -> None: emitter = started["emitter"] assert isinstance(emitter, LocalTelemetryEmitter) - assert emitter.available - assert started["returned_at"] > released_at - emitter.stage("compile", message="Compiling.") - emitter.complete() - assert emitter._process is not None - assert emitter._process.wait(timeout=30) == 0 + try: + assert emitter.available + assert started["returned_at"] > released_at + emitter.stage("compile", message="Compiling.") + emitter.complete() + assert emitter._process is not None + assert emitter._process.wait(timeout=30) == 0 + finally: + # A failed assertion must not leave the service running. + if emitter._process is not None and emitter._process.poll() is None: + emitter._process.terminate() + emitter._process.wait(timeout=10) spool = EventSpool(spool_path) try: @@ -550,23 +657,27 @@ def popen(command, **kwargs): now_unix=st.floats(-1e6, 1e6), now_monotonic=st.floats(0, 1e6), ) -def test_startup_deadline_is_the_ready_deadline_less_the_margin_on_this_clock( +def test_startup_gives_up_before_the_build_and_within_the_limit( ready_deadline, now_unix, now_monotonic ) -> None: deadline = main_module.startup_deadline( ready_deadline, now_unix=now_unix, now_monotonic=now_monotonic ) - remaining = min(ready_deadline - now_unix, STARTUP_RETRY_LIMIT_SECONDS) - assert deadline - now_monotonic == pytest.approx( - remaining - READY_DEADLINE_MARGIN_SECONDS, abs=1e-6 + waits = deadline - now_monotonic + budget = ready_deadline - now_unix + # It stops at least the margin before the build gives up, and within the + # limit however far away the build's deadline is. + assert waits <= budget - READY_DEADLINE_MARGIN_SECONDS + 1e-6 + assert waits <= STARTUP_RETRY_LIMIT_SECONDS - READY_DEADLINE_MARGIN_SECONDS + 1e-6 + # And it uses all of that time: nothing is left on the table. + assert waits >= min(budget, STARTUP_RETRY_LIMIT_SECONDS) - ( + READY_DEADLINE_MARGIN_SECONDS + 1e-6 ) - # However far away the build's deadline is, startup stops within the limit. - assert deadline - now_monotonic <= STARTUP_RETRY_LIMIT_SECONDS assert ( main_module.startup_deadline( None, now_unix=now_unix, now_monotonic=now_monotonic ) - == -math.inf + is None ) @@ -610,7 +721,7 @@ def fail(*args, **kwargs): assert tables == [] -def test_an_opener_waits_for_a_migration_in_progress(tmp_path) -> None: +def test_an_opener_waits_for_a_migration_in_progress(tmp_path, monkeypatch) -> None: """A second opener waits, then finds the spool at head and changes nothing. Before migrations took the write lock first, the waiting opener had @@ -630,10 +741,23 @@ def open_spool() -> None: failures.append(error) second = threading.Thread(target=open_spool, daemon=True) + waiting = threading.Event() + begin_immediate = database_module._begin_immediate + + def signalling_begin_immediate(connection) -> None: + waiting.set() + begin_immediate(connection) + try: with migrator.begin() as connection: + # Only the opener's migration engine is created after this point. + monkeypatch.setattr( + database_module, "_begin_immediate", signalling_begin_immediate + ) second.start() - time.sleep(0.5) + # The opener has asked for the write lock this migration holds. + assert waiting.wait(timeout=30), "the opener never began its migration" + time.sleep(0.2) assert second.is_alive() SpoolModel.metadata.create_all(connection) connection.exec_driver_sql( @@ -749,6 +873,8 @@ def test_appends_never_prune(tmp_path, monkeypatch) -> None: st.sampled_from(["finished", "unfinished", "failed"]), max_size=30 ), ) +# A call exactly one interval after a finished prune is due. +@example(gaps=[PRUNE_INTERVAL_SECONDS], results=["finished", "finished"]) @settings(suppress_health_check=[HealthCheck.function_scoped_fixture]) def test_prune_runs_once_per_interval_until_a_backlog_is_drained( monkeypatch, gaps, results @@ -798,30 +924,18 @@ def prune(*, time_budget_seconds): # --- Lock-error classification ------------------------------------------------ -# Extended result codes keep the primary code in the low byte, so draw busy and -# locked primaries with every extension as well as arbitrary codes. -_RESULT_CODES = st.one_of( - st.integers(0, 2**16), - st.builds( - lambda primary, extension: primary | extension << 8, - st.sampled_from([sqlite3.SQLITE_BUSY, sqlite3.SQLITE_LOCKED]), - st.integers(0, 255), - ), -) - +def test_only_busy_and_locked_result_codes_are_transient() -> None: + """Every primary code with several extensions, raw and wrapped.""" -@given(code=_RESULT_CODES) -@example(code=261) # SQLITE_BUSY_RECOVERY -@example(code=262) # SQLITE_LOCKED_SHAREDCACHE -@example(code=517) # SQLITE_BUSY_SNAPSHOT -@example(code=773) # SQLITE_BUSY_TIMEOUT -@example(code=1555) # SQLITE_CONSTRAINT_PRIMARYKEY -def test_only_busy_and_locked_result_codes_are_transient(code) -> None: - error = sqlite3.OperationalError("message") - error.sqlite_errorcode = code - transient = code & 0xFF in {sqlite3.SQLITE_BUSY, sqlite3.SQLITE_LOCKED} - assert is_transient_spool_error(error) is transient - assert is_transient_spool_error(_lock_error(code)) is transient + busy_or_locked = {5, 6} # SQLITE_BUSY and SQLITE_LOCKED, from the SQLite docs + for primary in range(256): + for extension in (0, 1, 2, 3, 255): + code = primary | extension << 8 + error = sqlite3.OperationalError("message") + error.sqlite_errorcode = code + expected = primary in busy_or_locked + assert is_transient_spool_error(error) is expected, code + assert is_transient_spool_error(_lock_error(code)) is expected, code def test_real_sqlite_errors_are_classified_by_what_retrying_can_fix(tmp_path) -> None: @@ -871,17 +985,27 @@ def test_real_sqlite_errors_are_classified_by_what_retrying_can_fix(tmp_path) -> ), window=st.one_of(st.just(-math.inf), st.floats(-1, 3), st.floats(-5, 60)), jitter_fraction=st.floats(0, 1), + # How far each sleep overruns, as sleeps do on a loaded host. + oversleeps=st.lists(st.one_of(st.just(0.0), st.floats(0, 3)), max_size=8), ) -# The first retry's full wait (0.05 s) would cross a deadline 0.04 s away. -@example(outcomes=_LOCKED_THEN_OK, window=0.04, jitter_fraction=1.0) +# The first retry's full wait (0.05 s) would cross a deadline 0.04 s away, or +# start exactly on one 0.05 s away. +@example(outcomes=_LOCKED_THEN_OK, window=0.04, jitter_fraction=1.0, oversleeps=[]) +@example(outcomes=_LOCKED_THEN_OK, window=0.05, jitter_fraction=1.0, oversleeps=[]) # Six retries fit, so the delays must double up to the cap. -@example(outcomes=_LOCKED_THEN_OK, window=10.0, jitter_fraction=0.5) -def test_retry_respects_its_deadline_and_backoff(outcomes, window, jitter_fraction): +@example(outcomes=_LOCKED_THEN_OK, window=10.0, jitter_fraction=0.5, oversleeps=[]) +# The first sleep fits a deadline 1 s away but wakes 3 s late, past it. +@example(outcomes=_LOCKED_THEN_OK, window=1.0, jitter_fraction=0.5, oversleeps=[3.0]) +def test_retry_respects_its_deadline_and_backoff( + outcomes, window, jitter_fraction, oversleeps +): clock = SimpleNamespace(now=100.0) deadline = clock.now + window attempt_starts: list[float] = [] jitters: list[tuple[float, float, float, float]] = [] # (at, low, high, wait) waits: list[tuple[float, float]] = [] # (started, seconds) + wakes: list[float] = [] + late = iter(oversleeps) def operation(): index = len(attempt_starts) @@ -901,7 +1025,8 @@ def jitter(low, high): def sleep(seconds): waits.append((clock.now, seconds)) - clock.now += seconds + clock.now += seconds + next(late, 0.0) + wakes.append(clock.now) try: result = retry_spool_contention( @@ -924,14 +1049,16 @@ def sleep(seconds): assert kinds[-1] == outcome if outcome == "ok": assert result == len(attempt_starts) - 1 - # The first attempt always runs; every later one starts before the deadline - # and no wait reaches it. + # The first attempt always runs; every later one starts before the deadline, + # even after a sleep that overran, and no wait is asked to reach it. assert all(start < deadline for start in attempt_starts[1:]) assert all(started + seconds < deadline for started, seconds in waits) + overran = bool(wakes) and wakes[-1] >= deadline # Each lock error draws one wait in [delay / 2, delay]; delays double to a - # cap; every wait drawn is slept except one that would reach the deadline. + # cap; every wait drawn is slept except one that would reach the deadline, + # and every sleep is followed by an attempt unless it woke too late. assert len(jitters) == kinds.count("locked") - assert len(waits) == len(attempt_starts) - 1 + assert len(waits) == len(attempt_starts) - 1 + overran for index, (_, low, high, wait) in enumerate(jitters): assert high == min( SPOOL_RETRY_MAX_SECONDS, SPOOL_RETRY_INITIAL_SECONDS * 2**index @@ -942,10 +1069,12 @@ def sleep(seconds): wait for _, _, _, wait in jitters[: len(waits)] ] # A lock error escapes only when its retry could not start before the - # deadline. + # deadline: the wait would reach it, or the sleep woke past it. if outcome == "locked": at, _, _, wait = jitters[-1] - assert at + wait >= deadline + assert at + wait >= deadline or overran + else: + assert not overran def test_retry_with_no_deadline_makes_one_attempt() -> None: @@ -1373,10 +1502,27 @@ def _stamp(age_days: float, index: int) -> str: max_size=18, ), run_ages=st.lists(st.floats(0, 2 * RETENTION_DAYS), min_size=3, max_size=3), - cap_fraction=st.floats(0, 1.2), + # None leaves the real 100 MiB cap, so only age removes rows. + cap_fraction=st.one_of(st.none(), st.floats(0, 1.2)), batch_rows=st.integers(1, 4), step_budget=st.sampled_from([0.0, math.inf]), ) +# More expired rows than one batch, with no size pressure to hide a leftover. +@example( + events=[(0, 10.0, 5)] * 6, + run_ages=[1.0, 1.0, 1.0], + cap_fraction=None, + batch_rows=2, + step_budget=math.inf, +) +# Removing the oldest live event (20 characters of 50) fits the cap exactly. +@example( + events=[(0, 1.0, 10), (0, 1.0, 20)], + run_ages=[1.0, 1.0, 1.0], + cap_fraction=0.6, + batch_rows=2, + step_budget=math.inf, +) def test_batched_prune_matches_the_single_pass_retention_rule( tmp_path_factory, events, run_ages, cap_fraction, batch_rows, step_budget ) -> None: @@ -1388,8 +1534,6 @@ def test_batched_prune_matches_the_single_pass_retention_rule( have no events left. """ - from unittest import mock - spool = EventSpool(tmp_path_factory.mktemp("prune") / "events.sqlite3") keys = [(f"run-{index}", f"producer-{index}") for index in range(3)] rows: list[dict[str, object]] = [] @@ -1432,16 +1576,20 @@ def test_batched_prune_matches_the_single_pass_retention_rule( ) cutoff = (_STAMP_ORIGIN - timedelta(days=RETENTION_DAYS)).isoformat() live = [row for row in rows if row["created_at"] >= cutoff] - cap = int(cap_fraction * sum(row["length"] for row in live)) + cap = ( + spool_module.MAX_QUEUED_BYTES + if cap_fraction is None + else int(cap_fraction * sum(row["length"] for row in live)) + ) batches: list[int] = [] - delete_events = spool._delete_events + delete = spool._delete - def recording_delete(event_ids) -> int: - deleted = delete_events(event_ids) + def recording_delete(model, criterion) -> int: + deleted = delete(model, criterion) batches.append(deleted) return deleted - spool._delete_events = recording_delete + spool._delete = recording_delete frozen_now = mock.Mock(wraps=datetime) frozen_now.now.return_value = _STAMP_ORIGIN with ( @@ -1566,3 +1714,109 @@ def test_a_prune_out_of_time_stops_between_batches_and_resumes( # One batch per call: five rows go two, two, then one. assert remaining == [3, 1, 0] + + +def test_a_service_that_closes_within_its_first_tick_still_prunes(tmp_path) -> None: + """Short builds must not leave retention unenforced on a host forever.""" + + spool_path = tmp_path / "events.sqlite3" + _prepare_spool(spool_path, "at_head_with_expired_rows") + spool = EventSpool(spool_path) + registration = _registration() + spool.register(registration) + socket_path = _short_socket_path() + service = EmitterService( + socket_path=socket_path, + registration=registration, + spool=spool, + delivery=SimpleNamespace(flush_once=lambda: False), + sampler=SimpleNamespace(sample=dict, parent_alive=lambda: True), + heartbeat_seconds=60, + drain_seconds=0, + ) + serving = threading.Thread(target=service.run, daemon=True) + try: + assert spool.has_pending() + serving.start() + deadline = time.monotonic() + 30 + while not _ping(socket_path): + assert time.monotonic() < deadline, "service never became ready" + time.sleep(0.01) + # Closed at once, well inside the worker's first one-second tick. + _close_service(socket_path) + serving.join(timeout=30) + assert not serving.is_alive() + assert not spool.has_pending() + finally: + service._stop.set() + serving.join(timeout=5) + _close(spool) + + +def test_expired_runs_are_deleted_in_batches(tmp_path, monkeypatch) -> None: + spool = EventSpool(tmp_path / "events.sqlite3") + deleted: list[tuple[str, int]] = [] + pauses: list[float] = [] + try: + for index in range(5): + spool.register({"run_id": f"run-{index}", "producer_id": "producer-a"}) + expired = (datetime.now(UTC) - timedelta(days=RETENTION_DAYS + 1)).isoformat() + with spool._session_factory.begin() as session: + session.query(TelemetryRunRecord).update({"updated_at": expired}) + delete = spool._delete + + def recording_delete(model, criterion) -> int: + count = delete(model, criterion) + deleted.append((model.__tablename__, count)) + return count + + spool._delete = recording_delete + monkeypatch.setattr(spool_module, "PRUNE_BATCH_ROWS", 2) + monkeypatch.setattr( + spool_module, + "time", + SimpleNamespace(monotonic=time.monotonic, sleep=pauses.append), + ) + assert spool.prune() is True + with spool._session_factory() as session: + remaining = session.scalars(select(TelemetryRunRecord.run_id)).all() + finally: + _close(spool) + + # Five runs in batches of two, with a pause after each full batch. + assert deleted == [ + ("telemetry_runs", 2), + ("telemetry_runs", 2), + ("telemetry_runs", 1), + ] + assert len(pauses) == 2 + assert remaining == [] + + +def test_a_spool_under_the_size_cap_is_not_scanned(tmp_path, monkeypatch) -> None: + """The size check reads the file's page counts, not every payload.""" + + spool = EventSpool(tmp_path / "events.sqlite3") + statements: list[str] = [] + try: + registration = _registration() + spool.register(registration) + for _ in range(3): + spool.append(registration, _event()) + sqlalchemy_event.listen( + spool._engine, + "before_cursor_execute", + lambda conn, cursor, statement, *rest: statements.append(statement), + ) + assert spool.prune() is True + assert not [text for text in statements if "sum(" in text.lower()] + assert len(spool.batch("run-a", "producer-a")) == 3 + + # Over the cap, the payloads are summed and the oldest events go. + statements.clear() + monkeypatch.setattr(spool_module, "MAX_QUEUED_BYTES", 1) + assert spool.prune() is True + assert [text for text in statements if "sum(" in text.lower()] + assert spool.batch("run-a", "producer-a") == [] + finally: + _close(spool) From 7ab903ef5a0782020799d666dcf307d51eed2614 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 9 Oct 2026 20:11:11 -0400 Subject: [PATCH 12/13] Bound the moving-deadline property so a stale deadline fails instead of hanging A helper that read its deadline once kept retrying a spool that stayed locked, and the test looped on its fake clock until an outer timeout. Co-Authored-By: Claude Opus 5.5 --- .../tests/engine_free/shared/test_telemetry_event_queue.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/packages/microcosm-build/tests/engine_free/shared/test_telemetry_event_queue.py b/packages/microcosm-build/tests/engine_free/shared/test_telemetry_event_queue.py index 761cf60998..2cb30450b2 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_telemetry_event_queue.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_telemetry_event_queue.py @@ -619,6 +619,10 @@ def deadline() -> float: def operation(): attempts.append(clock.now) + # Once the deadline is finite, at most window / (shortest wait) more + # attempts fit: 5 s / 0.025 s. A helper that kept an old reading of + # infinity would retry a spool that stays locked forever. + assert len(attempts) <= finite_from + 250, "the deadline was not re-read" kind = outcomes[min(len(attempts) - 1, len(outcomes) - 1)] clock.now += 0.01 if kind == "locked": From 1e321e84d7f8a1ecedc9960097e28be33fe06df8 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 9 Oct 2026 21:01:50 -0400 Subject: [PATCH 13/13] Give shutdown one window, and keep every append inside it Review of #1174 found four defects. run() could return while the writer was still retrying, with events unwritten and no warning: a drain flag recorded that a drain had begun, not that it had finished, so a worker held up in a delivery step let run() time out its join and leave. A hung worker instead earned a second full window. One window now opens at the first stop signal, the writer gets its deadline at once, and run() waits for the writer and reports what it left before returning. An append could touch the database after its cutoff: the cutoff was checked once before the retry helper, so a deadline arriving during a backoff, a backoff that overran, or a wait for this process's own spool lock could each carry a write past it. Every attempt now checks once it holds the spool, and waits a bounded time for it. One busy timeout did not bound an append: it waited for the write lock and again at COMMIT, 3.4 s against a 2 s timeout when measured. An append's statements now share one lock-wait budget. A long stage id, which the client does not limit, made the killed-build record too large for the queue's reserved room. The service's copy of the stage id is cut to the client's text limit. Co-Authored-By: Claude Opus 5.5 --- changelog.d/telemetry-event-queue.fixed.md | 2 +- .../telemetry_emitter_service/constants.py | 9 +- .../telemetry_emitter_service/contention.py | 14 + .../telemetry_emitter_service/event_queue.py | 96 +++-- .../telemetry_emitter_service/runtime.py | 101 ++++-- .../build/telemetry_emitter_service/spool.py | 148 +++++--- .../shared/test_telemetry_event_queue.py | 336 ++++++++++++++++-- .../shared/test_telemetry_spool_contention.py | 6 +- 8 files changed, 579 insertions(+), 133 deletions(-) diff --git a/changelog.d/telemetry-event-queue.fixed.md b/changelog.d/telemetry-event-queue.fixed.md index db0041bec6..2ca4591c6d 100644 --- a/changelog.d/telemetry-event-queue.fixed.md +++ b/changelog.d/telemetry-event-queue.fixed.md @@ -1,3 +1,3 @@ A build's telemetry sends no longer wait for the shared local spool. Before, the emitter service appended each event to the SQLite spool before replying. When another build held the spool's write lock, a send waited up to SQLite's 5 s busy wait while the build's client gave up after 0.2 s and printed "could not queue an update", and an append still locked after 5 s dropped the event. Now the service replies once the event is queued in memory, and one writer thread appends queued events in arrival order, up to 100 per transaction, retrying lock errors for as long as they last. Heartbeats and the record of a killed build go through the same queue, so the killed-build record follows every event the build sent. -The queue holds at most 10,000 events and 16 MiB of their encoded JSON, with 16 events and 256 KiB kept for run events (started, completed, failed, blocked), so a queue that progress updates have filled still records how the build ended. A full queue refuses updates and prints one warning line. When the build closes or dies, the service spends up to its 15 s drain writing the queue, starts no append in the last 0.25 s of it, and reports in one line how many events it could not write. +The queue holds at most 10,000 events and 16 MiB of their encoded JSON, with 16 events and 256 KiB kept for run events (started, completed, failed, blocked), so a queue that progress updates have filled still records how the build ended. A full queue refuses updates and prints one warning line. One 15 s shutdown window opens when the build closes or dies. The writer drains the queue inside it and starts no append in its last 0.25 s. The service does not exit while the writer may still be writing, and it reports in one line how many events it could not write. diff --git a/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/constants.py b/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/constants.py index 29218a0c90..bbfe3f5c48 100644 --- a/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/constants.py +++ b/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/constants.py @@ -90,9 +90,11 @@ QUEUE_RESERVED_EVENTS: Final = 16 QUEUE_RESERVED_BYTES: Final = 256 * 1024 # The writer appends up to WRITER_BATCH_EVENTS queued events per transaction. -# Each of its statements waits at most WRITER_BUSY_TIMEOUT_SECONDS for another -# process's lock, so its retry loop decides how long to keep trying and it -# holds the spool's in-process lock only briefly. +# One append's statements together wait at most WRITER_BUSY_TIMEOUT_SECONDS +# for other processes' locks, and the append waits no longer than that for +# another thread of this process to release the spool. So the writer's retry +# loop decides how long to keep trying, and during a shutdown drain no append +# touches the database after the deadline less that wait. WRITER_BATCH_EVENTS: Final = 100 WRITER_BUSY_TIMEOUT_SECONDS: Final = 0.25 @@ -109,6 +111,7 @@ EVENT_FIELDS_ERROR: Final = "event must have an event_type and a status" QUEUE_FULL_ERROR: Final = "local telemetry queue is full" QUEUE_CLOSED_ERROR: Final = "local telemetry queue is closed" +SPOOL_BUSY_ERROR: Final = "the spool is busy in this process" UNSUPPORTED_ACTION_ERROR: Final = "unsupported local telemetry action" LOCAL_MESSAGE_TOO_LARGE_ERROR: Final = "local telemetry message exceeds 1 MiB" FAILURE_CLASS_UNEXPECTED_PROCESS_EXIT: Final = "unexpected_process_exit" diff --git a/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/contention.py b/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/contention.py index e6441feb98..bfae3b5ba8 100644 --- a/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/contention.py +++ b/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/contention.py @@ -10,6 +10,7 @@ from sqlalchemy.exc import DBAPIError from microcosm.build.telemetry_emitter_service.constants import ( + SPOOL_BUSY_ERROR, SPOOL_RETRY_INITIAL_SECONDS, SPOOL_RETRY_MAX_SECONDS, ) @@ -19,6 +20,19 @@ _TRANSIENT_RESULT_CODES = frozenset({sqlite3.SQLITE_BUSY, sqlite3.SQLITE_LOCKED}) +class SpoolBusyError(sqlite3.OperationalError): + """Another thread of this process kept the spool past the caller's wait. + + It carries SQLite's busy code, so it is retried like a lock another + process holds. + """ + + def __init__(self) -> None: + super().__init__(SPOOL_BUSY_ERROR) + self.sqlite_errorcode = sqlite3.SQLITE_BUSY + self.sqlite_errorname = "SQLITE_BUSY" + + def is_transient_spool_error(error: BaseException) -> bool: """Return whether an error is SQLite lock contention that a retry can clear. diff --git a/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/event_queue.py b/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/event_queue.py index 341b9e9d3c..82bdf7d7e0 100644 --- a/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/event_queue.py +++ b/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/event_queue.py @@ -202,11 +202,15 @@ class SpoolWriter: While the service serves, a lock error is retried for as long as it lasts: an event leaves the queue only once the spool has stored it, or once the - spool has refused it for another reason. ``finish`` sets a drain deadline - and closes the queue; the writer then starts no append after - ``deadline - WRITER_BUSY_TIMEOUT_SECONDS``, and since each statement waits - at most that long for a lock, its last append ends by the deadline plus the - time its own inserts take. + spool has refused it for another reason. + + ``begin_drain`` closes the queue and sets a deadline. From then on no + append touches the database after ``deadline - + WRITER_BUSY_TIMEOUT_SECONDS``: every attempt checks that once it holds the + spool, having waited no longer than that for another thread to release it. + An append's statements together wait at most + ``WRITER_BUSY_TIMEOUT_SECONDS`` for other processes' locks, so the last + append ends by the deadline plus the time its own inserts take. """ def __init__( @@ -233,7 +237,7 @@ def __init__( self.dropped = 0 def start(self) -> None: - """Write queued events on a daemon thread until ``finish``.""" + """Write queued events on a daemon thread until the drain ends.""" self._thread = threading.Thread( target=self._run, name="telemetry-spool-writer", daemon=True @@ -243,27 +247,42 @@ def start(self) -> None: def is_alive(self) -> bool: return self._thread is not None and self._thread.is_alive() - def finish(self, deadline: float) -> int: - """Close the queue and write what it holds, until ``deadline``. + def begin_drain(self, deadline: float) -> None: + """Close the queue and give the writer until ``deadline`` to empty it. - ``deadline`` is a reading of this writer's clock. Returns how many - events were left unwritten. A writer whose thread was never started, - such as one a test drives step by step, writes on the calling thread. + ``deadline`` is a reading of this writer's clock. Returns at once; a + writer that is retrying a lock error picks the deadline up at its next + attempt. """ self._drain_deadline = deadline self.queue.close() + + def wait_drained(self) -> int: + """Wait for the drain to end and return how many events are unwritten. + + The writer's last append ends by the deadline plus its inserts; this + waits one statement wait past the deadline for it. A writer whose + thread was never started, such as one a test drives step by step, + writes on the calling thread. + """ + if self._thread is None: if len(self.queue): self.write_queued() else: - # The thread's last append ends by the deadline plus its inserts; - # this waits for that and gives up a statement wait later. self._thread.join( - timeout=max(0.0, deadline - self._clock()) + WRITER_BUSY_TIMEOUT_SECONDS + timeout=max(0.0, self._drain_deadline - self._clock()) + + WRITER_BUSY_TIMEOUT_SECONDS ) return len(self.queue) + def finish(self, deadline: float) -> int: + """Drain until ``deadline`` and return how many events are unwritten.""" + + self.begin_drain(deadline) + return self.wait_drained() + def _run(self) -> None: try: while self.queue.wait_for_events(): @@ -294,7 +313,10 @@ def write_queued(self) -> bool: or is_transient_spool_error(error) ): raise - self.stopped_by = error + # A deadline that passed between attempts ends the drain with + # no error of its own; keep the lock error that explains it. + if self.stopped_by is None or is_transient_spool_error(error): + self.stopped_by = error return False return True @@ -321,20 +343,44 @@ def _write_oldest(self, events: list[QueuedEvent]) -> None: self.queue.remove(len(events)) def _append(self, events: list[QueuedEvent]) -> None: - if self._clock() >= self._last_attempt_start(): - raise DrainDeadlineError() pairs = [event.decode() for event in events] + + def attempt() -> None: + # Checked at the start of every attempt, not once before the first: + # a backoff can outlast its plan, and the deadline can arrive + # while the writer sleeps. + left = self._last_start() - self._clock() + if left <= 0: + raise DrainDeadlineError() + try: + self.spool.append_many( + self.registration, + pairs, + busy_timeout_seconds=WRITER_BUSY_TIMEOUT_SECONDS, + lock_timeout_seconds=min(left, WRITER_BUSY_TIMEOUT_SECONDS), + before_write=self._check_may_write, + ) + except Exception as error: + if is_transient_spool_error(error): + self.stopped_by = error + raise + retry_spool_contention( - lambda: self.spool.append_many( - self.registration, - pairs, - busy_timeout_seconds=WRITER_BUSY_TIMEOUT_SECONDS, - ), - deadline=self._last_attempt_start, + attempt, + deadline=self._last_start, clock=self._clock, sleep=self._sleep, ) + # Written: an earlier lock error no longer explains anything. + self.stopped_by = None + + def _check_may_write(self) -> None: + # Runs once the spool is held, so time spent waiting for another + # thread to release it cannot carry database work past the deadline. + if self._clock() >= self._last_start(): + raise DrainDeadlineError() - def _last_attempt_start(self) -> float: - # Infinite until finish sets a drain deadline. + def _last_start(self) -> float: + # The latest an append may begin its database work. Infinite until + # begin_drain sets a deadline. return self._drain_deadline - WRITER_BUSY_TIMEOUT_SECONDS diff --git a/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/runtime.py b/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/runtime.py index 67b18a234d..deb7e61884 100644 --- a/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/runtime.py +++ b/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/runtime.py @@ -60,6 +60,7 @@ LOCAL_MESSAGE_DELIMITER, LOCAL_SOCKET_READ_BYTES, MAX_LOCAL_MESSAGE_BYTES, + MAX_TELEMETRY_TEXT_CHARS, STAGE_BLOCKED, STAGE_COMPLETE, STAGE_CREATED, @@ -153,8 +154,13 @@ def __init__( self._reported_error_types: set[str] = set() self._dropped_error_types: set[str] = set() self._warned_queue_full = False - self._drain_lock = threading.Lock() - self._drain_started = False + # One shutdown window, opened by the first stop signal and shared by + # the writer's drain, delivery and every thread that waits for them. + self._shutdown_lock = threading.Lock() + self._shutdown_deadline: float | None = None + self._delivery_drain_started = False + self._writing_lock = threading.Lock() + self._writing_finished = False self.event_queue = event_queue if event_queue is not None else EventQueue() # The writer reads this module's clock when it runs, so a test that # replaces the module's time drives the writer too. @@ -170,9 +176,13 @@ def __init__( def run(self) -> None: """Serve local messages until the client closes or exits. - The spool must already hold this producer's registration. Queued - events are written to the spool before this returns, within - ``drain_seconds`` of the stop. + The spool must already hold this producer's registration. One + shutdown window of ``drain_seconds`` opens at the first stop signal: + the build's close, the build's death, or a thread of this service + dying. This returns once the writer has emptied the queue or the + window has closed on it, at most ``WORKER_INTERVAL_SECONDS`` and one + statement wait after the window ends. Events still queued then are + reported in one line. """ self.socket_path.parent.mkdir(parents=True, exist_ok=True) @@ -187,11 +197,20 @@ def run(self) -> None: worker = threading.Thread(target=self._worker, daemon=True) worker.start() self._serve(server, worker) + deadline = self._begin_shutdown() self._stop.set() - worker.join(timeout=self.drain_seconds + WORKER_INTERVAL_SECONDS) - # The worker drains on its way out. One that died or hung - # before then leaves the queue to this thread. - self._drain() + # The worker waits for the writer, then delivers, inside the + # window that just opened. + worker.join( + timeout=max(0.0, deadline - time.monotonic()) + + WORKER_INTERVAL_SECONDS + ) + # A worker held up in a step has not waited for the writer, so + # this thread does, and reports what the writer left. + self._finish_writing() + if not worker.is_alive(): + # A worker that died before draining leaves delivery here. + self._drain() finally: self.socket_path.unlink(missing_ok=True) try: @@ -261,12 +280,17 @@ def _handle(self, message: Mapping[str, Any]) -> None: STAGE_COMPLETE, STAGE_FAILED, }: - self._last_stage = stage_id + # The client does not limit a stage id's length. This copy + # goes into heartbeats and the killed-build record, which must + # fit the queue's reserved room, so it is cut to the client's + # own text limit. + self._last_stage = stage_id[:MAX_TELEMETRY_TEXT_CHARS] self._enqueue(event) elif action == ACTION_CLOSE: # Nothing, not even a heartbeat, is queued behind the build's last - # event. - self.event_queue.close() + # event, and the writer's drain starts now, not when the worker + # next looks. + self._begin_shutdown() self._stop.set() elif action == ACTION_PING: return @@ -311,9 +335,9 @@ def _worker(self) -> None: # cleanly; only a build that never closed died unexpectedly. if not self._stop.is_set() and not self._parent_alive(): self._attempt(self._queue_unexpected_exit) - # Closed even when the record could not be queued: nothing - # arrives from a dead build. - self.event_queue.close() + # The queue closes even when the record could not be queued: + # nothing arrives from a dead build. + self._begin_shutdown() self._stop.set() break self._drain() @@ -380,19 +404,34 @@ def _report_dropped(self, error: Exception) -> None: ) ) - def _drain(self) -> None: - """Write the queue to the spool, then deliver, within ``drain_seconds``. + def _begin_shutdown(self) -> float: + """Open the shutdown window at the first stop signal; return its deadline. + + The queue closes and the writer gets the deadline at once, so the + queue drains from this moment whatever the worker is doing. Every + later call returns the same deadline. Stopping the serving loops is + the caller's business. + """ + + with self._shutdown_lock: + if self._shutdown_deadline is None: + self._shutdown_deadline = time.monotonic() + self.drain_seconds + self.writer.begin_drain(self._shutdown_deadline) + return self._shutdown_deadline + + def _finish_writing(self) -> None: + """Wait for the writer's drain and report what it left, exactly once. - Runs once: from the worker on its way out, or from ``run`` when the - worker stopped without draining. + Any thread may call this. A second caller waits for the first, so no + caller returns while the writer may still be writing in the window. """ - with self._drain_lock: - if self._drain_started: + self._begin_shutdown() + with self._writing_lock: + if self._writing_finished: return - self._drain_started = True - deadline = time.monotonic() + self.drain_seconds - unwritten = self.writer.finish(deadline) + unwritten = self.writer.wait_drained() + self._writing_finished = True if unwritten: stopped_by = self.writer.stopped_by write_warning( @@ -407,6 +446,20 @@ def _drain(self) -> None: ), ) ) + + def _drain(self) -> None: + """Write the queue to the spool, then deliver, in the shutdown window. + + The worker runs this on its way out, and ``run`` runs it when the + worker died first. Delivery is drained once. + """ + + deadline = self._begin_shutdown() + self._finish_writing() + with self._shutdown_lock: + if self._delivery_drain_started: + return + self._delivery_drain_started = True while time.monotonic() < deadline: progressed = False try: diff --git a/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/spool.py b/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/spool.py index 2ca81efdf9..9f56bae48c 100644 --- a/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/spool.py +++ b/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/spool.py @@ -6,12 +6,14 @@ import threading import time import uuid -from collections.abc import Mapping, Sequence +from collections.abc import Callable, Iterator, Mapping, Sequence +from contextlib import contextmanager from datetime import UTC, datetime, timedelta from pathlib import Path from typing import Any from sqlalchemy import Integer, delete, func, select +from sqlalchemy.orm import Session from microcosm.build.telemetry_emitter_service.constants import ( BATCH_SIZE, @@ -25,6 +27,7 @@ UPLOAD_STATE_LOCAL_ONLY, UPLOAD_STATE_PENDING, ) +from microcosm.build.telemetry_emitter_service.contention import SpoolBusyError from microcosm.build.telemetry_emitter_service.database import ( create_spool_engine, create_spool_session_factory, @@ -40,6 +43,29 @@ from microcosm.build.telemetry_protocol import TELEMETRY_SCHEMA_VERSION +class _LockWaitBudget: + """Caps the time one transaction's statements together wait for locks. + + SQLite's busy timeout applies to each lock a statement needs, so a + transaction that waits for the write lock and then again at COMMIT can + wait twice as long as the timeout. ``apply``, called before each statement + that can wait, sets the timeout to what is left of ``seconds``. The next + checkout of the connection restores the spool's own timeout. + """ + + def __init__(self, session: Session, seconds: float | None) -> None: + self._session = session + self._deadline = None if seconds is None else time.monotonic() + seconds + + def apply(self) -> None: + if self._deadline is None: + return + milliseconds = max(0, round((self._deadline - time.monotonic()) * 1000)) + self._session.connection().exec_driver_sql( + f"PRAGMA busy_timeout = {milliseconds}" + ) + + class EventSpool: """Small SQLite queue shared by successive emitter service processes.""" @@ -59,12 +85,9 @@ def __init__( # lowers it so its own retry loop decides when to give up; it applies # from each connection's next checkout. self.busy_timeout_seconds = busy_timeout_seconds - # One call's own wait, set by append_many while it holds _lock, which - # every connection checkout here happens under. - self._busy_timeout_override: float | None = None self._engine = create_spool_engine( self.path, - busy_timeout_seconds=self._statement_busy_timeout, + busy_timeout_seconds=lambda: self.busy_timeout_seconds, ) try: upgrade_spool_database( @@ -126,64 +149,91 @@ def append_many( events: Sequence[tuple[Mapping[str, Any], Mapping[str, Any] | None]], *, busy_timeout_seconds: float | None = None, + lock_timeout_seconds: float | None = None, + before_write: Callable[[], None] | None = None, ) -> list[dict[str, Any]]: """Append ``(event, resources)`` pairs in one transaction, in order. The events take consecutive producer sequences in the order given, or, when the transaction fails, none of them is stored and the producer's next sequence is unchanged, so a failed call can simply be repeated. - ``busy_timeout_seconds``, when given, caps how long this call's - statements wait for another process's lock instead of - ``busy_timeout_seconds`` on the spool. + + Three controls bound how long the call can take, for a caller with a + deadline: + + - ``lock_timeout_seconds`` caps the wait for another thread of this + process to finish its own spool operation. Past it the call raises + ``SpoolBusyError``, which counts as lock contention. + - ``before_write`` runs once this call holds the spool, before it + touches the database. Raising from it abandons the call. + - ``busy_timeout_seconds`` caps the time all of this call's statements + together wait for another process's lock: reading the producer's + row, taking the write lock, and committing past other processes' + readers. Without it each statement waits ``busy_timeout_seconds`` on + the spool. """ run_id = str(registration["run_id"]) producer_id = str(registration["producer_id"]) payloads: list[dict[str, Any]] = [] - with self._lock: - self._busy_timeout_override = busy_timeout_seconds - try: - with self._session_factory.begin() as session: - run = session.get(TelemetryRunRecord, (run_id, producer_id)) - if run is None: - raise KeyError(run_id) - for event, resources in events: - sequence = run.next_sequence - event_id = uuid.uuid4().hex - payload = { - "schema_version": TELEMETRY_SCHEMA_VERSION, - "event_id": event_id, - "run_id": run_id, - "producer_id": producer_id, - "sequence": sequence, - "timestamp": event.get("timestamp") or utc_now(), - "event_type": event["event_type"], - "stage_id": event.get("stage_id"), - "status": event["status"], - "message": event.get("message"), - "details": event.get("details") or {}, - "resources": resources, - } - session.add( - TelemetryEventRecord( - event_id=event_id, - run_id=run_id, - producer_id=producer_id, - sequence=sequence, - payload=payload, - created_at=utc_now(), - ) + with self._held(lock_timeout_seconds): + if before_write is not None: + before_write() + with self._session_factory.begin() as session: + lock_waits = _LockWaitBudget(session, busy_timeout_seconds) + lock_waits.apply() + run = session.get(TelemetryRunRecord, (run_id, producer_id)) + if run is None: + raise KeyError(run_id) + for event, resources in events: + sequence = run.next_sequence + event_id = uuid.uuid4().hex + payload = { + "schema_version": TELEMETRY_SCHEMA_VERSION, + "event_id": event_id, + "run_id": run_id, + "producer_id": producer_id, + "sequence": sequence, + "timestamp": event.get("timestamp") or utc_now(), + "event_type": event["event_type"], + "stage_id": event.get("stage_id"), + "status": event["status"], + "message": event.get("message"), + "details": event.get("details") or {}, + "resources": resources, + } + session.add( + TelemetryEventRecord( + event_id=event_id, + run_id=run_id, + producer_id=producer_id, + sequence=sequence, + payload=payload, + created_at=utc_now(), ) - run.next_sequence = sequence + 1 - payloads.append(payload) - run.updated_at = utc_now() - finally: - self._busy_timeout_override = None + ) + run.next_sequence = sequence + 1 + payloads.append(payload) + run.updated_at = utc_now() + # The inserts take the write lock; the commit that follows + # waits for other processes' readers. + lock_waits.apply() + session.flush() + lock_waits.apply() return payloads - def _statement_busy_timeout(self) -> float: - override = self._busy_timeout_override - return self.busy_timeout_seconds if override is None else override + @contextmanager + def _held(self, timeout_seconds: float | None) -> Iterator[None]: + """Hold this process's spool lock, waiting at most ``timeout_seconds``.""" + + if timeout_seconds is None: + self._lock.acquire() + elif not self._lock.acquire(timeout=max(0.0, timeout_seconds)): + raise SpoolBusyError() + try: + yield + finally: + self._lock.release() def pending_runs(self) -> list[dict[str, Any]]: """Return registrations that have events eligible for delivery.""" diff --git a/packages/microcosm-build/tests/engine_free/shared/test_telemetry_event_queue.py b/packages/microcosm-build/tests/engine_free/shared/test_telemetry_event_queue.py index 2cb30450b2..a6e41d645c 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_telemetry_event_queue.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_telemetry_event_queue.py @@ -41,6 +41,7 @@ WRITER_BUSY_TIMEOUT_SECONDS, ) from microcosm.build.telemetry_emitter_service.contention import ( + SpoolBusyError, is_transient_spool_error, retry_spool_contention, ) @@ -60,6 +61,7 @@ LOCAL_ACKNOWLEDGEMENT_OK, MAX_TELEMETRY_DETAILS_BYTES, MAX_TELEMETRY_MESSAGE_CHARS, + MAX_TELEMETRY_TEXT_CHARS, UNEXPECTED_PROCESS_EXIT_MESSAGE, ) from microcosm.build.telemetry_sanitization import sanitize_details, sanitize_text @@ -135,10 +137,11 @@ def _stored_events(path: Path, run_id: str, producer_id: str) -> list[dict]: class _Clock: - """A fake monotonic clock; sleeping advances it.""" + """A fake monotonic clock; sleeping advances it, by ``oversleep`` too long.""" - def __init__(self, now: float = 1_000.0) -> None: + def __init__(self, now: float = 1_000.0, oversleep: float = 0.0) -> None: self.now = now + self.oversleep = oversleep self.sleeps: list[tuple[float, float]] = [] def monotonic(self) -> float: @@ -146,7 +149,7 @@ def monotonic(self) -> float: def sleep(self, seconds: float) -> None: self.sleeps.append((self.now, seconds)) - self.now += seconds + self.now += seconds + self.oversleep def _service(spool, **overrides) -> EmitterService: @@ -269,12 +272,53 @@ def test_the_reserve_holds_every_run_event_of_a_build_at_the_clients_limits() -> } largest = QueuedEvent.encode(run_event, sample).size assert largest > MAX_TELEMETRY_DETAILS_BYTES + # The service writes the killed-build record itself, around the longest + # stage id it keeps. + killed = runtime_module._unexpected_exit_event( + "\U0001f600" * MAX_TELEMETRY_TEXT_CHARS + ) + largest_killed = QueuedEvent.encode(killed, sample).size assert QUEUE_RESERVED_EVENTS >= 3 - assert QUEUE_RESERVED_BYTES >= 3 * largest + assert QUEUE_RESERVED_BYTES >= 2 * largest + largest_killed assert QUEUE_RESERVED_EVENTS < QUEUE_MAX_EVENTS assert QUEUE_RESERVED_BYTES < QUEUE_MAX_BYTES +def test_a_long_stage_id_cannot_crowd_out_the_killed_build_record() -> None: + """The client does not limit a stage id; the service's copy of it is cut. + + Progress events with a 300,000-character stage id, each under the 1 MiB + message limit, fill the room outside the reserve. The killed-build record + names the last stage, and must still fit the reserve. + """ + + service = _service(SimpleNamespace()) + stage = "s" * 300_000 + accepted = 0 + error_output = io.StringIO() + with contextlib.redirect_stderr(error_output): + while _send(service, _event(stage + str(accepted))): + accepted += 1 + assert accepted > 0 + ordinary_room = ( + service.event_queue.max_bytes + - service.event_queue.reserved_bytes + - service.event_queue.queued_bytes + ) + assert ordinary_room < len(stage) + + assert service._attempt(service._queue_unexpected_exit) + + assert service.event_queue.closed + record, _ = service.event_queue.oldest(accepted + 1)[-1].decode() + assert record["message"] == UNEXPECTED_PROCESS_EXIT_MESSAGE + assert record["details"]["failed_during"] == stage[:MAX_TELEMETRY_TEXT_CHARS] + # A heartbeat carries the same bounded copy. + assert len(runtime_module._heartbeat_event(service._last_stage)["stage_id"]) == ( + MAX_TELEMETRY_TEXT_CHARS + ) + + def test_queued_bytes_count_the_encoded_event_and_its_sample() -> None: sampler_reading = {"cpu_user_seconds": 1.5, "rss_bytes": 10} service = _service( @@ -369,7 +413,17 @@ def __init__(self, lock_errors) -> None: self.locked_attempts = 0 self.busy_timeouts: set[float | None] = set() - def append_many(self, registration, pairs, *, busy_timeout_seconds=None): + def append_many( + self, + registration, + pairs, + *, + busy_timeout_seconds=None, + lock_timeout_seconds=None, + before_write=None, + ): + if before_write is not None: + before_write() self.busy_timeouts.add(busy_timeout_seconds) if next(self._lock_errors, False): self.locked_attempts += 1 @@ -469,8 +523,10 @@ def test_every_acknowledged_event_is_stored_once_in_acknowledgement_order( class _LockedUntil: """A spool another process keeps locked until ``lock_until`` on a fake clock. - An attempt waits in SQLite's busy handler for up to its statement wait and - succeeds as soon as the lock is released within it. + Like the real spool, an attempt runs ``before_write`` once it holds the + spool, then its statements together wait up to ``busy_timeout_seconds`` + for the lock and succeed as soon as it is released within that. Only + attempts that reach the database are recorded. """ def __init__(self, clock: _Clock, lock_until: float, insert_seconds: float): @@ -478,15 +534,28 @@ def __init__(self, clock: _Clock, lock_until: float, insert_seconds: float): self.lock_until = lock_until self.insert_seconds = insert_seconds self.attempt_starts: list[float] = [] + self.attempt_ends: list[float] = [] self.stored: list[str] = [] - def append_many(self, registration, pairs, *, busy_timeout_seconds=None): + def append_many( + self, + registration, + pairs, + *, + busy_timeout_seconds=None, + lock_timeout_seconds=None, + before_write=None, + ): + if before_write is not None: + before_write() started = self.clock.now self.attempt_starts.append(started) if self.lock_until > started + busy_timeout_seconds: self.clock.now = started + busy_timeout_seconds + self.attempt_ends.append(self.clock.now) raise _lock_error() self.clock.now = max(started, self.lock_until) + self.insert_seconds + self.attempt_ends.append(self.clock.now) self.stored.extend(event["stage_id"] for event, _ in pairs) def has_deliverable(self) -> bool: @@ -503,13 +572,22 @@ def has_deliverable(self) -> bool: locked_for=st.one_of(st.just(0.0), st.floats(0, 25)), drain_seconds=st.one_of(st.just(0.0), st.floats(0, 20)), insert_seconds=st.floats(0, 0.05), + oversleep=st.one_of(st.just(0.0), st.floats(0, 1.0)), +) +@example( + events=450, locked_for=3.0, drain_seconds=15.0, insert_seconds=0.05, oversleep=0.0 +) +@example( + events=10, locked_for=30.0, drain_seconds=15.0, insert_seconds=0.0, oversleep=0.0 +) +# A backoff that overruns its plan by more than the margin before the deadline. +@example( + events=10, locked_for=30.0, drain_seconds=1.0, insert_seconds=0.0, oversleep=0.9 ) -@example(events=450, locked_for=3.0, drain_seconds=15.0, insert_seconds=0.05) -@example(events=10, locked_for=30.0, drain_seconds=15.0, insert_seconds=0.0) def test_the_drain_writes_in_order_until_its_deadline_and_reports_the_rest( - monkeypatch, events, locked_for, drain_seconds, insert_seconds + monkeypatch, events, locked_for, drain_seconds, insert_seconds, oversleep ) -> None: - clock = _Clock() + clock = _Clock(oversleep=oversleep) monkeypatch.setattr(runtime_module, "time", clock) spool = _LockedUntil(clock, clock.now + locked_for, insert_seconds) service = _service(spool, drain_seconds=drain_seconds) @@ -525,10 +603,11 @@ def test_the_drain_writes_in_order_until_its_deadline_and_reports_the_rest( service._drain() last_start = deadline - WRITER_BUSY_TIMEOUT_SECONDS - # No append starts within one statement wait of the deadline, so the last - # ends by the deadline plus its own inserts. + # No append touches the database within one lock-wait budget of the + # deadline, however late a backoff wakes, so every append ends by the + # deadline plus its own inserts. assert all(start < last_start for start in spool.attempt_starts) - assert clock.now <= deadline + insert_seconds + 1e-9 + assert all(end <= deadline + insert_seconds + 1e-9 for end in spool.attempt_ends) # What was written is the oldest events, in order. assert spool.stored == sent[: len(spool.stored)] unwritten = events - len(spool.stored) @@ -539,18 +618,26 @@ def test_the_drain_writes_in_order_until_its_deadline_and_reports_the_rest( else: assert lines == [] # Contention that clears with room to spare loses nothing: an attempt in - # its busy wait succeeds at the release, a sleeping one within a backoff. + # its busy wait succeeds at the release, a sleeping one within a backoff + # and its overrun. batches = math.ceil(events / WRITER_BATCH_EVENTS) release = started + locked_for - if release + SPOOL_RETRY_MAX_SECONDS + batches * insert_seconds < last_start: + wakes_by = release + SPOOL_RETRY_MAX_SECONDS + oversleep + if wakes_by + batches * insert_seconds < last_start: assert unwritten == 0 # A lock held through the whole drain writes nothing and loses all. if release >= deadline: assert spool.stored == [] -def test_a_writer_stops_at_the_drain_deadline_even_mid_retry() -> None: - """The deadline is read after every lock error, so finish reaches a retry.""" +@pytest.mark.parametrize("window", [2.0, 0.1, 0.0]) +def test_a_deadline_that_arrives_mid_retry_stops_every_later_attempt(window) -> None: + """The cutoff is checked in every attempt, not once before the first. + + ``window`` is how far ahead the deadline is when it arrives, during a + backoff sleep of a writer retrying with no deadline. At 0.1 s and 0 s the + cutoff has already passed, so no further attempt may touch the database. + """ clock = _Clock() queue = EventQueue() @@ -564,23 +651,26 @@ def test_a_writer_stops_at_the_drain_deadline_even_mid_retry() -> None: clock=clock.monotonic, sleep=clock.sleep, ) - finish_at = clock.now + 30.0 - real_sleep = clock.sleep + arrives_at = clock.now + 30.0 + advance = clock.sleep def sleep(seconds): - real_sleep(seconds) - # The service stops while the writer is retrying an unbounded wait. - if clock.now >= finish_at and writer._drain_deadline == math.inf: - writer._drain_deadline = clock.now + 2.0 - assert clock.now < finish_at + 60, "the writer ignored the drain deadline" + advance(seconds) + # The service stops while the writer sleeps between attempts. + if clock.now >= arrives_at and writer._drain_deadline == math.inf: + writer.begin_drain(clock.now + window) + assert clock.now < arrives_at + 60, "the writer ignored the drain deadline" writer._sleep = sleep assert not writer.write_queued() assert is_transient_spool_error(writer.stopped_by) assert len(queue) == 1 - assert ( - spool.attempt_starts[-1] < writer._drain_deadline - WRITER_BUSY_TIMEOUT_SECONDS - ) + cutoff = writer._drain_deadline - WRITER_BUSY_TIMEOUT_SECONDS + assert spool.attempt_starts, "the writer never reached the spool" + assert all(start < cutoff for start in spool.attempt_starts[-3:]) + if window < WRITER_BUSY_TIMEOUT_SECONDS: + # The cutoff had passed when the deadline arrived: nothing ran after. + assert spool.attempt_starts[-1] < arrives_at # --- The retry helper with a deadline that moves ------------------------------ @@ -788,6 +878,133 @@ def test_a_commit_a_reader_blocks_stores_nothing_so_a_retry_cannot_duplicate( spool._engine.dispose() +def test_an_appends_lock_waits_share_one_budget(tmp_path) -> None: + """Waiting for the write lock and again at COMMIT must not add up. + + Another connection holds the write lock for 1.5 s while a reader holds a + SHARED lock throughout. With a 2 s budget the append gets the write lock + after 1.5 s and then has 0.5 s, not another 2 s, to wait at COMMIT. + """ + + spool_path = tmp_path / "events.sqlite3" + spool = EventSpool(spool_path) + registration = _registration() + spool.register(registration) + reader = sqlite3.connect(spool_path, isolation_level=None) + holding = threading.Event() + + def hold_write_lock() -> None: + with _write_lock(spool_path): + holding.set() + time.sleep(1.5) + + holder = threading.Thread(target=hold_write_lock, daemon=True) + try: + reader.execute("BEGIN") + reader.execute("SELECT count(*) FROM telemetry_events").fetchone() + holder.start() + assert holding.wait(10) + started = time.monotonic() + with pytest.raises(OperationalError) as locked: + spool.append_many( + registration, [(_event("a"), None)], busy_timeout_seconds=2.0 + ) + waited = time.monotonic() - started + assert is_transient_spool_error(locked.value) + # Past the other writer's 1.5 s, so the wait at COMMIT happened, and + # well short of the 3.5 s that a full second wait would take. + assert 1.5 <= waited < 2.9 + reader.execute("COMMIT") + assert _stored_stages(spool_path) == [] + finally: + holder.join(timeout=10) + reader.close() + spool._engine.dispose() + + +def test_this_processes_own_spool_lock_cannot_carry_a_write_past_the_cutoff( + tmp_path, +) -> None: + """The worker can hold the spool's in-process lock for a whole busy wait. + + Another thread holds it for 3 s; the drain deadline is 1 s away. The + writer must give up at the cutoff, not write when the lock comes free. + """ + + spool_path = tmp_path / "events.sqlite3" + spool = EventSpool(spool_path) + registration = _registration() + spool.register(registration) + queue = EventQueue() + for stage in ("a", "b"): + queue.offer(QueuedEvent.encode(_event(stage), None)) + writer = SpoolWriter(spool, registration, queue, report_dropped=pytest.fail) + holding = threading.Event() + + def hold_spool_lock() -> None: + with spool._lock: + holding.set() + time.sleep(3.0) + + holder = threading.Thread(target=hold_spool_lock, daemon=True) + try: + holder.start() + assert holding.wait(10) + started = time.monotonic() + writer.begin_drain(started + 1.0) + assert not writer.write_queued() + gave_up_after = time.monotonic() - started + assert isinstance(writer.stopped_by, SpoolBusyError) + assert is_transient_spool_error(writer.stopped_by) + # At the cutoff, well before the other thread lets go. + assert gave_up_after < 2.5 + holder.join(timeout=10) + assert len(queue) == 2 + assert _stored_stages(spool_path) == [] + # Asked directly, the spool refuses within the caller's wait. + with spool._lock: + refused = [] + + def ask() -> None: + try: + spool.append_many( + registration, [(_event("c"), None)], lock_timeout_seconds=0.1 + ) + except Exception as error: + refused.append(error) + + asking = threading.Thread(target=ask, daemon=True) + asking.start() + asking.join(timeout=10) + assert [type(error) for error in refused] == [SpoolBusyError] + finally: + holder.join(timeout=10) + spool._engine.dispose() + + +def test_before_write_runs_with_the_spool_held_and_can_abandon_the_append( + tmp_path, +) -> None: + spool = EventSpool(tmp_path / "events.sqlite3") + registration = _registration() + spool.register(registration) + held: list[bool] = [] + + def refuse() -> None: + # RLock._is_owned is true only for the thread that holds it. + held.append(spool._lock._is_owned()) + raise RuntimeError("too late") + + try: + with pytest.raises(RuntimeError, match="too late"): + spool.append_many(registration, [(_event("a"), None)], before_write=refuse) + assert held == [True] + assert _stored_stages(spool.path) == [] + assert not spool._lock._is_owned() + finally: + spool._engine.dispose() + + # --- Real lock contention ----------------------------------------------------- @@ -961,6 +1178,67 @@ def test_a_lock_held_through_the_drain_loses_only_the_queue_and_says_so( assert "(database is locked)" in lines[0] +class _HeldDelivery: + """A delivery step that blocks, as a collector request or a busy wait does.""" + + def __init__(self) -> None: + self.entered = threading.Event() + self.release = threading.Event() + + def flush_once(self) -> bool: + self.entered.set() + self.release.wait(60) + return False + + +@pytest.mark.parametrize("worker", ["returns after the window", "never returns"]) +def test_run_does_not_return_while_the_writer_may_still_write( + tmp_path, capsys, worker +) -> None: + """One shutdown window, opened at the close, whatever the worker is doing. + + The worker is inside a delivery step when the build closes, and the spool + stays locked. The writer must still get its deadline at the close, and + ``run`` must not return before the writer has stopped and the events it + left are reported: not earlier with the writer still retrying, and not a + second window later. + """ + + spool_path = tmp_path / "events.sqlite3" + drain_seconds = 2.0 + delivery = _HeldDelivery() + service, serving, client = _start_service( + spool_path, drain_seconds=drain_seconds, delivery=delivery + ) + try: + assert delivery.entered.wait(15), "the worker never reached delivery" + with _write_lock(spool_path): + for index in range(3): + client.stage(f"stage-{index}") + client.close() + closed_at = time.monotonic() + if worker == "returns after the window": + threading.Timer(drain_seconds + 0.5, delivery.release.set).start() + serving.join(timeout=30) + returned_after = time.monotonic() - closed_at + # Read at the moment run returned, with the spool still locked. + writer_alive = service.writer.is_alive() + lines = capsys.readouterr().err.splitlines() + finally: + delivery.release.set() + _stop(service, serving, client) + + assert not serving.is_alive() + assert not writer_alive + assert len(lines) == 1 + assert "could not write 3 queued update(s)" in lines[0] + assert "(database is locked)" in lines[0] + # The window, the worker's interval, a statement wait and the accept wait; + # a second window would add drain_seconds. + assert returned_after < drain_seconds + 1.0 + 0.25 + 0.5 + 0.75 + assert _stored_stages(spool_path) == [] + + def test_a_killed_build_is_recorded_after_every_event_it_sent(tmp_path) -> None: spool_path = tmp_path / "events.sqlite3" service, serving, client = _start_service(spool_path, drain_seconds=10) diff --git a/packages/microcosm-build/tests/engine_free/shared/test_telemetry_spool_contention.py b/packages/microcosm-build/tests/engine_free/shared/test_telemetry_spool_contention.py index 812d9641f8..14a1a3cf88 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_telemetry_spool_contention.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_telemetry_spool_contention.py @@ -1041,8 +1041,9 @@ def set(self): def is_set(self): return self.flag - def append_many(registration, pairs, *, busy_timeout_seconds=None): + def append_many(registration, pairs, *, before_write=None, **waits): # The worker only queues events; the drain's writer stores them. + before_write() appended.extend( event["event_type"] + ":" + event["status"] for event, _ in pairs ) @@ -1226,7 +1227,8 @@ def test_unexpected_exit_waits_out_lock_contention(monkeypatch) -> None: attempts: list[float] = [] stored: list[str] = [] - def append_many(registration, pairs, *, busy_timeout_seconds=None): + def append_many(registration, pairs, *, before_write=None, **waits): + before_write() attempts.append(clock.now) clock.now += 0.2 if len(attempts) < 4: