From 961565317e5f5f04e13ab93b7794b8b545cbadf0 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Fri, 9 Oct 2026 00:22:24 +0400 Subject: [PATCH 1/4] Delegate local delivery to provider packages without activating the migration --- README.md | 8 +- changelog.d/1152-publication-types.changed.md | 1 + changelog.d/1152.changed.md | 1 + conftest.py | 34 - docs/agent-guide.md | 12 +- docs/orrery-publication.md | 39 +- packages/microcosm-build/pyproject.toml | 12 +- .../microcosm/build/graph_publication_cli.py | 58 -- .../build/graph_publication_types.py | 60 -- .../src/microcosm/build/telemetry_emitter.py | 564 +--------- .../build/telemetry_emitter_constants.py | 38 - .../telemetry_emitter_service/__init__.py | 17 - .../telemetry_emitter_service/__main__.py | 5 - .../telemetry_emitter_service/alembic/env.py | 28 - .../alembic/script.py.mako | 31 - .../versions/v20261007_01_initial_spool.py | 140 --- .../v20261008_02_graph_publication.py | 27 - .../build/telemetry_emitter_service/auth.py | 180 ---- .../telemetry_emitter_service/collector.py | 191 ---- .../telemetry_emitter_service/constants.py | 75 -- .../telemetry_emitter_service/database.py | 34 - .../graph_publication.py | 414 -------- .../build/telemetry_emitter_service/main.py | 66 -- .../telemetry_emitter_service/migrations.py | 65 -- .../build/telemetry_emitter_service/models.py | 152 --- .../telemetry_emitter_service/resources.py | 137 --- .../telemetry_emitter_service/runtime.py | 264 ----- .../build/telemetry_emitter_service/spool.py | 258 ----- .../telemetry_emitter_service/timestamps.py | 11 - .../src/microcosm/build/telemetry_protocol.py | 70 -- .../microcosm/build/telemetry_sanitization.py | 94 -- .../build/uk_runtime/full_build_cli.py | 4 +- .../build/uk_runtime/orrery_publication.py | 22 +- .../build/us_runtime/worker_identity.py | 2 +- .../shared/test_graph_publication.py | 358 ------- .../shared/test_provider_adapter.py | 157 +++ .../shared/test_publication_workers.py | 142 --- .../shared/test_telemetry_emitter.py | 970 ------------------ .../uk/test_uk_orrery_publication.py | 18 + test_support/microcosm_build/telemetry.py | 14 +- uv.lock | 53 +- 41 files changed, 342 insertions(+), 4484 deletions(-) create mode 100644 changelog.d/1152-publication-types.changed.md create mode 100644 changelog.d/1152.changed.md delete mode 100644 packages/microcosm-build/src/microcosm/build/graph_publication_cli.py delete mode 100644 packages/microcosm-build/src/microcosm/build/graph_publication_types.py delete mode 100644 packages/microcosm-build/src/microcosm/build/telemetry_emitter_constants.py delete mode 100644 packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/__init__.py delete mode 100644 packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/__main__.py delete mode 100644 packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/alembic/env.py delete mode 100644 packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/alembic/script.py.mako delete mode 100644 packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/alembic/versions/v20261007_01_initial_spool.py delete mode 100644 packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/alembic/versions/v20261008_02_graph_publication.py delete mode 100644 packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/auth.py delete mode 100644 packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/collector.py delete mode 100644 packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/constants.py delete mode 100644 packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/database.py delete mode 100644 packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/graph_publication.py delete mode 100644 packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/main.py delete mode 100644 packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/migrations.py delete mode 100644 packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/models.py delete mode 100644 packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/resources.py delete mode 100644 packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/runtime.py delete mode 100644 packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/spool.py delete mode 100644 packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/timestamps.py delete mode 100644 packages/microcosm-build/src/microcosm/build/telemetry_protocol.py delete mode 100644 packages/microcosm-build/src/microcosm/build/telemetry_sanitization.py delete mode 100644 packages/microcosm-build/tests/engine_free/shared/test_graph_publication.py create mode 100644 packages/microcosm-build/tests/engine_free/shared/test_provider_adapter.py delete mode 100644 packages/microcosm-build/tests/engine_free/shared/test_publication_workers.py delete mode 100644 packages/microcosm-build/tests/engine_free/shared/test_telemetry_emitter.py diff --git a/README.md b/README.md index 820921a1f3..37a4bae9b3 100644 --- a/README.md +++ b/README.md @@ -51,8 +51,12 @@ immutable publication before Hugging Face artifact staging, and preserves durable retry jobs and receipt snapshots. See [graph publication](docs/orrery-publication.md) for configuration and retry. -Supported US and UK build commands always start the local telemetry emitter -service. It reports live progress to the hosted collector when the operator's +Supported US and UK build commands start one local provider service with separate +telemetry and graph-publication workers. Microcosm supplies build metadata and +graph files; provider packages own authentication, durable queues and delivery. +This draft migration requires published provider packages before merge; see the +[migration prerequisites](docs/orrery-publication.md#deferred-migration-prerequisites). +The telemetry worker reports live progress to the hosted collector when the operator's existing Hugging Face login is accepted; otherwise it retains the events locally and the build continues. This live event path is independent of the staging run files described below. diff --git a/changelog.d/1152-publication-types.changed.md b/changelog.d/1152-publication-types.changed.md new file mode 100644 index 0000000000..3449f91a60 --- /dev/null +++ b/changelog.d/1152-publication-types.changed.md @@ -0,0 +1 @@ +Use the emitter package's explicit publication contracts and pin the deferred adapter to the typed implementation. diff --git a/changelog.d/1152.changed.md b/changelog.d/1152.changed.md new file mode 100644 index 0000000000..08a1bd1d38 --- /dev/null +++ b/changelog.d/1152.changed.md @@ -0,0 +1 @@ +Delegate local telemetry and durable Orrery graph delivery to microcosm-local-provider while retaining UK graph generation and artifact staging in Microcosm. diff --git a/conftest.py b/conftest.py index 0e09de4aad..307a9bcad4 100644 --- a/conftest.py +++ b/conftest.py @@ -10,7 +10,6 @@ import importlib.util import sys from pathlib import Path -from urllib.parse import urlsplit import pytest @@ -76,7 +75,6 @@ def fake_telemetry_emitters(monkeypatch, tmp_path): from huggingface_hub import constants as hf_constants from microcosm.build import telemetry_emitter - from microcosm.build.telemetry_emitter_service import collector from test_support.microcosm_build.telemetry import FakeTelemetryEmitter for name in ("HF_TOKEN", "HUGGING_FACE_HUB_TOKEN", "HUGGINGFACE_TOKEN"): @@ -112,38 +110,6 @@ def start(_cls, **kwargs): ): monkeypatch.setattr(module, "_ACTIVE_EMITTER", None) - real_post = collector._http_post - - def local_post(url, *args, **kwargs): - parsed = urlsplit(url) - collector._development_collector_url(f"{parsed.scheme}://{parsed.netloc}") - return real_post(url, *args, **kwargs) - - monkeypatch.setattr(collector, "_http_post", local_post) yield emitters for emitter in emitters: emitter.close() - - -@pytest.fixture -def real_local_telemetry(monkeypatch, fake_telemetry_emitters): - """Permit actual startup only with an explicit loopback collector.""" - from microcosm.build import telemetry_emitter - from microcosm.build.telemetry_emitter_service.collector import ( - _development_collector_url, - ) - - def start(cls, **kwargs): - _development_collector_url(kwargs.get("development_collector_url") or "") - return _REAL_TELEMETRY_START(cls, **kwargs) - - monkeypatch.setattr( - telemetry_emitter.LocalTelemetryEmitter, "start", classmethod(start) - ) - - -# Save the underlying method before per-test patches so imported aliases and -# class references all share the same default fake and explicit local opt-in. -from microcosm.build.telemetry_emitter import LocalTelemetryEmitter # noqa: E402 - -_REAL_TELEMETRY_START = LocalTelemetryEmitter.start.__func__ diff --git a/docs/agent-guide.md b/docs/agent-guide.md index d121fe37f0..e0aa80a485 100644 --- a/docs/agent-guide.md +++ b/docs/agent-guide.md @@ -15,7 +15,7 @@ the PEP 420 namespace `microcosm.`: `frame`, `fit`, `calibrate`, `build`, ## Commands UK build completion exports and stages Orrery graph evidence. Publication uses -an independent worker inside the existing local service and a durable queue; +an independent worker inside the external local provider and a durable queue; see [configuration and retry](orrery-publication.md). No new producer secret or environment variable is required. Keep graph export/publication tests independent of the hosted runs app and collector; use synthetic files and fake HTTP. @@ -97,11 +97,11 @@ uses the same pinned parser for groups, statuses, activities and artifacts. These CI tests perform no browser rendering. Workspace tests use an in-memory telemetry emitter by default. The shared -fixture isolates Hugging Face credentials and cache paths and blocks collector -HTTP requests outside loopback. Tests of actual emitter startup must request -`real_local_telemetry` and supply an explicit loopback development collector; -that opt-in retains credential isolation. Other build tests can inspect -`fake_telemetry_emitters` without creating sockets or service processes. +fixture isolates Hugging Face credentials and cache paths. Adapter tests replace +provider startup and publication APIs with fakes. Provider process, persistence +and HTTP-delivery tests belong in microcosm-emitter; do not copy them +into Microcosm. Build tests inspect `fake_telemetry_emitters` without creating +sockets or service processes. New commits to a PR cancel older unfinished CI runs for that same PR. Each main-push run has a unique concurrency group, so all main-push runs diff --git a/docs/orrery-publication.md b/docs/orrery-publication.md index c7e6c65d96..4bf8ce7e38 100644 --- a/docs/orrery-publication.md +++ b/docs/orrery-publication.md @@ -6,10 +6,11 @@ public-safe file inventory in the attempt directory, attempt publication before Hugging Face artifact staging, and register graph/evidence files and an `orrery.publication.json` receipt snapshot in the artifact manifest. -The existing local service has two independent workers: live telemetry and -completed graph publication. Slow uploads cannot delay socket acknowledgement -or telemetry delivery. This change does not introduce another generic runtime, -rename the telemetry package, or add compatibility wrappers. +Microcosm's metadata adapter starts one provider service with two independent +workers: live telemetry and completed graph publication. The provider packages +own sockets, authentication, persistence, retries and resource sampling. +Microcosm retains graph generation, public-file selection and HF staging. +Slow uploads cannot delay socket acknowledgement or telemetry delivery. Publication IDs do not depend on telemetry run IDs. Jobs use SQLAlchemy and Alembic in the existing local SQLite database. Pending jobs survive missing or @@ -29,7 +30,7 @@ aggregate summaries can be published publicly. | Setting | Source and consumer | Provisioning | | --- | --- | --- | | `HF_TOKEN` / `HUGGINGFACE_TOKEN` | Existing operator credential, exchanged by the service for a short-lived collector session | Existing environment or Hugging Face login; never committed | -| Collector origin | Tracked `PRODUCTION_COLLECTOR_URL` in the existing service | No producer variable or destination override | +| Collector origin | Tracked `PRODUCTION_COLLECTOR_URL` in provider core | No producer variable or destination override | | Runs origin | Tracked `DEFAULT_GRAPH_PUBLICATION_ORIGIN` | No new producer variable | | `XDG_CACHE_HOME` | Optional existing cache-location setting | Defaults to the user's cache directory | @@ -56,15 +57,31 @@ an upload or stage to Hugging Face. ## Retry ```bash -microcosm-publish-graph --publication-id PUBLICATION_ID \ +microcosm-provider-publish-graph --publication-id PUBLICATION_ID \ --directory /path/to/attempt/orrery-publication ``` The preserved directory contains `orrery.upload.json` and the exact files. Retries update a separate `publication.status.json`, never the original HF receipt snapshot. Pending jobs also resume when a subsequent local service -starts. A job whose preserved directory has been removed stays pending with -error code `preserved_files_missing` until its files are restored to the same -directory; it does not stop delivery of other jobs. Moving this -service into the `microcosm-emitter` package is a separate, deferred consumer -change; this implementation does not require that package. +starts. The retry command comes from the provider package; Microcosm does not +retain a second delivery implementation or a compatibility wrapper. + +## Deferred migration prerequisites + +This adapter is implemented and tested in a draft PR, not merged or activated. +It depends on PolicyEngine/microcosm-emitter#4 (stacked on #2). +The draft uses exact Git source pins in `packages/microcosm-build/pyproject.toml` +and `uv.lock` because provider version 0.1.0 has not been released by this work. +Before merge, verify registry-owner configuration, release all four Python +packages, remove the draft source overrides, regenerate `uv.lock`, update its +approved worker-identity digest and rerun tests and wheel checks. Merely building +a Microcosm wheel does not make its unpublished dependencies installable. + +The provider rejects unversioned databases without modification. Before switching +an existing workstation, upgrade any old unversioned queue using the preceding +Microcosm implementation. Versioned telemetry and graph jobs keep their existing +IDs, eligibility, path and Alembic revision; do not delete a queue to migrate. +No new runtime variable, credential, hosted service or infrastructure is required +by this migration. The previously required hosted authorization/API deployments +remain prerequisites for remote publication. diff --git a/packages/microcosm-build/pyproject.toml b/packages/microcosm-build/pyproject.toml index ed07ea12da..c3d1a7eeae 100644 --- a/packages/microcosm-build/pyproject.toml +++ b/packages/microcosm-build/pyproject.toml @@ -7,12 +7,10 @@ requires-python = ">=3.13" license = { text = "MIT" } authors = [{ name = "PolicyEngine" }] dependencies = [ - "alembic>=1.13.3,<2", "numpy>=1.26", "pandas>=2", "scipy>=1.13", "huggingface-hub>=0.20", - "httpx>=0.27,<1", "microcosm-frame>=0.1,<0.2", "microcosm-graph>=0.1,<0.2", "microcosm-fit>=0.1,<0.2", @@ -26,7 +24,9 @@ dependencies = [ "pyyaml>=6", "jsonschema>=4.23,<5", "referencing>=0.35,<1", - "sqlalchemy>=2,<3", + "microcosm-provider-client==0.1.0", + "microcosm-provider-telemetry==0.1.0", + "microcosm-provider-orrery==0.1.0", ] [project.optional-dependencies] @@ -51,7 +51,6 @@ us = [ uk = ["policyengine-uk>=2.123.0", "microcosm-data>=0.1,<0.2", "h5py>=3", "tables>=3", "openpyxl>=3.1"] [project.scripts] -microcosm-publish-graph = "microcosm.build.graph_publication_cli:main" microcosm-build-uk = "microcosm.build.uk_runtime.full_build_cli:main" microcosm-export-us-l0-refit-h5 = "microcosm.build.us_runtime.l0_refit_export:main" @@ -84,3 +83,8 @@ microcosm-fit = { workspace = true } microcosm-calibrate = { workspace = true } microcosm-diagnostics = { workspace = true } microcosm-data = { workspace = true } +# Draft-only source pins. Replace with published 0.1.0 distributions before merge. +microcosm-provider-client = { git = "https://github.com/PolicyEngine/microcosm-emitter", rev = "06e47847983ee67ccdb6a596062251322d07f793", subdirectory = "packages/provider-client" } +microcosm-provider-core = { git = "https://github.com/PolicyEngine/microcosm-emitter", rev = "06e47847983ee67ccdb6a596062251322d07f793", subdirectory = "packages/provider-core" } +microcosm-provider-telemetry = { git = "https://github.com/PolicyEngine/microcosm-emitter", rev = "06e47847983ee67ccdb6a596062251322d07f793", subdirectory = "packages/provider-telemetry" } +microcosm-provider-orrery = { git = "https://github.com/PolicyEngine/microcosm-emitter", rev = "06e47847983ee67ccdb6a596062251322d07f793", subdirectory = "packages/provider-orrery" } diff --git a/packages/microcosm-build/src/microcosm/build/graph_publication_cli.py b/packages/microcosm-build/src/microcosm/build/graph_publication_cli.py deleted file mode 100644 index 89318d838a..0000000000 --- a/packages/microcosm-build/src/microcosm/build/graph_publication_cli.py +++ /dev/null @@ -1,58 +0,0 @@ -"""Retry preserved graph uploads without registering a telemetry run.""" - -from __future__ import annotations - -import argparse -import json -from pathlib import Path - -from microcosm.build.telemetry_emitter import default_spool_path -from microcosm.build.telemetry_emitter_service.auth import ( - CollectorSession, - validated_origin, -) -from microcosm.build.telemetry_emitter_service.constants import PRODUCTION_COLLECTOR_URL -from microcosm.build.telemetry_emitter_service.graph_publication import ( - DEFAULT_GRAPH_PUBLICATION_ORIGIN, - GraphPublicationDelivery, - GraphPublicationQueue, -) - - -def main(argv: list[str] | None = None) -> int: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--spool", type=Path, default=default_spool_path()) - parser.add_argument("--publication-id", required=True) - parser.add_argument( - "--directory", - type=Path, - help="Optional preserved bundle containing orrery.upload.json; re-enqueues after moving machines.", - ) - parser.add_argument("--origin", default=DEFAULT_GRAPH_PUBLICATION_ORIGIN) - args = parser.parse_args(argv) - queue = GraphPublicationQueue(args.spool) - if args.directory is not None: - inventory = json.loads((args.directory / "orrery.upload.json").read_text()) - if inventory["publication_id"] != args.publication_id: - parser.error("The preserved inventory has a different publication ID.") - queue.enqueue(args.directory, inventory) - if queue.receipt(args.publication_id) is None: - parser.error("Publication job not found; supply its preserved directory.") - queue.retry(args.publication_id) - session = CollectorSession(validated_origin(PRODUCTION_COLLECTOR_URL)) - delivery = GraphPublicationDelivery( - queue, - credential=session.credential, - origin=args.origin, - invalidate_credential=session.invalidate, - ) - # Explicit retries process this job only, not unrelated pending jobs. - delivery.flush_once(publication_id=args.publication_id) - receipt = queue.receipt(args.publication_id) - assert receipt is not None, "A queued graph publication must have a receipt." - print(json.dumps(receipt, sort_keys=True)) - return 0 if receipt["status"] == "published" else 1 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/packages/microcosm-build/src/microcosm/build/graph_publication_types.py b/packages/microcosm-build/src/microcosm/build/graph_publication_types.py deleted file mode 100644 index a0f03f6a8c..0000000000 --- a/packages/microcosm-build/src/microcosm/build/graph_publication_types.py +++ /dev/null @@ -1,60 +0,0 @@ -"""Explicit data contracts for immutable graph publication.""" - -from dataclasses import dataclass -from pathlib import Path -from typing import Literal, NotRequired, TypedDict - -type PublicationFileRole = Literal[ - "graph", - "schema", - "index", - "declaration", - "manifest", - "binding", - "summaries", - "upstream", -] - - -type GraphJobStatus = Literal["pending", "published"] - - -class PublicationFile(TypedDict): - """One public file, identified by its exact bytes and allowed role.""" - - name: str - role: PublicationFileRole - bytes: int - sha256: str - - -class PublicationInventory(TypedDict): - """The versioned file inventory submitted to the publication API.""" - - version: Literal[1] - publication_id: str - files: list[PublicationFile] - - -class PublicationReceipt(TypedDict): - """Local publication result, including retry and build-snapshot fields.""" - - version: Literal[1] - publication_id: str | None - status: Literal["pending", "published", "skipped", "failed", "no_evidence"] - error_code: str | None - url: NotRequired[str | None] - attempts: NotRequired[int] - inventory_sha256: NotRequired[str] - preserved_directory: NotRequired[str] - recorded_execution: NotRequired[bool] - - -@dataclass(frozen=True) -class ClaimedGraphPublication: - """A job reserved until lease_until; the timestamp also identifies its lease.""" - - publication_id: str - directory: Path - inventory: PublicationInventory - lease_until: float diff --git a/packages/microcosm-build/src/microcosm/build/telemetry_emitter.py b/packages/microcosm-build/src/microcosm/build/telemetry_emitter.py index 336b846fed..f8b91c6e3d 100644 --- a/packages/microcosm-build/src/microcosm/build/telemetry_emitter.py +++ b/packages/microcosm-build/src/microcosm/build/telemetry_emitter.py @@ -1,147 +1,34 @@ -"""Best-effort client for the local telemetry emitter service. - -The build process only writes small messages to a private local socket. A -separate process owns retry, authentication, resource sampling, and network -I/O so collector availability cannot delay or fail a dataset build. -""" +"""Microcosm metadata adapter for the independently packaged local provider.""" from __future__ import annotations -import json -import os -import socket -import subprocess import sys -import tempfile -import time -import uuid -from collections.abc import Mapping -from dataclasses import dataclass, field -from datetime import UTC, datetime from pathlib import Path from typing import Any -from microcosm.build.graph_publication_types import ( - PublicationInventory, - PublicationReceipt, +from microcosm_provider_client.launcher import start_services +from microcosm_provider_orrery.client import default_spool_path, publish_graph +from microcosm_provider_orrery.contracts import PublicationInventory, PublicationReceipt +from microcosm_provider_telemetry.client import ( + LocalTelemetryEmitter as ProviderTelemetryEmitter, +) +from microcosm_provider_telemetry.client import ( + TelemetryRun, ) -from microcosm.build.telemetry_emitter_constants import ( +from microcosm_provider_telemetry.constants import ( DEFAULT_HEARTBEAT_SECONDS, - DEFAULT_SEND_TIMEOUT_SECONDS, - DEFAULT_STARTUP_TIMEOUT_SECONDS, - INVALID_ACKNOWLEDGEMENT_ERROR, - NO_LOCAL_SOCKET_WARNING, - QUEUE_WARNING, - RUNTIME_DIRECTORY_PREFIX, - SERVICE_NOT_READY_WARNING, - SERVICE_PROCESS_EXIT_TIMEOUT_SECONDS, - SERVICE_READY_TIMEOUT_SECONDS, SERVICE_START_WARNING, - SOCKET_FILENAME, - STARTUP_POLL_SECONDS, - TELEMETRY_CACHE_PARTS, - TELEMETRY_SERVICE_MODULE, - TELEMETRY_SPOOL_FILENAME, - TEMPORARY_DIRECTORY_ALIAS, -) -from microcosm.build.telemetry_identity import runtime_identity -from microcosm.build.telemetry_protocol import ( - ACTION_CLOSE, - ACTION_EVENT, - BUILD_BLOCKED_MESSAGE, - BUILD_COMPLETED_MESSAGE, - BUILD_STARTED_MESSAGE, - CALIBRATION_EVENT_KIND, - EVENT_TYPE_CALIBRATION, - EVENT_TYPE_PROGRESS, - EVENT_TYPE_RUN, - EVENT_TYPE_STAGE, - LOCAL_ACKNOWLEDGEMENT_OK, - LOCAL_ACKNOWLEDGEMENT_READ_BYTES, - LOCAL_MESSAGE_DELIMITER, - LOCAL_PING_MESSAGE, - MAX_TELEMETRY_MESSAGE_CHARS, - SEQUENTIAL_STATUS_MAP, - STAGE_BLOCKED, - STAGE_CALIBRATING, - STAGE_COMPLETE, - STAGE_CREATED, - STAGE_FAILED, - STATUS_BLOCKED, - STATUS_COMPLETED, - STATUS_FAILED, - STATUS_PROGRESS, - STATUS_STARTED, - TelemetryEventType, - TelemetryStatus, -) -from microcosm.build.telemetry_sanitization import ( - sanitize_details, - sanitize_text, ) - -def _now() -> str: - return datetime.now(UTC).isoformat() - - -def _cache_dir() -> Path: - configured = os.environ.get("XDG_CACHE_HOME", "").strip() - root = Path(configured).expanduser() if configured else Path.home() / ".cache" - return root.joinpath(*TELEMETRY_CACHE_PARTS) - - -def default_spool_path() -> Path: - """The local SQLite queue shared by telemetry events and graph publication.""" - return _cache_dir() / TELEMETRY_SPOOL_FILENAME - - -@dataclass(frozen=True) -class TelemetryRun: - """Identity registered with the hosted collector for one build.""" - - run_id: str - country_code: str - pipeline: str - candidate_id: str | None = None - release_id: str | None = None - run_kind: str = "build" - producer_id: str = field(default_factory=lambda: uuid.uuid4().hex) - - def as_registration(self) -> dict[str, Any]: - return { - "run_id": self.run_id, - "producer_id": self.producer_id, - "country_code": self.country_code, - "pipeline": self.pipeline, - "candidate_id": self.candidate_id, - "release_id": self.release_id, - "run_kind": self.run_kind, - } +from microcosm.build.telemetry_identity import runtime_identity -class LocalTelemetryEmitter: - """Non-blocking handle owned by the instrumented build process.""" +class LocalTelemetryEmitter(ProviderTelemetryEmitter): + """Supply build identity and select both workers in one provider process.""" - def __init__( - self, - *, - run: TelemetryRun, - process: subprocess.Popen[bytes] | None, - socket_path: Path | None, - runtime_dir: Path | None, - send_timeout_seconds: float = DEFAULT_SEND_TIMEOUT_SECONDS, - spool_path: Path | None = None, - ) -> None: - self.run = run - self._process = process - self._socket_path = socket_path - self._runtime_dir = runtime_dir - self._send_timeout_seconds = send_timeout_seconds - self._spool_path = spool_path - self._closed = False - self._warned = False - self._transition_stage: str | None = None + def __init__(self, *, run, transport, handle=None, spool_path=None): + super().__init__(run=run, transport=transport, handle=handle) + self._spool_path = Path(spool_path) if spool_path else None @classmethod def start( @@ -155,11 +42,9 @@ def start( run_kind: str = "build", development_collector_url: str | None = None, heartbeat_seconds: float = DEFAULT_HEARTBEAT_SECONDS, - startup_timeout_seconds: float = DEFAULT_STARTUP_TIMEOUT_SECONDS, + startup_timeout_seconds: float = 3.0, spool_path: Path | str | None = None, ) -> LocalTelemetryEmitter: - """Start the service, returning a harmless disabled handle on failure.""" - run = TelemetryRun( run_id=run_id, country_code=country_code, @@ -168,301 +53,45 @@ def start( release_id=release_id, run_kind=run_kind, ) - if not hasattr(socket, "AF_UNIX"): - print(NO_LOCAL_SOCKET_WARNING, file=sys.stderr) - return cls(run=run, process=None, socket_path=None, runtime_dir=None) - - runtime_dir: Path | None = None - socket_path: Path | None = None - process: subprocess.Popen[bytes] | None = None + path = None try: - # macOS limits AF_UNIX paths to roughly 100 bytes. Its default - # temporary directory is already long, so use the short system alias. - temporary_root = ( - TEMPORARY_DIRECTORY_ALIAS - if TEMPORARY_DIRECTORY_ALIAS.is_dir() - else None - ) - runtime_dir = Path( - tempfile.mkdtemp(prefix=RUNTIME_DIRECTORY_PREFIX, dir=temporary_root) - ) - runtime_dir.chmod(0o700) - socket_path = runtime_dir / SOCKET_FILENAME - queue_path = Path(spool_path) if spool_path else default_spool_path() - command = [ - sys.executable, - "-m", - TELEMETRY_SERVICE_MODULE, - "--socket", - str(socket_path), - "--spool", - str(queue_path), - "--registration-json", - json.dumps(run.as_registration(), separators=(",", ":")), - "--parent-pid", - str(os.getpid()), - "--heartbeat-seconds", - str(heartbeat_seconds), - ] - if development_collector_url is not None: - command.extend( - ["--development-collector-url", development_collector_url] - ) - process = subprocess.Popen( - command, - stdin=subprocess.DEVNULL, - stdout=subprocess.DEVNULL, - start_new_session=True, + path = Path(spool_path) if spool_path else default_spool_path() + identity = runtime_identity() + shared = { + "spool_path": str(path), + "development_collector_url": development_collector_url, + } + handle = start_services( + { + "telemetry": { + **shared, + "registration": run.as_registration(), + "heartbeat_seconds": heartbeat_seconds, + }, + "orrery": shared, + }, + startup_timeout=startup_timeout_seconds, ) - deadline = time.monotonic() + startup_timeout_seconds - while time.monotonic() < deadline: - if socket_path.exists() and _service_ready(socket_path): - emitter = cls( - run=run, - process=process, - socket_path=socket_path, - runtime_dir=runtime_dir, - spool_path=queue_path, - ) - emitter.emit( - event_type=EVENT_TYPE_RUN, - stage_id=STAGE_CREATED, - status=STATUS_STARTED, - message=BUILD_STARTED_MESSAGE, - details={"identity": runtime_identity()}, - ) - return emitter - if process.poll() is not None: - break - time.sleep(STARTUP_POLL_SECONDS) except Exception as error: print( - SERVICE_START_WARNING.format( - error_type=type(error).__name__, - error=error, - ), + SERVICE_START_WARNING.format(error_type=type(error).__name__), file=sys.stderr, ) - else: - print(SERVICE_NOT_READY_WARNING, file=sys.stderr) - if process is not None: - _terminate_process(process) - try: - if socket_path is not None: - socket_path.unlink(missing_ok=True) - if runtime_dir is not None: - runtime_dir.rmdir() - except OSError: - pass - return cls( + return cls(run=run, transport=None, spool_path=path) + emitter = cls( run=run, - process=None, - socket_path=None, - runtime_dir=runtime_dir, - spool_path=Path(spool_path) if spool_path else None, - ) - - @property - def available(self) -> bool: - return self._socket_path is not None and not self._closed - - def emit( - self, - *, - event_type: TelemetryEventType, - status: TelemetryStatus, - stage_id: str | None = None, - message: str | None = None, - details: Mapping[str, Any] | None = None, - ) -> None: - """Queue one event locally, never raising into the build.""" - - if not self.available: - return - self._send( - { - "action": ACTION_EVENT, - "event": { - "timestamp": _now(), - "event_type": event_type, - "stage_id": stage_id, - "status": status, - "message": ( - sanitize_text(message, limit=MAX_TELEMETRY_MESSAGE_CHARS) - if message - else None - ), - "details": sanitize_details(details or {}), - }, - } - ) - - def stage( - self, - stage_id: str, - *, - status: TelemetryStatus = STATUS_STARTED, - message: str | None = None, - **details: Any, - ) -> None: - self.emit( - event_type=EVENT_TYPE_STAGE, - stage_id=stage_id, - status=status, - message=message, - details=details, + transport=handle.client.for_module("telemetry"), + handle=handle, + spool_path=path, ) - - def transition_stage( - self, - stage_id: str, - *, - status: str = "running", - message: str | None = None, - **details: Any, - ) -> None: - """Translate a sequential stage update into explicit lifecycle events.""" - - collector_status = SEQUENTIAL_STATUS_MAP.get(status, STATUS_PROGRESS) - if collector_status == STATUS_STARTED: - if self._transition_stage == stage_id: - self.stage( - stage_id, - status=STATUS_PROGRESS, - message=message, - **details, - ) - return - self._close_transition_stage() - self._transition_stage = stage_id - elif self._transition_stage == stage_id: - self._transition_stage = None - else: - self._close_transition_stage() - self.stage( - stage_id, - status=collector_status, - message=message, - **details, + emitter.emit( + event_type="run", + stage_id="created", + status="started", + message="Build started", + details={"identity": identity}, ) - - def progress( - self, - stage_id: str, - *, - done: int, - total: int, - unit: str | None = None, - **details: Any, - ) -> None: - self.emit( - event_type=EVENT_TYPE_PROGRESS, - stage_id=stage_id, - status=STATUS_PROGRESS, - details={"done": done, "total": total, "unit": unit, **details}, - ) - - def calibration_progress(self, event: Mapping[str, Any]) -> None: - if event.get("kind") != CALIBRATION_EVENT_KIND: - return - self.emit( - event_type=EVENT_TYPE_CALIBRATION, - stage_id=STAGE_CALIBRATING, - status=STATUS_PROGRESS, - details=event, - ) - - def transition_calibration_progress(self, event: Mapping[str, Any]) -> None: - """Enter the sequential calibration stage, then report one epoch.""" - - if event.get("kind") != CALIBRATION_EVENT_KIND: - return - if self._transition_stage != STAGE_CALIBRATING: - self.transition_stage(STAGE_CALIBRATING) - self.calibration_progress(event) - - def fail( - self, - error: BaseException, - *, - failed_during: str | None = None, - failure_class: str = "build_failure", - error_code: str | None = None, - ) -> None: - failed_stage = failed_during or self._transition_stage - message = str(error)[:MAX_TELEMETRY_MESSAGE_CHARS] - details = { - "error_type": type(error).__name__, - "failure_class": failure_class, - "failed_during": failed_stage, - } - if error_code is not None: - details["error_code"] = error_code - self._close_transition_stage(status=STATUS_FAILED, message=message, **details) - self.emit( - event_type=EVENT_TYPE_RUN, - stage_id=STAGE_FAILED, - status=STATUS_FAILED, - message=message, - details=details, - ) - self.close() - - def block( - self, - *, - phase: str, - blocking_gate_ids: list[str] | tuple[str, ...], - blocking_failure_count: int | None = None, - gate_statuses: Mapping[str, str] | None = None, - ) -> None: - """End the run as ``blocked``: the gates refused its candidate. - - A dedicated run event rather than a stage transition, whose status map - would read an unknown ``blocked`` as progress. The open stage itself - finished; it closes as completed.""" - - gate_ids = [str(gate_id) for gate_id in blocking_gate_ids] - details: dict[str, Any] = { - "phase": phase, - "blocking_failure_count": ( - len(gate_ids) - if blocking_failure_count is None - else blocking_failure_count - ), - "blocking_gate_ids": gate_ids, - } - if gate_statuses is not None: - details["gate_statuses"] = { - str(gate_id): str(status) for gate_id, status in gate_statuses.items() - } - self._close_transition_stage() - self.emit( - event_type=EVENT_TYPE_RUN, - stage_id=STAGE_BLOCKED, - status=STATUS_BLOCKED, - message=BUILD_BLOCKED_MESSAGE.format(phase=phase), - details=details, - ) - self.close() - - def complete(self) -> None: - self._close_transition_stage() - self.emit( - event_type=EVENT_TYPE_RUN, - stage_id=STAGE_COMPLETE, - status=STATUS_COMPLETED, - message=BUILD_COMPLETED_MESSAGE, - ) - self.close() - - def close(self) -> None: - """Ask the service to flush in the background, then release the handle.""" - - if self._closed: - return - self._send({"action": ACTION_CLOSE}) - self._closed = True + return emitter def publish_graph( self, @@ -471,98 +100,15 @@ def publish_graph( *, wait_seconds: float = 30.0, ) -> PublicationReceipt: - """Preserve a job first, then wait a bounded time for its first result. - - Unlike telemetry, missing credentials never discard this job. Enqueue - directly into the service's shared SQLite database so a socket timeout - or unavailable service cannot lose the publication request. - """ - from microcosm.build.telemetry_emitter_service.graph_publication import ( - GraphPublicationQueue, + return publish_graph( + directory, + inventory, + spool_path=self._spool_path, + available=lambda: self.available, + wait_seconds=wait_seconds, ) - queue = GraphPublicationQueue(self._spool_path or default_spool_path()) - queue.enqueue(directory, inventory) - deadline = time.monotonic() + max(0, wait_seconds) - while self.available and time.monotonic() < deadline: - receipt = queue.receipt(inventory["publication_id"]) - assert receipt is not None, ( - "A queued graph publication must have a receipt." - ) - if receipt.get("attempts", 0) or receipt["status"] == "published": - return receipt - time.sleep(0.1) - receipt = queue.receipt(inventory["publication_id"]) - assert receipt is not None, "A queued graph publication must have a receipt." - return receipt - - def _close_transition_stage( - self, - *, - status: TelemetryStatus = STATUS_COMPLETED, - message: str | None = None, - **details: Any, - ) -> None: - if self._transition_stage is None: - return - stage_id = self._transition_stage - self._transition_stage = None - self.stage(stage_id, status=status, message=message, **details) - - def _send(self, payload: Mapping[str, Any]) -> None: - if self._socket_path is None: - return - try: - with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as client: - client.settimeout(self._send_timeout_seconds) - client.connect(str(self._socket_path)) - client.sendall( - json.dumps(payload, separators=(",", ":"), allow_nan=False).encode() - + LOCAL_MESSAGE_DELIMITER - ) - acknowledgement = client.recv(LOCAL_ACKNOWLEDGEMENT_READ_BYTES) - if acknowledgement != LOCAL_ACKNOWLEDGEMENT_OK: - raise OSError(INVALID_ACKNOWLEDGEMENT_ERROR) - except Exception as error: - if not self._warned: - print( - QUEUE_WARNING.format(error_type=type(error).__name__), - file=sys.stderr, - ) - self._warned = True - - -def start_local_telemetry_emitter_service( - **run: Any, -) -> LocalTelemetryEmitter: - """Named construction seam for build entrypoints and tests.""" +def start_local_telemetry_emitter_service(**run: Any) -> LocalTelemetryEmitter: + """Construct the build adapter without importing provider service internals.""" return LocalTelemetryEmitter.start(**run) - - -def _service_ready(socket_path: Path) -> bool: - try: - with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as client: - client.settimeout(SERVICE_READY_TIMEOUT_SECONDS) - client.connect(str(socket_path)) - client.sendall(LOCAL_PING_MESSAGE) - return ( - client.recv(LOCAL_ACKNOWLEDGEMENT_READ_BYTES) - == LOCAL_ACKNOWLEDGEMENT_OK - ) - except OSError: - return False - - -def _terminate_process(process: subprocess.Popen[bytes]) -> None: - """Terminate and reap an emitter service that did not become usable.""" - - try: - if process.poll() is None: - process.terminate() - process.wait(timeout=SERVICE_PROCESS_EXIT_TIMEOUT_SECONDS) - except subprocess.TimeoutExpired: - process.kill() - process.wait(timeout=SERVICE_PROCESS_EXIT_TIMEOUT_SECONDS) - except OSError: - pass diff --git a/packages/microcosm-build/src/microcosm/build/telemetry_emitter_constants.py b/packages/microcosm-build/src/microcosm/build/telemetry_emitter_constants.py deleted file mode 100644 index 57a477f82b..0000000000 --- a/packages/microcosm-build/src/microcosm/build/telemetry_emitter_constants.py +++ /dev/null @@ -1,38 +0,0 @@ -"""Configuration and user-facing messages for the telemetry emitter client.""" - -from __future__ import annotations - -from pathlib import Path -from typing import Final - -TELEMETRY_SERVICE_MODULE: Final = "microcosm.build.telemetry_emitter_service" -TELEMETRY_CACHE_PARTS: Final = ("microcosm", "telemetry") -TELEMETRY_SPOOL_FILENAME: Final = "events.sqlite3" -RUNTIME_DIRECTORY_PREFIX: Final = "microcosm-telemetry-" -SOCKET_FILENAME: Final = "emitter.sock" -TEMPORARY_DIRECTORY_ALIAS: Final = Path("/tmp") - -DEFAULT_SEND_TIMEOUT_SECONDS: Final = 0.2 -DEFAULT_HEARTBEAT_SECONDS: Final = 60.0 -DEFAULT_STARTUP_TIMEOUT_SECONDS: Final = 3.0 -STARTUP_POLL_SECONDS: Final = 0.02 -SERVICE_READY_TIMEOUT_SECONDS: Final = 0.1 -SERVICE_PROCESS_EXIT_TIMEOUT_SECONDS: Final = 1.0 - -NO_LOCAL_SOCKET_WARNING: Final = ( - "warning: Microcosm telemetry is unavailable because this platform has no " - "local Unix sockets." -) -SERVICE_START_WARNING: Final = ( - "warning: the local telemetry emitter service could not start: " - "{error_type}: {error}" -) -SERVICE_NOT_READY_WARNING: Final = ( - "warning: the local telemetry emitter service did not become ready; " - "the build will continue without hosted telemetry." -) -QUEUE_WARNING: Final = ( - "warning: the local telemetry emitter service could not queue an update " - "({error_type}); the build will continue." -) -INVALID_ACKNOWLEDGEMENT_ERROR: Final = "invalid acknowledgement" diff --git a/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/__init__.py b/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/__init__.py deleted file mode 100644 index ef5335561f..0000000000 --- a/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/__init__.py +++ /dev/null @@ -1,17 +0,0 @@ -"""Local service that queues and delivers Microcosm telemetry.""" - -from microcosm.build.telemetry_emitter_service.collector import CollectorDelivery -from microcosm.build.telemetry_emitter_service.constants import ( - PRODUCTION_COLLECTOR_URL, -) -from microcosm.build.telemetry_emitter_service.resources import ProcessTreeSampler -from microcosm.build.telemetry_emitter_service.runtime import EmitterService -from microcosm.build.telemetry_emitter_service.spool import EventSpool - -__all__ = [ - "CollectorDelivery", - "EmitterService", - "EventSpool", - "PRODUCTION_COLLECTOR_URL", - "ProcessTreeSampler", -] diff --git a/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/__main__.py b/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/__main__.py deleted file mode 100644 index 2981038314..0000000000 --- a/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/__main__.py +++ /dev/null @@ -1,5 +0,0 @@ -"""Module execution entry point for the telemetry emitter service.""" - -from microcosm.build.telemetry_emitter_service.main import main - -raise SystemExit(main()) diff --git a/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/alembic/env.py b/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/alembic/env.py deleted file mode 100644 index 2cb2edcc25..0000000000 --- a/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/alembic/env.py +++ /dev/null @@ -1,28 +0,0 @@ -"""Alembic environment for the local telemetry spool.""" - -from alembic import context -from sqlalchemy.engine import Connection - -from microcosm.build.telemetry_emitter_service.models import SpoolModel - -_MISSING_CONNECTION_ERROR = ( - "telemetry spool migrations require a programmatically supplied connection" -) - - -def run_migrations() -> None: - """Run migrations on the connection supplied by the emitter service.""" - - connection = context.config.attributes.get("connection") - if not isinstance(connection, Connection): - raise RuntimeError(_MISSING_CONNECTION_ERROR) - context.configure( - connection=connection, - target_metadata=SpoolModel.metadata, - render_as_batch=True, - ) - with context.begin_transaction(): - context.run_migrations() - - -run_migrations() diff --git a/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/alembic/script.py.mako b/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/alembic/script.py.mako deleted file mode 100644 index f059f7642d..0000000000 --- a/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/alembic/script.py.mako +++ /dev/null @@ -1,31 +0,0 @@ -"""${message} - -Revision ID: ${up_revision} -Revises: ${down_revision | comma,n} -Create Date: ${create_date} -""" - -from __future__ import annotations - -from collections.abc import Sequence - -import sqlalchemy as sa -from alembic import op -${imports if imports else ""} - -revision: str = ${repr(up_revision)} -down_revision: str | None = ${repr(down_revision)} -branch_labels: str | Sequence[str] | None = ${repr(branch_labels)} -depends_on: str | Sequence[str] | None = ${repr(depends_on)} - - -def upgrade() -> None: - """Apply this schema revision.""" - - ${upgrades if upgrades else "pass"} - - -def downgrade() -> None: - """Reverse this schema revision.""" - - ${downgrades if downgrades else "pass"} diff --git a/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/alembic/versions/v20261007_01_initial_spool.py b/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/alembic/versions/v20261007_01_initial_spool.py deleted file mode 100644 index 87290fd653..0000000000 --- a/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/alembic/versions/v20261007_01_initial_spool.py +++ /dev/null @@ -1,140 +0,0 @@ -"""Create or adopt the telemetry spool schema. - -Revision ID: 20261007_01 -Revises: -""" - -from __future__ import annotations - -from collections.abc import Sequence - -import sqlalchemy as sa -from alembic import op - -revision: str = "20261007_01" -down_revision: str | None = None -branch_labels: str | Sequence[str] | None = None -depends_on: str | Sequence[str] | None = None - -_RUNS_TABLE = "telemetry_runs" -_EVENTS_TABLE = "telemetry_events" -_EVENTS_INDEX = "telemetry_events_run_sequence" -_PENDING_UPLOAD_STATE = "pending" -_LOCAL_ONLY_UPLOAD_STATE = "local_only" -_PRE_ELIGIBILITY_REASON = "created_before_upload_eligibility" - - -def _create_runs_table() -> None: - op.create_table( - _RUNS_TABLE, - sa.Column("run_id", sa.Text(), nullable=False), - sa.Column("producer_id", sa.Text(), nullable=False), - sa.Column("registration_json", sa.Text(), nullable=False), - sa.Column( - "next_sequence", - sa.Integer(), - nullable=False, - server_default="1", - ), - sa.Column( - "upload_state", - sa.Text(), - nullable=False, - server_default=_PENDING_UPLOAD_STATE, - ), - sa.Column("local_only_reason", sa.Text(), nullable=True), - sa.Column("updated_at", sa.Text(), nullable=False), - sa.PrimaryKeyConstraint("run_id", "producer_id"), - ) - - -def _adopt_runs_table(inspector: sa.Inspector) -> None: - column_names = {column["name"] for column in inspector.get_columns(_RUNS_TABLE)} - added_upload_state = "upload_state" not in column_names - if added_upload_state: - op.add_column( - _RUNS_TABLE, - sa.Column( - "upload_state", - sa.Text(), - nullable=False, - server_default=_PENDING_UPLOAD_STATE, - ), - ) - added_local_only_reason = "local_only_reason" not in column_names - if added_local_only_reason: - op.add_column( - _RUNS_TABLE, - sa.Column("local_only_reason", sa.Text(), nullable=True), - ) - - runs = sa.table( - _RUNS_TABLE, - sa.column("upload_state", sa.Text()), - sa.column("local_only_reason", sa.Text()), - ) - if added_upload_state: - op.execute(sa.update(runs).values(upload_state=_LOCAL_ONLY_UPLOAD_STATE)) - if added_upload_state or added_local_only_reason: - op.execute( - sa.update(runs) - .where(runs.c.upload_state == _LOCAL_ONLY_UPLOAD_STATE) - .values(local_only_reason=_PRE_ELIGIBILITY_REASON) - ) - - -def _create_events_table() -> None: - op.create_table( - _EVENTS_TABLE, - sa.Column("event_id", sa.Text(), nullable=False), - sa.Column("run_id", sa.Text(), nullable=False), - sa.Column("producer_id", sa.Text(), nullable=False), - sa.Column("sequence", sa.Integer(), nullable=False), - sa.Column("payload_json", sa.Text(), nullable=False), - sa.Column("created_at", sa.Text(), nullable=False), - sa.ForeignKeyConstraint( - ["run_id", "producer_id"], - [f"{_RUNS_TABLE}.run_id", f"{_RUNS_TABLE}.producer_id"], - name="fk_telemetry_events_run", - ), - sa.PrimaryKeyConstraint("event_id"), - sa.UniqueConstraint( - "run_id", - "producer_id", - "sequence", - name="uq_telemetry_events_run_producer_sequence", - ), - ) - - -def upgrade() -> None: - """Create the current schema or adopt a pre-Alembic spool.""" - - inspector = sa.inspect(op.get_bind()) - table_names = set(inspector.get_table_names()) - if _RUNS_TABLE not in table_names: - _create_runs_table() - else: - _adopt_runs_table(inspector) - - inspector = sa.inspect(op.get_bind()) - table_names = set(inspector.get_table_names()) - if _EVENTS_TABLE not in table_names: - _create_events_table() - - inspector = sa.inspect(op.get_bind()) - index_names = {index["name"] for index in inspector.get_indexes(_EVENTS_TABLE)} - if _EVENTS_INDEX not in index_names: - op.create_index( - _EVENTS_INDEX, - _EVENTS_TABLE, - ["run_id", "producer_id", "sequence"], - ) - - -def downgrade() -> None: - """Remove the local spool schema.""" - - op.drop_index(_EVENTS_INDEX, table_name=_EVENTS_TABLE) - op.drop_table(_EVENTS_TABLE) - op.drop_table(_RUNS_TABLE) diff --git a/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/alembic/versions/v20261008_02_graph_publication.py b/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/alembic/versions/v20261008_02_graph_publication.py deleted file mode 100644 index 14ae0318a6..0000000000 --- a/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/alembic/versions/v20261008_02_graph_publication.py +++ /dev/null @@ -1,27 +0,0 @@ -"""Add durable graph publication jobs without changing event retention.""" - -import sqlalchemy as sa -from alembic import op - -revision = "20261008_02" -down_revision = "20261007_01" -branch_labels = None -depends_on = None - - -def upgrade() -> None: - op.create_table( - "graph_publication_jobs", - sa.Column("publication_id", sa.Text(), primary_key=True), - sa.Column("inventory", sa.Text(), nullable=False), - sa.Column("directory", sa.Text(), nullable=False), - sa.Column("status", sa.Text(), nullable=False), - sa.Column("attempts", sa.Integer(), nullable=False), - sa.Column("next_attempt_at", sa.Float(), nullable=False), - sa.Column("lease_until", sa.Float(), nullable=False), - sa.Column("receipt", sa.Text(), nullable=False), - ) - - -def downgrade() -> None: - op.drop_table("graph_publication_jobs") diff --git a/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/auth.py b/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/auth.py deleted file mode 100644 index 205fef3dd5..0000000000 --- a/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/auth.py +++ /dev/null @@ -1,180 +0,0 @@ -"""HF-to-collector session shared by telemetry and graph publication.""" - -from __future__ import annotations - -import json -import os -import threading -import time -import urllib.error -import urllib.request -from collections.abc import Callable, Mapping -from http import HTTPStatus -from typing import Any, Literal, NamedTuple -from urllib.parse import urlsplit - -from huggingface_hub import get_token - -from microcosm.build.telemetry_emitter_service.constants import ( - COLLECTOR_RESPONSE_TOO_LARGE_ERROR, - DEFAULT_TOKEN_LIFETIME_SECONDS, - DEVELOPMENT_COLLECTOR_LOOPBACK_ERROR, - HTTP_TIMEOUT_SECONDS, - HTTP_USER_AGENT, - LOOPBACK_HOSTS, - MAX_HTTP_RESPONSE_BYTES, - MINIMUM_TOKEN_LIFETIME_SECONDS, - SERVICE_ORIGIN_FORMAT_ERROR, - SERVICE_ORIGIN_HTTPS_ERROR, - TOKEN_EXCHANGE_PATH, - TOKEN_REFRESH_MARGIN_SECONDS, -) - - -class _NoRedirectHandler(urllib.request.HTTPRedirectHandler): - """Keep bearer credentials on the explicitly configured origin.""" - - def redirect_request(self, req, fp, code, msg, headers, newurl): - return None - - -type CredentialStatus = Literal["ok", "missing", "rejected", "unavailable"] - - -class CollectorCredential(NamedTuple): - """A session token, or the reason none is available for this attempt.""" - - status: CredentialStatus - token: str | None - - -def validated_origin(value: str, *, allow_loopback_http: bool = False) -> str: - """Return an HTTPS origin, or plain HTTP only for an allowed loopback host.""" - parsed = urlsplit(value) - loopback = parsed.hostname in LOOPBACK_HOSTS - valid_scheme = parsed.scheme == "https" or ( - allow_loopback_http and loopback and parsed.scheme == "http" - ) - if not parsed.hostname or not valid_scheme: - raise ValueError(SERVICE_ORIGIN_HTTPS_ERROR) - if ( - parsed.username - or parsed.password - or parsed.query - or parsed.fragment - or parsed.path not in {"", "/"} - ): - raise ValueError(SERVICE_ORIGIN_FORMAT_ERROR) - return value.rstrip("/") - - -def _development_collector_url(value: str) -> str: - parsed = urlsplit(value) - if parsed.hostname not in LOOPBACK_HOSTS: - raise ValueError(DEVELOPMENT_COLLECTOR_LOOPBACK_ERROR) - return validated_origin(value, allow_loopback_http=True) - - -def _decode_response(body: bytes) -> dict[str, Any]: - try: - response = json.loads(body) if body else {} - except (json.JSONDecodeError, UnicodeDecodeError): - return {} - return response if isinstance(response, dict) else {} - - -def _http_post( - url: str, - payload: Mapping[str, Any], - bearer_token: str, - *, - timeout: float = HTTP_TIMEOUT_SECONDS, -) -> tuple[int, dict[str, Any]]: - request = urllib.request.Request( - url, - data=json.dumps(payload, separators=(",", ":")).encode(), - headers={ - "Authorization": f"Bearer {bearer_token}", - "Content-Type": "application/json", - "User-Agent": HTTP_USER_AGENT, - }, - method="POST", - ) - try: - opener = urllib.request.build_opener(_NoRedirectHandler) - with opener.open(request, timeout=timeout) as response: - body = response.read(MAX_HTTP_RESPONSE_BYTES + 1) - if len(body) > MAX_HTTP_RESPONSE_BYTES: - raise OSError(COLLECTOR_RESPONSE_TOO_LARGE_ERROR) - return response.status, _decode_response(body) - except urllib.error.HTTPError as error: - body = error.read(MAX_HTTP_RESPONSE_BYTES + 1) - if len(body) > MAX_HTTP_RESPONSE_BYTES: - return error.code, {} - return error.code, _decode_response(body) - - -type HttpPost = Callable[[str, Mapping[str, Any], str], tuple[int, dict[str, Any]]] - - -def _huggingface_token() -> str | None: - return ( - os.environ.get("HF_TOKEN", "").strip() - or os.environ.get("HUGGINGFACE_TOKEN", "").strip() - or get_token() - ) - - -class CollectorSession: - """One thread-safe short-lived credential shared by independent publishers.""" - - def __init__( - self, - origin: str, - *, - post: HttpPost | None = None, - credential: Callable[[], str | None] | None = None, - ) -> None: - self.origin = origin - self._post = post or _http_post - self._credential = credential or _huggingface_token - self._session_token: tuple[str, float] | None = None - self._lock = threading.Lock() - - def invalidate(self) -> None: - with self._lock: - self._session_token = None - - def credential(self) -> CollectorCredential: - with self._lock: - if ( - self._session_token is not None - and self._session_token[1] - > time.monotonic() + TOKEN_REFRESH_MARGIN_SECONDS - ): - return CollectorCredential("ok", self._session_token[0]) - hf_token = self._credential() - if not hf_token: - return CollectorCredential("missing", None) - try: - status, response = self._post( - self.origin + TOKEN_EXCHANGE_PATH, {}, hf_token - ) - except (OSError, TimeoutError): - return CollectorCredential("unavailable", None) - if status == HTTPStatus.OK and isinstance( - response.get("access_token"), str - ): - try: - expires_in = max( - MINIMUM_TOKEN_LIFETIME_SECONDS, - int(response.get("expires_in", DEFAULT_TOKEN_LIFETIME_SECONDS)), - ) - except (ValueError, TypeError): - return CollectorCredential("unavailable", None) - token = response["access_token"] - self._session_token = (token, time.monotonic() + expires_in) - return CollectorCredential("ok", token) - if status in {HTTPStatus.UNAUTHORIZED, HTTPStatus.FORBIDDEN}: - return CollectorCredential("rejected", None) - return CollectorCredential("unavailable", None) diff --git a/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/collector.py b/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/collector.py deleted file mode 100644 index 8bc19a3b73..0000000000 --- a/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/collector.py +++ /dev/null @@ -1,191 +0,0 @@ -"""Best-effort telemetry delivery; shared authentication lives outside this component.""" - -from __future__ import annotations - -import sys -import time -from collections.abc import Mapping -from http import HTTPStatus -from typing import Any - -from microcosm.build.telemetry_emitter_service.auth import ( - CollectorCredential, - CollectorSession, - _development_collector_url, - _http_post, - _huggingface_token, - validated_origin, -) -from microcosm.build.telemetry_emitter_service.constants import ( - INITIAL_RETRY_SECONDS, - LOCAL_ONLY_MISSING_CREDENTIAL, - LOCAL_ONLY_REJECTED_COLLECTOR_AUTHORIZATION, - LOCAL_ONLY_REJECTED_CREDENTIAL, - LOCAL_ONLY_REJECTED_EVENTS, - LOCAL_ONLY_REJECTED_REGISTRATION, - MAX_RETRY_SECONDS, - NO_CREDENTIAL_MESSAGE, - PRODUCTION_COLLECTOR_URL, - REJECTED_CREDENTIAL_MESSAGE, - REJECTED_EVENTS_MESSAGE, - RUN_EVENTS_PATH_TEMPLATE, - RUN_REGISTRATION_PATH, -) -from microcosm.build.telemetry_emitter_service.spool import EventSpool - -#: Client errors worth retrying: a timeout and a rate limit pass with time. -#: Every other 4xx is the collector's settled answer to the same request, so -#: retrying it only wedges the queue (a payload shape it does not accept, say). -_RETRYABLE_CLIENT_ERRORS = frozenset( - {HTTPStatus.REQUEST_TIMEOUT, HTTPStatus.TOO_MANY_REQUESTS} -) - - -def _permanent_client_error(status: int) -> bool: - return 400 <= int(status) < 500 and status not in _RETRYABLE_CLIENT_ERRORS - - -def _registration_key(registration: Mapping[str, Any]) -> str: - return f"{registration['run_id']}:{registration['producer_id']}" - - -class CollectorDelivery: - """Authenticate queued runs and deliver idempotent event batches.""" - - def __init__( - self, - spool: EventSpool, - *, - development_collector_url: str | None = None, - session: CollectorSession | None = None, - ) -> None: - self.collector_url = ( - _development_collector_url(development_collector_url) - if development_collector_url is not None - else validated_origin(PRODUCTION_COLLECTOR_URL) - ) - self.spool = spool - self.session = session or CollectorSession( - self.collector_url, - post=lambda *args: _http_post(*args), - credential=lambda: _huggingface_token(), - ) - self._registered: set[str] = set() - self._warned_no_token = False - self._warned_denied: set[str] = set() - self._next_attempt_at = 0.0 - self._retry_seconds = INITIAL_RETRY_SECONDS - - def flush_once(self) -> bool: - """Attempt one delivery pass without waiting for retry deadlines.""" - - if time.monotonic() < self._next_attempt_at: - return False - made_progress = False - for registration in self.spool.pending_runs(): - run_id = str(registration["run_id"]) - registration_key = _registration_key(registration) - token = self._collector_token(registration) - if token is None: - continue - if registration_key not in self._registered: - if not self._register(registration, token): - continue - events = self.spool.batch(run_id, str(registration["producer_id"])) - if not events: - continue - try: - status, _ = _http_post( - self.collector_url + RUN_EVENTS_PATH_TEMPLATE.format(run_id=run_id), - {"events": events}, - token, - ) - except (OSError, TimeoutError): - self._defer_retry() - continue - if status == HTTPStatus.ACCEPTED: - self.spool.acknowledge([event["event_id"] for event in events]) - made_progress = True - self._retry_seconds = INITIAL_RETRY_SECONDS - elif status == HTTPStatus.UNAUTHORIZED: - self.session.invalidate() - elif status == HTTPStatus.FORBIDDEN: - self._make_local_only( - registration, - LOCAL_ONLY_REJECTED_COLLECTOR_AUTHORIZATION, - ) - elif _permanent_client_error(status): - self._make_local_only( - registration, - LOCAL_ONLY_REJECTED_EVENTS, - message=REJECTED_EVENTS_MESSAGE.format(status=int(status)), - ) - else: - self._defer_retry() - return made_progress - - def _collector_token(self, registration: Mapping[str, Any]) -> str | None: - result, token = self.session_credential() - if result == "missing": - if not self._warned_no_token: - print(NO_CREDENTIAL_MESSAGE, file=sys.stderr, flush=True) - self._warned_no_token = True - self._make_local_only(registration, LOCAL_ONLY_MISSING_CREDENTIAL) - elif result == "rejected": - self._make_local_only(registration, LOCAL_ONLY_REJECTED_CREDENTIAL) - elif result != "ok": - self._defer_retry() - return token - - def session_credential(self) -> CollectorCredential: - """Compatibility accessor; authentication has no telemetry queue policy.""" - return self.session.credential() - - def _register(self, registration: Mapping[str, Any], token: str) -> bool: - registration_key = _registration_key(registration) - try: - status, _ = _http_post( - self.collector_url + RUN_REGISTRATION_PATH, - registration, - token, - ) - except (OSError, TimeoutError): - self._defer_retry() - return False - if status == HTTPStatus.CREATED: - self._registered.add(registration_key) - self._retry_seconds = INITIAL_RETRY_SECONDS - return True - if status == HTTPStatus.UNAUTHORIZED: - self.session.invalidate() - elif status in {HTTPStatus.FORBIDDEN, HTTPStatus.CONFLICT} or ( - _permanent_client_error(status) - ): - self._make_local_only(registration, LOCAL_ONLY_REJECTED_REGISTRATION) - else: - self._defer_retry() - return False - - def _make_local_only( - self, - registration: Mapping[str, Any], - reason: str, - *, - message: str = REJECTED_CREDENTIAL_MESSAGE, - ) -> None: - run_id = str(registration["run_id"]) - producer_id = str(registration["producer_id"]) - registration_key = _registration_key(registration) - self.spool.make_local_only(run_id, producer_id, reason) - if reason != LOCAL_ONLY_MISSING_CREDENTIAL: - self._warn_denied(registration_key, message) - - def _defer_retry(self) -> None: - self._next_attempt_at = time.monotonic() + self._retry_seconds - self._retry_seconds = min(MAX_RETRY_SECONDS, self._retry_seconds * 2) - - def _warn_denied(self, registration_key: str, message: str) -> None: - if registration_key in self._warned_denied: - return - print(message, file=sys.stderr, flush=True) - self._warned_denied.add(registration_key) diff --git a/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/constants.py b/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/constants.py deleted file mode 100644 index bd7559467b..0000000000 --- a/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/constants.py +++ /dev/null @@ -1,75 +0,0 @@ -"""Configuration and protocol values for the telemetry emitter service.""" - -from __future__ import annotations - -from typing import Final - -RETENTION_DAYS: Final = 7 -MAX_QUEUED_BYTES: Final = 100 * 1024 * 1024 -BATCH_SIZE: Final = 100 -PRODUCTION_COLLECTOR_URL: Final = ( - "https://microcosm-telemetry-389282473430.us-central1.run.app" -) - -LOOPBACK_HOSTS: Final = frozenset({"localhost", "127.0.0.1", "::1"}) -TOKEN_EXCHANGE_PATH: Final = "/v1/auth/huggingface/exchange" -RUN_REGISTRATION_PATH: Final = "/v1/runs" -RUN_EVENTS_PATH_TEMPLATE: Final = "/v1/runs/{run_id}/events" -HTTP_USER_AGENT: Final = "microcosm-telemetry-emitter/1" -MAX_HTTP_RESPONSE_BYTES: Final = 1_048_576 -HTTP_TIMEOUT_SECONDS: Final = 5.0 - -UPLOAD_STATE_PENDING: Final = "pending" -UPLOAD_STATE_LOCAL_ONLY: Final = "local_only" -LOCAL_ONLY_MISSING_CREDENTIAL: Final = "missing_huggingface_credential" -LOCAL_ONLY_REJECTED_CREDENTIAL: Final = "huggingface_credential_rejected" -LOCAL_ONLY_REJECTED_REGISTRATION: Final = "run_registration_rejected" -LOCAL_ONLY_REJECTED_COLLECTOR_AUTHORIZATION: Final = "collector_authorization_rejected" -LOCAL_ONLY_REJECTED_EVENTS: Final = "collector_rejected_events" -LOCAL_ONLY_PRE_ELIGIBILITY: Final = "created_before_upload_eligibility" - -NO_CREDENTIAL_MESSAGE: Final = ( - "Microcosm telemetry is local-only: no ambient Hugging Face credential was " - "found. The dataset build will continue." -) -REJECTED_CREDENTIAL_MESSAGE: Final = ( - "Microcosm telemetry is local-only for this run: the ambient Hugging Face " - "credential was not accepted as a PolicyEngine organization member. The " - "dataset build will continue." -) -REJECTED_EVENTS_MESSAGE: Final = ( - "Microcosm telemetry is local-only for this run: the collector rejected its " - "events (HTTP {status}), so they will not be retried. The dataset build will " - "continue." -) -SERVICE_ORIGIN_HTTPS_ERROR: Final = "service URL must be an HTTPS origin" -SERVICE_ORIGIN_FORMAT_ERROR: Final = ( - "service URL must be an origin without credentials or path data" -) -DEVELOPMENT_COLLECTOR_LOOPBACK_ERROR: Final = ( - "development collector URL must use a loopback address" -) -COLLECTOR_RESPONSE_TOO_LARGE_ERROR: Final = "collector response exceeds 1 MiB" - -DEFAULT_TOKEN_LIFETIME_SECONDS: Final = 3_600 -MINIMUM_TOKEN_LIFETIME_SECONDS: Final = 60 -TOKEN_REFRESH_MARGIN_SECONDS: Final = 30 -INITIAL_RETRY_SECONDS: Final = 1.0 -MAX_RETRY_SECONDS: Final = 60.0 - -DATABASE_TIMEOUT_SECONDS: Final = 5 -PRUNE_INTERVAL_SECONDS: Final = 60.0 - -DEFAULT_HEARTBEAT_SECONDS: Final = 60.0 -DEFAULT_DRAIN_SECONDS: Final = 15.0 -MINIMUM_HEARTBEAT_SECONDS: Final = 1.0 -SOCKET_LISTEN_BACKLOG: Final = 16 -SOCKET_ACCEPT_TIMEOUT_SECONDS: Final = 0.5 -SOCKET_CONNECTION_TIMEOUT_SECONDS: Final = 0.25 -WORKER_INTERVAL_SECONDS: Final = 1.0 -DRAIN_RETRY_SECONDS: Final = 0.5 - -EVENT_OBJECT_ERROR: Final = "event must be an object" -UNSUPPORTED_ACTION_ERROR: Final = "unsupported local telemetry action" -LOCAL_MESSAGE_TOO_LARGE_ERROR: Final = "local telemetry message exceeds 1 MiB" -FAILURE_CLASS_UNEXPECTED_PROCESS_EXIT: Final = "unexpected_process_exit" diff --git a/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/database.py b/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/database.py deleted file mode 100644 index 75d32d96ba..0000000000 --- a/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/database.py +++ /dev/null @@ -1,34 +0,0 @@ -"""SQLAlchemy engine and session construction for the telemetry spool.""" - -from __future__ import annotations - -from pathlib import Path - -from sqlalchemy import URL, create_engine -from sqlalchemy.engine import Engine -from sqlalchemy.orm import Session, sessionmaker - -from microcosm.build.telemetry_emitter_service.constants import ( - DATABASE_TIMEOUT_SECONDS, -) - - -def sqlite_database_url(path: Path | str) -> URL: - """Return a SQLAlchemy URL for a local SQLite spool path.""" - - return URL.create("sqlite+pysqlite", database=str(Path(path))) - - -def create_spool_engine(path: Path | str) -> Engine: - """Create the SQLAlchemy engine used by one emitter service process.""" - - return create_engine( - sqlite_database_url(path), - connect_args={"timeout": DATABASE_TIMEOUT_SECONDS}, - ) - - -def create_spool_session_factory(engine: Engine) -> sessionmaker[Session]: - """Create short-lived ORM sessions bound to the spool engine.""" - - return sessionmaker(engine, expire_on_commit=False) diff --git a/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/graph_publication.py b/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/graph_publication.py deleted file mode 100644 index f1f95ed137..0000000000 --- a/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/graph_publication.py +++ /dev/null @@ -1,414 +0,0 @@ -"""Durable, retryable immutable graph uploads owned by the emitter service. - -Graph bytes stay in preserved files, never in event details. Pending graph jobs -are not subject to the telemetry event queue's time or size pruning. -""" - -from __future__ import annotations - -import hashlib -import json -import re -import time -import uuid -from collections.abc import Callable, Mapping -from pathlib import Path -from urllib.parse import urlsplit - -import httpx -from sqlalchemy import select, update - -from microcosm.build.graph_publication_types import ( - ClaimedGraphPublication, - PublicationFile, - PublicationFileRole, - PublicationInventory, - PublicationReceipt, -) -from microcosm.build.telemetry_emitter_service.auth import ( - CollectorCredential, - validated_origin, -) -from microcosm.build.telemetry_emitter_service.database import ( - create_spool_engine, - create_spool_session_factory, -) -from microcosm.build.telemetry_emitter_service.migrations import upgrade_spool_database -from microcosm.build.telemetry_emitter_service.models import GraphPublicationJob - -MAX_GRAPH_FILE_BYTES = 64 * 1024 * 1024 -MAX_GRAPH_TOTAL_BYTES = 512 * 1024 * 1024 -GRAPH_PUBLICATION_WAIT_SECONDS = 30.0 -GRAPH_PUBLICATION_LEASE_SECONDS = 300.0 -GRAPH_PUBLICATION_STATUS_FILENAME = "publication.status.json" -GRAPH_WORKER_ERROR_MESSAGE = ( - "Microcosm graph publication hit a local queue error; pending jobs remain " - "queued for a later attempt." -) -DEFAULT_GRAPH_PUBLICATION_ORIGIN = "https://microcosm-runs.vercel.app" -GRAPH_PUBLICATION_ROLES = { - "graph": r"graph\.orrery\.json", - "schema": r"graph\.schema\.json", - "index": r"execution\.evidence\.json", - "declaration": r"evidence-[A-Za-z0-9_-]+-graph\.json", - "manifest": r"evidence-[A-Za-z0-9_-]+-manifest\.json", - "binding": r"evidence-[A-Za-z0-9_-]+-binding\.json", - "summaries": r"evidence-[A-Za-z0-9_-]+-summaries\.json", - "upstream": r"upstream-[a-f0-9]{64}\.json", -} - - -def _digest(payload: bytes) -> str: - return hashlib.sha256(payload).hexdigest() - - -def inventory_digest(inventory: PublicationInventory) -> str: - """Digest the versioned field order shared with the publication API.""" - ordered: PublicationInventory = { - "version": inventory["version"], - "publication_id": inventory["publication_id"], - "files": [ - { - "name": item["name"], - "role": item["role"], - "bytes": item["bytes"], - "sha256": item["sha256"], - } - for item in sorted(inventory["files"], key=lambda file: file["name"]) - ], - } - return _digest( - json.dumps(ordered, separators=(",", ":"), ensure_ascii=True).encode() - ) - - -def publication_inventory( - directory: Path, - roles: Mapping[str, PublicationFileRole], - *, - publication_id: str | None = None, -) -> PublicationInventory: - """Hash an explicit allowlist of files; publication identity is not a run ID.""" - files: list[PublicationFile] = [] - for name, role in sorted(roles.items()): - if role not in GRAPH_PUBLICATION_ROLES or not re.fullmatch( - GRAPH_PUBLICATION_ROLES[role], name - ): - raise ValueError("Ineligible graph publication filename or role.") - path = Path(directory) / name - if path.is_symlink() or not path.is_file(): - raise ValueError( - "Graph publication files must be regular, preserved files." - ) - size = path.stat().st_size - if not 0 < size <= MAX_GRAPH_FILE_BYTES: - raise ValueError("Graph publication file exceeds its size limit.") - files.append( - { - "name": name, - "role": role, - "bytes": size, - "sha256": _digest(path.read_bytes()), - } - ) - if ( - not files - or len(files) > 256 - or sum(f["bytes"] for f in files) > MAX_GRAPH_TOTAL_BYTES - or sum(f["role"] == "graph" for f in files) != 1 - ): - raise ValueError("Invalid graph publication inventory.") - id_ = publication_id or uuid.uuid4().hex - if not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_-]{0,127}", id_): - raise ValueError("Invalid graph publication ID.") - return {"version": 1, "publication_id": id_, "files": files} - - -def _write_status(directory: Path, receipt: PublicationReceipt) -> None: - path = directory / GRAPH_PUBLICATION_STATUS_FILENAME - temporary = path.with_suffix(f".{uuid.uuid4().hex}.tmp") - try: - temporary.write_text(json.dumps(receipt, sort_keys=True) + "\n") - temporary.replace(path) - except OSError: - temporary.unlink(missing_ok=True) - - -class GraphPublicationQueue: - """SQLite job records with leases; retained until publication or operator action.""" - - def __init__(self, path: Path) -> None: - self.path = Path(path) - self.path.parent.mkdir(parents=True, exist_ok=True) - self._engine = create_spool_engine(self.path) - upgrade_spool_database(self._engine) - self._sessions = create_spool_session_factory(self._engine) - - def enqueue(self, directory: Path, inventory: PublicationInventory) -> None: - """Durably register exact bytes without a telemetry run or credential.""" - directory = Path(directory).resolve() - expected = publication_inventory( - directory, - {f["name"]: f["role"] for f in inventory["files"]}, - publication_id=inventory["publication_id"], - ) - if expected != inventory: - raise ValueError("Graph publication digest mismatch.") - id_ = inventory["publication_id"] - with self._sessions.begin() as session: - existing = session.get(GraphPublicationJob, id_) - if existing is not None: - if existing.inventory != inventory or existing.directory != str( - directory - ): - raise ValueError( - "Graph publication identity conflicts with a queued job." - ) - return - session.add( - GraphPublicationJob( - publication_id=id_, - inventory=inventory, - directory=str(directory), - status="pending", - attempts=0, - next_attempt_at=0, - lease_until=0, - receipt={ - "version": 1, - "publication_id": id_, - "status": "pending", - "error_code": None, - }, - ) - ) - - def claim( - self, - *, - lease_seconds: float = GRAPH_PUBLICATION_LEASE_SECONDS, - publication_id: str | None = None, - ) -> ClaimedGraphPublication | None: - """Reserve one pending job whose retry delay and previous lease expired. - - A conditional database update allows only one current lease. Workers - that exit leave their jobs eligible again after lease expiry; completion - uses the returned lease timestamp to reject outdated workers. - """ - now = time.time() - with self._sessions.begin() as session: - statement = select(GraphPublicationJob.publication_id) - if publication_id is not None: - statement = statement.where( - GraphPublicationJob.publication_id == publication_id - ) - id_ = session.scalar( - statement.where( - GraphPublicationJob.status == "pending", - GraphPublicationJob.next_attempt_at <= now, - GraphPublicationJob.lease_until <= now, - ) - # Earliest-due first, so a job that keeps failing cannot - # repeatedly displace other pending publications. - .order_by(GraphPublicationJob.next_attempt_at) - .limit(1) - ) - if id_ is None: - return None - claimed_id = session.scalar( - update(GraphPublicationJob) - .where( - GraphPublicationJob.publication_id == id_, - GraphPublicationJob.status == "pending", - GraphPublicationJob.lease_until <= now, - ) - .values(lease_until=now + lease_seconds) - .returning(GraphPublicationJob.publication_id) - ) - if claimed_id is None: - return None - job = session.get(GraphPublicationJob, id_) - assert job is not None, ( - "A claimed publication must exist in its transaction." - ) - return ClaimedGraphPublication( - publication_id=id_, - directory=Path(job.directory), - inventory=job.inventory, - lease_until=job.lease_until, - ) - - def receipt(self, id_: str) -> PublicationReceipt | None: - """Read the latest local result; old HF receipt snapshots remain unchanged.""" - with self._sessions() as session: - job = session.get(GraphPublicationJob, id_) - return None if job is None else job.receipt - - def finish( - self, - id_: str, - *, - error_code: str | None = None, - url: str | None = None, - expected_lease: float | None = None, - ) -> None: - with self._sessions.begin() as session: - statement = update(GraphPublicationJob).where( - GraphPublicationJob.publication_id == id_, - GraphPublicationJob.status == "pending", - ) - if expected_lease is not None: - statement = statement.where( - GraphPublicationJob.lease_until == expected_lease - ) - job = session.scalar( - statement.values( - attempts=GraphPublicationJob.attempts + 1, - status="published" if url else "pending", - lease_until=0, - ).returning(GraphPublicationJob) - ) - # Completion is conditional in SQL, not a read-then-write check: - # expired workers cannot release another lease or replace success. - if job is None: - return - job.next_attempt_at = time.time() + min(300, 2 ** min(job.attempts, 8)) - receipt: PublicationReceipt = { - "version": 1, - "publication_id": id_, - "status": job.status, - "error_code": error_code, - "url": url, - "attempts": job.attempts, - "inventory_sha256": inventory_digest(job.inventory), - } - job.receipt = receipt - directory = Path(job.directory) - # The queue row is authoritative. This separate latest-result file (not - # the build/HF receipt) is a convenience copy, written only after the - # commit: a deleted preserved directory must not roll back the result. - _write_status(directory, receipt) - - def retry(self, id_: str | None = None) -> None: - """Make preserved pending jobs eligible again, including missing credentials.""" - statement = update(GraphPublicationJob).where( - GraphPublicationJob.status == "pending" - ) - if id_ is not None: - statement = statement.where(GraphPublicationJob.publication_id == id_) - with self._sessions.begin() as session: - session.execute(statement.values(next_attempt_at=0)) - - -class GraphPublicationDelivery: - """Upload exact files with scoped signed URLs, then seal the publication.""" - - def __init__( - self, - queue: GraphPublicationQueue, - *, - credential: Callable[[], CollectorCredential], - origin: str, - client: httpx.Client | None = None, - invalidate_credential: Callable[[], None] | None = None, - ) -> None: - self.queue = queue - self.credential = credential - self.origin = validated_origin(origin, allow_loopback_http=True) - self.client = client or httpx.Client(timeout=60, follow_redirects=False) - self.invalidate_credential = invalidate_credential - - def flush_once(self, *, publication_id: str | None = None) -> bool: - """Try one job; failures retain it without changing telemetry eligibility.""" - job = self.queue.claim(publication_id=publication_id) - if job is None: - return False - id_, inventory, directory = ( - job.publication_id, - job.inventory, - job.directory, - ) - error = "publication_unavailable" - lease = job.lease_until - if not directory.is_dir(): - # Retained until an operator restores the files to this directory; - # the retry backoff lets other pending jobs proceed meanwhile. - self.queue.finish( - id_, error_code="preserved_files_missing", expected_lease=lease - ) - return False - try: - status, token = self.credential() - if token is None: - self.queue.finish( - id_, error_code=f"credential_{status}", expected_lease=lease - ) - return False - current = publication_inventory( - directory, - {f["name"]: f["role"] for f in inventory["files"]}, - publication_id=id_, - ) - if current != inventory: - raise ValueError("preserved_bytes_changed") - headers = {"Authorization": f"Bearer {token}"} - response = self.client.post( - self.origin + "/api/uploads/", json=inventory, headers=headers - ) - response.raise_for_status() - body = response.json() - uploads = body.get("uploads") - if not isinstance(uploads, list): - raise ValueError("invalid_upload_response") - files = {f["name"]: f for f in inventory["files"]} - for upload in uploads: - name = upload.get("name") - if name not in files: - raise ValueError("invalid_upload_response") - if upload.get("already_uploaded"): - continue - url = urlsplit(upload["url"]) - if ( - url.scheme != "https" - or not ( - (url.hostname == "vercel.com" and url.path == "/api/blob/") - or url.hostname == "blob.vercel-storage.com" - or (url.hostname or "").endswith(".blob.vercel-storage.com") - ) - or url.username - or url.password - ): - raise ValueError("invalid_upload_url") - with (directory / name).open("rb") as stream: - result = self.client.put( - upload["url"], - content=stream, - headers={ - "Content-Type": "application/json", - "Content-Length": str(files[name]["bytes"]), - }, - ) - result.raise_for_status() - response = self.client.post( - self.origin + "/api/publications/", json=inventory, headers=headers - ) - response.raise_for_status() - result = response.json() - if result.get("publication_id") != id_ or result.get( - "inventory_sha256" - ) != inventory_digest(inventory): - raise ValueError("invalid_publication_response") - self.queue.finish( - id_, url=f"{self.origin}/runs/{id_}/", expected_lease=lease - ) - return True - except httpx.HTTPStatusError as exception: - error = f"publication_http_{exception.response.status_code}" - if exception.response.status_code == 401 and self.invalidate_credential: - self.invalidate_credential() - except httpx.HTTPError: - error = "publication_network_error" - except (OSError, ValueError, KeyError, TypeError): - error = "publication_invalid_evidence_or_response" - self.queue.finish(id_, error_code=error, expected_lease=lease) - return False diff --git a/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/main.py b/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/main.py deleted file mode 100644 index 51d18fa742..0000000000 --- a/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/main.py +++ /dev/null @@ -1,66 +0,0 @@ -"""Command-line construction for the telemetry emitter service.""" - -from __future__ import annotations - -import argparse -import json -from pathlib import Path - -from microcosm.build.telemetry_emitter_service.collector import CollectorDelivery -from microcosm.build.telemetry_emitter_service.constants import ( - DEFAULT_HEARTBEAT_SECONDS, -) -from microcosm.build.telemetry_emitter_service.graph_publication import ( - DEFAULT_GRAPH_PUBLICATION_ORIGIN, - GraphPublicationDelivery, - GraphPublicationQueue, -) -from microcosm.build.telemetry_emitter_service.resources import ProcessTreeSampler -from microcosm.build.telemetry_emitter_service.runtime import EmitterService -from microcosm.build.telemetry_emitter_service.spool import EventSpool - - -def build_parser() -> argparse.ArgumentParser: - """Create the service command-line parser.""" - - parser = argparse.ArgumentParser() - parser.add_argument("--socket", type=Path, required=True) - parser.add_argument("--spool", type=Path, required=True) - parser.add_argument("--development-collector-url") - parser.add_argument("--registration-json", help="Omit for graph-only delivery.") - parser.add_argument("--parent-pid", type=int, required=True) - parser.add_argument( - "--heartbeat-seconds", - type=float, - default=DEFAULT_HEARTBEAT_SECONDS, - ) - return parser - - -def main(argv: list[str] | None = None) -> int: - """Run the telemetry emitter service until the build disconnects.""" - - args = build_parser().parse_args(argv) - registration = ( - json.loads(args.registration_json) if args.registration_json else None - ) - spool = EventSpool(args.spool) - delivery = CollectorDelivery( - spool, development_collector_url=args.development_collector_url - ) - service = EmitterService( - socket_path=args.socket, - registration=registration, - spool=spool, - delivery=delivery, - graph_delivery=GraphPublicationDelivery( - GraphPublicationQueue(args.spool), - credential=delivery.session.credential, - invalidate_credential=delivery.session.invalidate, - origin=DEFAULT_GRAPH_PUBLICATION_ORIGIN, - ), - sampler=ProcessTreeSampler(args.parent_pid), - heartbeat_seconds=args.heartbeat_seconds, - ) - service.run() - return 0 diff --git a/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/migrations.py b/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/migrations.py deleted file mode 100644 index 8446a5a2f8..0000000000 --- a/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/migrations.py +++ /dev/null @@ -1,65 +0,0 @@ -"""Programmatic Alembic migration entry points for the telemetry spool.""" - -from __future__ import annotations - -import fcntl -from collections.abc import Iterator -from contextlib import contextmanager -from importlib.resources import as_file, files -from pathlib import Path - -from alembic import command -from alembic.config import Config -from alembic.migration import MigrationContext -from alembic.script import ScriptDirectory -from sqlalchemy.engine import Connection, Engine - -from microcosm.build.telemetry_emitter_service.database import create_spool_engine - -_MIGRATION_TARGET = "head" - - -@contextmanager -def alembic_config( - *, - connection: Connection | None = None, -) -> Iterator[Config]: - """Yield Alembic configuration with migrations on the filesystem.""" - - migration_resources = files(__package__).joinpath("alembic") - with as_file(migration_resources) as migration_directory: - config = Config() - config.set_main_option("script_location", str(migration_directory)) - if connection is not None: - config.attributes["connection"] = connection - yield config - - -def upgrade_spool_database(engine: Engine) -> None: - """Apply every committed spool migration to an engine.""" - - # Concurrent build processes share this spool and must serialize upgrades. - path = Path(engine.url.database) - with path.with_suffix(".migration.lock").open("a") as lock: - fcntl.flock(lock, fcntl.LOCK_EX) - with engine.begin() as connection: - with alembic_config(connection=connection) as config: - command.upgrade(config, _MIGRATION_TARGET) - - -def current_database_revision(path: Path | str) -> str | None: - """Return the Alembic revision recorded by one spool database.""" - - engine = create_spool_engine(path) - try: - with engine.connect() as connection: - return MigrationContext.configure(connection).get_current_revision() - finally: - engine.dispose() - - -def migration_head_revision() -> str | None: - """Return the single current head from the packaged migration history.""" - - with alembic_config() as config: - return ScriptDirectory.from_config(config).get_current_head() diff --git a/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/models.py b/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/models.py deleted file mode 100644 index 610576a94f..0000000000 --- a/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/models.py +++ /dev/null @@ -1,152 +0,0 @@ -"""SQLAlchemy ORM models for the local telemetry spool.""" - -from __future__ import annotations - -import json -from typing import Any - -from sqlalchemy import ( - Float, - ForeignKeyConstraint, - Index, - Integer, - Text, - UniqueConstraint, -) -from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column, relationship -from sqlalchemy.types import TypeDecorator - -from microcosm.build.graph_publication_types import ( - GraphJobStatus, - PublicationInventory, - PublicationReceipt, -) -from microcosm.build.telemetry_emitter_service.constants import ( - UPLOAD_STATE_PENDING, -) - - -class JsonObjectText(TypeDecorator[dict[str, Any]]): - """Store JSON objects in the spool's existing text-column format.""" - - impl = Text - cache_ok = True - - def process_bind_param( - self, - value: dict[str, Any] | None, - dialect, - ) -> str | None: - if value is None: - return None - return _serialize_json_object(value) - - def process_result_value( - self, - value: str | None, - dialect, - ) -> dict[str, Any] | None: - if value is None: - return None - decoded = json.loads(value) - if not isinstance(decoded, dict): - raise ValueError("telemetry spool JSON must contain an object") - return decoded - - -def serialized_json_length(value: dict[str, Any]) -> int: - """Return the stored character count for one mapped JSON object.""" - - return len(_serialize_json_object(value)) - - -def _serialize_json_object(value: dict[str, Any]) -> str: - return json.dumps( - value, - separators=(",", ":"), - sort_keys=True, - allow_nan=False, - ) - - -class SpoolModel(DeclarativeBase): - """Declarative base for the local telemetry spool.""" - - -class TelemetryRunRecord(SpoolModel): - """One telemetry producer registered for one build run.""" - - __tablename__ = "telemetry_runs" - - run_id: Mapped[str] = mapped_column(Text, primary_key=True) - producer_id: Mapped[str] = mapped_column(Text, primary_key=True) - registration: Mapped[dict[str, Any]] = mapped_column( - "registration_json", - JsonObjectText, - nullable=False, - ) - next_sequence: Mapped[int] = mapped_column(Integer, nullable=False, default=1) - upload_state: Mapped[str] = mapped_column( - Text, - nullable=False, - default=UPLOAD_STATE_PENDING, - ) - local_only_reason: Mapped[str | None] = mapped_column(Text) - updated_at: Mapped[str] = mapped_column(Text, nullable=False) - events: Mapped[list[TelemetryEventRecord]] = relationship( - back_populates="run", - cascade="all, delete-orphan", - ) - - -class TelemetryEventRecord(SpoolModel): - """One ordered telemetry event awaiting collector acknowledgement.""" - - __tablename__ = "telemetry_events" - __table_args__ = ( - UniqueConstraint( - "run_id", - "producer_id", - "sequence", - name="uq_telemetry_events_run_producer_sequence", - ), - ForeignKeyConstraint( - ["run_id", "producer_id"], - ["telemetry_runs.run_id", "telemetry_runs.producer_id"], - name="fk_telemetry_events_run", - ), - Index( - "telemetry_events_run_sequence", - "run_id", - "producer_id", - "sequence", - ), - ) - - event_id: Mapped[str] = mapped_column(Text, primary_key=True) - run_id: Mapped[str] = mapped_column(Text, nullable=False) - producer_id: Mapped[str] = mapped_column(Text, nullable=False) - sequence: Mapped[int] = mapped_column(Integer, nullable=False) - payload: Mapped[dict[str, Any]] = mapped_column( - "payload_json", - JsonObjectText, - nullable=False, - ) - created_at: Mapped[str] = mapped_column(Text, nullable=False) - run: Mapped[TelemetryRunRecord] = relationship(back_populates="events") - - -class GraphPublicationJob(SpoolModel): - """Durable artifact delivery, independent of event retention and run IDs.""" - - __tablename__ = "graph_publication_jobs" - publication_id: Mapped[str] = mapped_column(Text, primary_key=True) - inventory: Mapped[PublicationInventory] = mapped_column( - JsonObjectText, nullable=False - ) - directory: Mapped[str] = mapped_column(Text, nullable=False) - status: Mapped[GraphJobStatus] = mapped_column(Text, nullable=False) - attempts: Mapped[int] = mapped_column(Integer, nullable=False) - next_attempt_at: Mapped[float] = mapped_column(Float, nullable=False) - lease_until: Mapped[float] = mapped_column(Float, nullable=False) - receipt: Mapped[PublicationReceipt] = mapped_column(JsonObjectText, nullable=False) diff --git a/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/resources.py b/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/resources.py deleted file mode 100644 index f8ede94ee7..0000000000 --- a/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/resources.py +++ /dev/null @@ -1,137 +0,0 @@ -"""Process-tree resource sampling for telemetry events.""" - -from __future__ import annotations - -import os -from pathlib import Path -from typing import Any - -try: - import psutil -except ModuleNotFoundError: # Base installs use the standard-library fallback. - psutil = None - - -def _empty_sample(peak_rss: int) -> dict[str, Any]: - return { - "cpu_user_seconds": 0.0, - "cpu_system_seconds": 0.0, - "rss_bytes": 0, - "peak_rss_bytes": peak_rss, - } - - -class ProcessTreeSampler: - """Collect cumulative CPU and resident memory for a build process tree.""" - - def __init__(self, parent_pid: int) -> None: - self.parent_pid = parent_pid - self._peak_rss = 0 - self._parent_create_time = self._create_time() - - def _create_time(self) -> float | str | None: - if psutil is not None: - try: - return psutil.Process(self.parent_pid).create_time() - except (psutil.Error, OSError): - return None - stat_path = Path(f"/proc/{self.parent_pid}/stat") - try: - return stat_path.read_text().rsplit(")", 1)[1].split()[19] - except (OSError, IndexError): - return None - - def parent_alive(self) -> bool: - """Return whether the sampled process still has its original identity.""" - - if psutil is not None: - try: - process = psutil.Process(self.parent_pid) - return ( - self._parent_create_time is not None - and process.create_time() == self._parent_create_time - and process.is_running() - and process.status() != psutil.STATUS_ZOMBIE - ) - except (psutil.Error, OSError): - return False - try: - os.kill(self.parent_pid, 0) - except OSError: - return False - current = self._create_time() - # /proc supplies a process-start tick that protects against PID reuse. - # On platforms without it, existence is the best standard-library test. - return self._parent_create_time is None or current == self._parent_create_time - - def sample(self) -> dict[str, Any]: - """Sample the parent and every currently visible child process.""" - - if psutil is None: - return self._fallback_sample() - processes = [] - try: - parent = psutil.Process(self.parent_pid) - processes = [parent, *parent.children(recursive=True)] - except (psutil.Error, OSError): - pass - user = 0.0 - system = 0.0 - rss = 0 - for process in processes: - try: - cpu = process.cpu_times() - user += float(cpu.user) + float(getattr(cpu, "children_user", 0.0)) - system += float(cpu.system) + float( - getattr(cpu, "children_system", 0.0) - ) - rss += int(process.memory_info().rss) - except (psutil.Error, OSError): - continue - self._peak_rss = max(self._peak_rss, rss) - return { - "cpu_user_seconds": user, - "cpu_system_seconds": system, - "rss_bytes": rss, - "peak_rss_bytes": self._peak_rss, - } - - def _fallback_sample(self) -> dict[str, Any]: - """Sample Linux process trees when the country engine is not installed.""" - - process_rows: dict[int, tuple[int, float, float, int]] = {} - try: - clock_ticks = float(os.sysconf("SC_CLK_TCK")) - page_size = int(os.sysconf("SC_PAGE_SIZE")) - except (AttributeError, OSError, ValueError): - return _empty_sample(self._peak_rss) - for stat_path in Path("/proc").glob("[0-9]*/stat"): - try: - pid = int(stat_path.parent.name) - fields = stat_path.read_text().rsplit(")", 1)[1].split() - process_rows[pid] = ( - int(fields[1]), - (int(fields[11]) + int(fields[13])) / clock_ticks, - (int(fields[12]) + int(fields[14])) / clock_ticks, - int(fields[21]) * page_size, - ) - except (OSError, ValueError, IndexError): - continue - selected = {self.parent_pid} - changed = True - while changed: - changed = False - for pid, (parent, *_rest) in process_rows.items(): - if parent in selected and pid not in selected: - selected.add(pid) - changed = True - user = sum(process_rows[pid][1] for pid in selected if pid in process_rows) - system = sum(process_rows[pid][2] for pid in selected if pid in process_rows) - rss = sum(process_rows[pid][3] for pid in selected if pid in process_rows) - self._peak_rss = max(self._peak_rss, rss) - return { - "cpu_user_seconds": user, - "cpu_system_seconds": system, - "rss_bytes": rss, - "peak_rss_bytes": self._peak_rss, - } diff --git a/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/runtime.py b/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/runtime.py deleted file mode 100644 index 3a95be8157..0000000000 --- a/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/runtime.py +++ /dev/null @@ -1,264 +0,0 @@ -"""Existing local service with independent telemetry and graph workers.""" - -from __future__ import annotations - -import json -import os -import socket -import sys -import threading -import time -from collections.abc import Mapping -from pathlib import Path -from typing import Any - -from microcosm.build.telemetry_emitter_service.collector import CollectorDelivery -from microcosm.build.telemetry_emitter_service.constants import ( - DEFAULT_DRAIN_SECONDS, - DRAIN_RETRY_SECONDS, - EVENT_OBJECT_ERROR, - FAILURE_CLASS_UNEXPECTED_PROCESS_EXIT, - LOCAL_MESSAGE_TOO_LARGE_ERROR, - MINIMUM_HEARTBEAT_SECONDS, - SOCKET_ACCEPT_TIMEOUT_SECONDS, - SOCKET_CONNECTION_TIMEOUT_SECONDS, - SOCKET_LISTEN_BACKLOG, - UNSUPPORTED_ACTION_ERROR, - WORKER_INTERVAL_SECONDS, -) -from microcosm.build.telemetry_emitter_service.graph_publication import ( - GRAPH_WORKER_ERROR_MESSAGE, - GraphPublicationDelivery, -) -from microcosm.build.telemetry_emitter_service.resources import ProcessTreeSampler -from microcosm.build.telemetry_emitter_service.spool import EventSpool -from microcosm.build.telemetry_emitter_service.timestamps import utc_now -from microcosm.build.telemetry_protocol import ( - ACTION_CLOSE, - ACTION_EVENT, - ACTION_PING, - EVENT_TYPE_HEARTBEAT, - EVENT_TYPE_RUN, - LOCAL_ACKNOWLEDGEMENT_ERROR, - LOCAL_ACKNOWLEDGEMENT_OK, - LOCAL_MESSAGE_DELIMITER, - LOCAL_SOCKET_READ_BYTES, - MAX_LOCAL_MESSAGE_BYTES, - STAGE_BLOCKED, - STAGE_COMPLETE, - STAGE_CREATED, - STAGE_FAILED, - STATUS_FAILED, - STATUS_PROGRESS, - UNEXPECTED_PROCESS_EXIT_MESSAGE, -) - - -def _heartbeat_event(stage_id: str) -> dict[str, Any]: - return { - "timestamp": utc_now(), - "event_type": EVENT_TYPE_HEARTBEAT, - "stage_id": stage_id, - "status": STATUS_PROGRESS, - "message": None, - "details": {}, - } - - -def _unexpected_exit_event(stage_id: str) -> dict[str, Any]: - return { - "timestamp": utc_now(), - "event_type": EVENT_TYPE_RUN, - "stage_id": STAGE_FAILED, - "status": STATUS_FAILED, - "message": UNEXPECTED_PROCESS_EXIT_MESSAGE, - "details": { - "failure_class": FAILURE_CLASS_UNEXPECTED_PROCESS_EXIT, - "failed_during": stage_id, - }, - } - - -class EmitterService: - """Private local socket server with a concurrent delivery worker.""" - - def __init__( - self, - *, - socket_path: Path, - registration: Mapping[str, Any] | None, - spool: EventSpool, - delivery: CollectorDelivery, - sampler: ProcessTreeSampler, - heartbeat_seconds: float, - drain_seconds: float = DEFAULT_DRAIN_SECONDS, - graph_delivery: GraphPublicationDelivery | None = None, - ) -> None: - self.socket_path = socket_path - self.registration = dict(registration) if registration is not None else None - self.graph_delivery = graph_delivery - self.spool = spool - self.delivery = delivery - self.sampler = sampler - self.heartbeat_seconds = max( - MINIMUM_HEARTBEAT_SECONDS, - heartbeat_seconds, - ) - self.drain_seconds = max(0.0, drain_seconds) - self._stop = threading.Event() - self._last_stage = STAGE_CREATED - - def run(self) -> None: - """Serve local messages until the client closes or exits.""" - - if self.registration is not None: - self.spool.register(self.registration) - self.socket_path.parent.mkdir(parents=True, exist_ok=True) - try: - self.socket_path.unlink(missing_ok=True) - with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as server: - server.bind(str(self.socket_path)) - os.chmod(self.socket_path, 0o600) - server.listen(SOCKET_LISTEN_BACKLOG) - server.settimeout(SOCKET_ACCEPT_TIMEOUT_SECONDS) - workers = [] - if self.registration is not None: - workers.append( - threading.Thread( - target=self._worker, - args=(self.registration,), - daemon=True, - ) - ) - if self.graph_delivery is not None: - workers.append( - threading.Thread( - target=self._graph_worker, - args=(self.graph_delivery,), - daemon=True, - ) - ) - for worker in workers: - worker.start() - self._serve(server) - deadline = ( - time.monotonic() + self.drain_seconds + WORKER_INTERVAL_SECONDS - ) - for worker in workers: - worker.join(timeout=max(0, deadline - time.monotonic())) - finally: - self._stop.set() - self.socket_path.unlink(missing_ok=True) - try: - self.socket_path.parent.rmdir() - except OSError: - pass - - def _serve(self, server: socket.socket) -> None: - while not self._stop.is_set(): - if not self.sampler.parent_alive(): - if self.registration is not None: - self.spool.append( - self.registration, - _unexpected_exit_event(self._last_stage), - resources=self.sampler.sample(), - ) - self._stop.set() - break - try: - connection, _ = server.accept() - except TimeoutError: - continue - with connection: - connection.settimeout(SOCKET_CONNECTION_TIMEOUT_SECONDS) - response = self._serve_connection(connection) - try: - connection.sendall(response) - except OSError: - pass - - def _serve_connection(self, connection: socket.socket) -> bytes: - try: - message = self._read_message(connection) - self._handle(message) - except Exception: - return LOCAL_ACKNOWLEDGEMENT_ERROR - return LOCAL_ACKNOWLEDGEMENT_OK - - @staticmethod - def _read_message(connection: socket.socket) -> dict[str, Any]: - chunks = bytearray() - while len(chunks) <= MAX_LOCAL_MESSAGE_BYTES: - data = connection.recv(LOCAL_SOCKET_READ_BYTES) - if not data: - break - chunks.extend(data) - if LOCAL_MESSAGE_DELIMITER in data: - break - if len(chunks) > MAX_LOCAL_MESSAGE_BYTES: - raise ValueError(LOCAL_MESSAGE_TOO_LARGE_ERROR) - return json.loads(bytes(chunks).split(LOCAL_MESSAGE_DELIMITER, 1)[0]) - - def _handle(self, message: Mapping[str, Any]) -> None: - action = message.get("action") - if action == ACTION_EVENT and self.registration is not None: - event = message.get("event") - if not isinstance(event, Mapping): - raise ValueError(EVENT_OBJECT_ERROR) - stage_id = event.get("stage_id") - if isinstance(stage_id, str) and stage_id not in { - STAGE_BLOCKED, - STAGE_COMPLETE, - STAGE_FAILED, - }: - self._last_stage = stage_id - self.spool.append( - self.registration, - event, - resources=self.sampler.sample(), - ) - elif action == ACTION_CLOSE: - self._stop.set() - elif action == ACTION_PING: - return - else: - raise ValueError(UNSUPPORTED_ACTION_ERROR) - - def _worker(self, registration: Mapping[str, Any]) -> None: - next_heartbeat = time.monotonic() + self.heartbeat_seconds - while not self._stop.wait(WORKER_INTERVAL_SECONDS): - now = time.monotonic() - # Sample every worker iteration so short-lived build children are - # much less likely to disappear between stage and heartbeat events. - self.sampler.sample() - if now >= next_heartbeat: - self.spool.append( - registration, - _heartbeat_event(self._last_stage), - resources=self.sampler.sample(), - ) - next_heartbeat = now + self.heartbeat_seconds - self.delivery.flush_once() - self._drain() - - def _graph_worker(self, delivery: GraphPublicationDelivery) -> None: - # Uploads may take much longer than a heartbeat. They never run on the - # telemetry worker or the local socket thread. Pending jobs survive exit. - warned = False - while not self._stop.wait(WORKER_INTERVAL_SECONDS): - try: - delivery.flush_once() - except Exception: - # A database or filesystem error leaves the job leased for a - # later retry; it must not end delivery of every other job. - if not warned: - print(GRAPH_WORKER_ERROR_MESSAGE, file=sys.stderr, flush=True) - warned = True - - def _drain(self) -> None: - deadline = time.monotonic() + self.drain_seconds - while self.spool.has_deliverable() and time.monotonic() < deadline: - if not self.delivery.flush_once(): - remaining = deadline - time.monotonic() - if remaining > 0: - time.sleep(min(DRAIN_RETRY_SECONDS, remaining)) diff --git a/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/spool.py b/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/spool.py deleted file mode 100644 index 1ef43c5269..0000000000 --- a/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/spool.py +++ /dev/null @@ -1,258 +0,0 @@ -"""Durable local queue for telemetry events.""" - -from __future__ import annotations - -import threading -import time -import uuid -from collections.abc import Mapping -from datetime import UTC, datetime, timedelta -from pathlib import Path -from typing import Any - -from sqlalchemy import func, select -from sqlalchemy.orm import Session - -from microcosm.build.telemetry_emitter_service.constants import ( - BATCH_SIZE, - MAX_QUEUED_BYTES, - PRUNE_INTERVAL_SECONDS, - RETENTION_DAYS, - UPLOAD_STATE_LOCAL_ONLY, - UPLOAD_STATE_PENDING, -) -from microcosm.build.telemetry_emitter_service.database import ( - create_spool_engine, - create_spool_session_factory, -) -from microcosm.build.telemetry_emitter_service.migrations import ( - upgrade_spool_database, -) -from microcosm.build.telemetry_emitter_service.models import ( - TelemetryEventRecord, - TelemetryRunRecord, - serialized_json_length, -) -from microcosm.build.telemetry_emitter_service.timestamps import utc_now -from microcosm.build.telemetry_protocol import TELEMETRY_SCHEMA_VERSION - - -class EventSpool: - """Small SQLite queue shared by successive emitter service processes.""" - - def __init__(self, path: Path | str) -> None: - self.path = Path(path) - self.path.parent.mkdir(parents=True, exist_ok=True) - try: - self.path.parent.chmod(0o700) - except OSError: - pass - self._engine = create_spool_engine(self.path) - upgrade_spool_database(self._engine) - self._session_factory = create_spool_session_factory(self._engine) - self._lock = threading.RLock() - self._last_prune_at = 0.0 - self.prune() - - def register(self, registration: Mapping[str, Any]) -> None: - """Create or refresh a producer registration.""" - - run_id = str(registration["run_id"]) - producer_id = str(registration["producer_id"]) - with self._lock, self._session_factory.begin() as session: - existing = session.get(TelemetryRunRecord, (run_id, producer_id)) - if existing is not None: - if existing.registration.get("producer_id") != registration.get( - "producer_id" - ): - raise ValueError(f"run_id {run_id!r} already has another producer") - existing.registration = dict(registration) - existing.updated_at = utc_now() - return - session.add( - TelemetryRunRecord( - run_id=run_id, - producer_id=producer_id, - registration=dict(registration), - next_sequence=1, - upload_state=UPLOAD_STATE_PENDING, - local_only_reason=None, - updated_at=utc_now(), - ) - ) - - def append( - self, - registration: Mapping[str, Any], - event: Mapping[str, Any], - *, - resources: Mapping[str, Any] | None = None, - ) -> dict[str, Any]: - """Append an event and assign its stable producer sequence.""" - - run_id = str(registration["run_id"]) - producer_id = str(registration["producer_id"]) - with self._lock, self._session_factory.begin() as session: - run = session.get(TelemetryRunRecord, (run_id, producer_id)) - if run is None: - raise KeyError(run_id) - sequence = run.next_sequence - event_id = uuid.uuid4().hex - payload = { - "schema_version": TELEMETRY_SCHEMA_VERSION, - "event_id": event_id, - "run_id": run_id, - "producer_id": producer_id, - "sequence": sequence, - "timestamp": event.get("timestamp") or utc_now(), - "event_type": event["event_type"], - "stage_id": event.get("stage_id"), - "status": event["status"], - "message": event.get("message"), - "details": event.get("details") or {}, - "resources": resources, - } - session.add( - TelemetryEventRecord( - event_id=event_id, - run_id=run_id, - producer_id=producer_id, - sequence=sequence, - payload=payload, - created_at=utc_now(), - ) - ) - run.next_sequence = sequence + 1 - run.updated_at = utc_now() - if time.monotonic() - self._last_prune_at >= PRUNE_INTERVAL_SECONDS: - self.prune() - return payload - - def pending_runs(self) -> list[dict[str, Any]]: - """Return registrations that have events eligible for delivery.""" - - statement = ( - select(TelemetryRunRecord) - .join(TelemetryRunRecord.events) - .where(TelemetryRunRecord.upload_state == UPLOAD_STATE_PENDING) - .order_by(TelemetryRunRecord.updated_at) - .distinct() - ) - with self._lock, self._session_factory() as session: - runs = session.scalars(statement).all() - return [dict(run.registration) for run in runs] - - def make_local_only( - self, - run_id: str, - producer_id: str, - reason: str, - ) -> None: - """Permanently exclude one producer's queued events from upload.""" - - with self._lock, self._session_factory.begin() as session: - run = session.get(TelemetryRunRecord, (run_id, producer_id)) - if run is None: - return - run.upload_state = UPLOAD_STATE_LOCAL_ONLY - run.local_only_reason = reason - run.updated_at = utc_now() - - def has_deliverable(self) -> bool: - """Return whether any queued event remains eligible for delivery.""" - - statement = ( - select(TelemetryEventRecord) - .join(TelemetryEventRecord.run) - .where(TelemetryRunRecord.upload_state == UPLOAD_STATE_PENDING) - .limit(1) - ) - with self._lock, self._session_factory() as session: - return session.scalar(statement) is not None - - def batch( - self, - run_id: str, - producer_id: str, - limit: int = BATCH_SIZE, - ) -> list[dict[str, Any]]: - """Return the next ordered batch for one producer.""" - - statement = ( - select(TelemetryEventRecord) - .where( - TelemetryEventRecord.run_id == run_id, - TelemetryEventRecord.producer_id == producer_id, - ) - .order_by(TelemetryEventRecord.sequence) - .limit(limit) - ) - with self._lock, self._session_factory() as session: - events = session.scalars(statement).all() - return [dict(event.payload) for event in events] - - def acknowledge(self, event_ids: list[str]) -> None: - """Remove events acknowledged by the collector.""" - - if not event_ids: - return - statement = select(TelemetryEventRecord).where( - TelemetryEventRecord.event_id.in_(event_ids) - ) - with self._lock, self._session_factory.begin() as session: - for event in session.scalars(statement): - session.delete(event) - - def has_pending(self) -> bool: - """Return whether any event remains in local storage.""" - - statement = select(TelemetryEventRecord).limit(1) - with self._lock, self._session_factory() as session: - return session.scalar(statement) is not None - - def prune(self) -> None: - """Enforce the age and total-size retention limits.""" - - cutoff = (datetime.now(UTC) - timedelta(days=RETENTION_DAYS)).isoformat() - with self._lock, self._session_factory.begin() as session: - expired_events = session.scalars( - select(TelemetryEventRecord).where( - TelemetryEventRecord.created_at < cutoff - ) - ) - for event in expired_events: - session.delete(event) - session.flush() - stored_characters = session.scalar( - select( - func.coalesce( - func.sum(func.length(TelemetryEventRecord.payload)), - 0, - ) - ) - ) - excess = int(stored_characters or 0) - MAX_QUEUED_BYTES - if excess > 0: - self._remove_oldest_bytes(session, excess) - expired_runs = session.scalars( - select(TelemetryRunRecord).where( - TelemetryRunRecord.updated_at < cutoff, - ~TelemetryRunRecord.events.any(), - ) - ) - for run in expired_runs: - session.delete(run) - self._last_prune_at = time.monotonic() - - @staticmethod - def _remove_oldest_bytes(session: Session, excess: int) -> None: - statement = select(TelemetryEventRecord).order_by( - TelemetryEventRecord.created_at, - TelemetryEventRecord.sequence, - ) - removed = 0 - for event in session.scalars(statement): - session.delete(event) - removed += serialized_json_length(event.payload) - if removed >= excess: - break diff --git a/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/timestamps.py b/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/timestamps.py deleted file mode 100644 index 3f1f39a911..0000000000 --- a/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/timestamps.py +++ /dev/null @@ -1,11 +0,0 @@ -"""Timestamp helpers shared by the telemetry emitter service.""" - -from __future__ import annotations - -from datetime import UTC, datetime - - -def utc_now() -> str: - """Return the current UTC timestamp in ISO 8601 format.""" - - return datetime.now(UTC).isoformat() diff --git a/packages/microcosm-build/src/microcosm/build/telemetry_protocol.py b/packages/microcosm-build/src/microcosm/build/telemetry_protocol.py deleted file mode 100644 index cb27eb4e5c..0000000000 --- a/packages/microcosm-build/src/microcosm/build/telemetry_protocol.py +++ /dev/null @@ -1,70 +0,0 @@ -"""Shared constants for local and hosted Microcosm telemetry messages.""" - -from __future__ import annotations - -from types import MappingProxyType -from typing import Final, Literal - -type TelemetryAction = Literal["event", "close", "ping"] -type TelemetryEventType = Literal[ - "run", "stage", "progress", "calibration", "heartbeat" -] -type TelemetryStatus = Literal["started", "progress", "completed", "failed", "blocked"] - -TELEMETRY_SCHEMA_VERSION: Final = 1 - -ACTION_EVENT: Final = "event" -ACTION_CLOSE: Final = "close" -ACTION_PING: Final = "ping" - -EVENT_TYPE_RUN: Final = "run" -EVENT_TYPE_STAGE: Final = "stage" -EVENT_TYPE_PROGRESS: Final = "progress" -EVENT_TYPE_CALIBRATION: Final = "calibration" -EVENT_TYPE_HEARTBEAT: Final = "heartbeat" - -STATUS_STARTED: Final = "started" -STATUS_PROGRESS: Final = "progress" -STATUS_COMPLETED: Final = "completed" -STATUS_FAILED: Final = "failed" -#: The run reached a gate decision and the gates refused its candidate. -STATUS_BLOCKED: Final = "blocked" - -STAGE_CREATED: Final = "created" -STAGE_CALIBRATING: Final = "calibrating" -STAGE_COMPLETE: Final = "complete" -STAGE_FAILED: Final = "failed" -STAGE_BLOCKED: Final = "blocked" - -CALIBRATION_EVENT_KIND: Final = "calibration_epoch" -BUILD_STARTED_MESSAGE: Final = "Microcosm build started." -BUILD_COMPLETED_MESSAGE: Final = "Microcosm build completed." -BUILD_BLOCKED_MESSAGE: Final = "The gates refused the candidate at {phase}." -UNEXPECTED_PROCESS_EXIT_MESSAGE: Final = ( - "The build process exited without reporting completion." -) - -LOCAL_ACKNOWLEDGEMENT_OK: Final = b"ok\n" -LOCAL_ACKNOWLEDGEMENT_ERROR: Final = b"error\n" -LOCAL_MESSAGE_DELIMITER: Final = b"\n" -LOCAL_PING_MESSAGE: Final = b'{"action":"ping"}\n' - -MAX_TELEMETRY_TEXT_CHARS: Final = 2_000 -MAX_TELEMETRY_MESSAGE_CHARS: Final = 500 -MAX_TELEMETRY_DETAILS_DEPTH: Final = 6 -MAX_TELEMETRY_COLLECTION_ITEMS: Final = 200 -MAX_TELEMETRY_DETAILS_BYTES: Final = 8_192 -MAX_LOCAL_MESSAGE_BYTES: Final = 1_048_576 -LOCAL_SOCKET_READ_BYTES: Final = 65_536 -LOCAL_ACKNOWLEDGEMENT_READ_BYTES: Final = 16 - -SEQUENTIAL_STATUS_MAP: Final = MappingProxyType( - { - "failed": STATUS_FAILED, - "completed": STATUS_COMPLETED, - "passed": STATUS_COMPLETED, - "progress": STATUS_PROGRESS, - "started": STATUS_STARTED, - "running": STATUS_STARTED, - } -) diff --git a/packages/microcosm-build/src/microcosm/build/telemetry_sanitization.py b/packages/microcosm-build/src/microcosm/build/telemetry_sanitization.py deleted file mode 100644 index cabd12f338..0000000000 --- a/packages/microcosm-build/src/microcosm/build/telemetry_sanitization.py +++ /dev/null @@ -1,94 +0,0 @@ -"""Size bounds and credential redaction for telemetry payloads.""" - -from __future__ import annotations - -import json -import re -from collections.abc import Mapping -from pathlib import Path -from typing import Any, Final - -from microcosm.build.telemetry_protocol import ( - MAX_TELEMETRY_COLLECTION_ITEMS, - MAX_TELEMETRY_DETAILS_BYTES, - MAX_TELEMETRY_DETAILS_DEPTH, - MAX_TELEMETRY_TEXT_CHARS, -) - -_SENSITIVE_KEY_PARTS: Final = ( - "authorization", - "credential", - "password", - "secret", - "token", - "traceback", -) -_SECRET_TEXT: Final = re.compile( - r"(?i)(?:bearer\s+[^\s]+|(?:token|secret|password|credential)\s*[:=]\s*[^\s]+|hf_[A-Za-z0-9_-]{8,})" -) -_REDACTED_VALUE: Final = "[redacted]" -_MAXIMUM_DEPTH_VALUE: Final = "[maximum depth]" -_DETAILS_TRUNCATED_KEY: Final = "telemetry_details_truncated" - - -def sanitize_text(value: str, *, limit: int = MAX_TELEMETRY_TEXT_CHARS) -> str: - """Redact credential-like text and apply a character limit.""" - - return _SECRET_TEXT.sub(_REDACTED_VALUE, value)[:limit] - - -def sanitize_json(value: Any, *, depth: int = 0) -> Any: - """Return bounded JSON-compatible telemetry without model objects.""" - - if depth >= MAX_TELEMETRY_DETAILS_DEPTH: - return _MAXIMUM_DEPTH_VALUE - if isinstance(value, Mapping): - result = {} - for key, item in list(value.items())[:MAX_TELEMETRY_COLLECTION_ITEMS]: - name = str(key) - if any(part in name.lower() for part in _SENSITIVE_KEY_PARTS): - result[name] = _REDACTED_VALUE - else: - result[name] = sanitize_json(item, depth=depth + 1) - return result - if isinstance(value, (list, tuple)): - return [ - sanitize_json(item, depth=depth + 1) - for item in value[:MAX_TELEMETRY_COLLECTION_ITEMS] - ] - if isinstance(value, Path): - return str(value) - if isinstance(value, float): - return value if value == value and abs(value) != float("inf") else None - if isinstance(value, str): - return sanitize_text(value) - if isinstance(value, (int, bool)) or value is None: - return value - item = getattr(value, "item", None) - if callable(item): - return sanitize_json(item()) - return sanitize_text(str(value)) - - -def sanitize_details(value: Mapping[str, Any]) -> dict[str, Any]: - """Return a redacted details object within the wire-size limit.""" - - sanitized = sanitize_json(value) - assert isinstance(sanitized, dict) - if ( - len(json.dumps(sanitized, separators=(",", ":")).encode()) - <= MAX_TELEMETRY_DETAILS_BYTES - ): - return sanitized - compact: dict[str, Any] = {_DETAILS_TRUNCATED_KEY: True} - for key, item in sanitized.items(): - if not (isinstance(item, (str, int, float, bool)) or item is None): - continue - candidate = {**compact, key: item} - if ( - len(json.dumps(candidate, separators=(",", ":")).encode()) - > MAX_TELEMETRY_DETAILS_BYTES - ): - break - compact[key] = item - return compact diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/full_build_cli.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/full_build_cli.py index 69cdfef272..78e3107ab1 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/full_build_cli.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/full_build_cli.py @@ -1293,14 +1293,12 @@ def _publish_failure_evidence( temporary JSON are never published. """ from microcosm.build.artifact_files import publish_staged_bundle - from microcosm.build.telemetry_emitter_service.graph_publication import ( - GRAPH_PUBLICATION_WAIT_SECONDS, - ) from .orrery_publication import ( EVIDENCE_MANIFEST_NAME, FAILURE_BUNDLE_DIRECTORY, FAILURE_MARKER_NAME, + GRAPH_PUBLICATION_WAIT_SECONDS, finalize_graph, stage_graph_evidence, ) diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/orrery_publication.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/orrery_publication.py index 81422e9dc7..97a2f7480b 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/orrery_publication.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/orrery_publication.py @@ -15,18 +15,14 @@ from pathlib import Path from typing import Any, TypedDict -from microcosm.build.artifact_files import file_artifact, materialize_bytes -from microcosm.build.graph_publication_types import ( +from microcosm_provider_orrery.client import publication_inventory, publish_graph +from microcosm_provider_orrery.contracts import ( PublicationFileRole, PublicationReceipt, ) + +from microcosm.build.artifact_files import file_artifact, materialize_bytes from microcosm.build.staging_dataset import StagedDatasetDelivery -from microcosm.build.telemetry_emitter import default_spool_path -from microcosm.build.telemetry_emitter_service.graph_publication import ( - GRAPH_PUBLICATION_WAIT_SECONDS, - GraphPublicationQueue, - publication_inventory, -) from microcosm.graph import ContentStore from microcosm.graph.canonical import canonical_json from microcosm.graph.evidence import collect_execution_evidence, load_run_evidence @@ -37,6 +33,8 @@ EVIDENCE_MANIFEST_NAME = "orrery.evidence-manifest.json" FAILURE_MARKER_NAME = "orrery.failure.json" FAILURE_BUNDLE_DIRECTORY = "orrery-failure" +# How long a build waits for a first upload result after the durable enqueue. +GRAPH_PUBLICATION_WAIT_SECONDS = 30.0 class GraphEvidenceOutput(TypedDict): @@ -173,13 +171,7 @@ def finalize_graph( if emitter is None: # Durable enqueue is still possible without a running # emitter; the retry command requires no telemetry run. - queue = GraphPublicationQueue(default_spool_path()) - queue.enqueue(preserved, inventory) - queued_receipt = queue.receipt(inventory["publication_id"]) - assert queued_receipt is not None, ( - "A queued graph publication must have a receipt." - ) - receipt.update(queued_receipt) + receipt.update(publish_graph(preserved, inventory, wait_seconds=0)) else: receipt.update( emitter.publish_graph( diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/worker_identity.py b/packages/microcosm-build/src/microcosm/build/us_runtime/worker_identity.py index 2d5aff5b0b..2124fb6a40 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/worker_identity.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/worker_identity.py @@ -28,7 +28,7 @@ PRIMARY_QRF_WORKER_MODULE = "microcosm.build.us_runtime.puf_qrf_worker" PRIMARY_QRF_INTERPRETER_PLACEHOLDER = "{python_interpreter}" APPROVED_UV_LOCK_SHA256 = ( - "d73966a4020231e1b6d52bb5d69d852bbfc95827519dd4c98fc5c71ac8b20e19" + "95346afaf219870b7428fea331f74533dfbc2eb7e9549d654fa968a39e023394" ) LEGACY_CAMPAIGN_UV_LOCK_SHA256 = ( "27f47e385cfa35e2644a37410d1804b361ad9aee123577551c8421547bda65ee" diff --git a/packages/microcosm-build/tests/engine_free/shared/test_graph_publication.py b/packages/microcosm-build/tests/engine_free/shared/test_graph_publication.py deleted file mode 100644 index f3997d9451..0000000000 --- a/packages/microcosm-build/tests/engine_free/shared/test_graph_publication.py +++ /dev/null @@ -1,358 +0,0 @@ -"""Durable graph jobs use fake HTTP, independently of the visualization app.""" - -import json -from dataclasses import FrozenInstanceError - -import httpx -import pytest - -from microcosm.build.graph_publication_types import ClaimedGraphPublication -from microcosm.build.telemetry_emitter_service.graph_publication import ( - GraphPublicationDelivery, - GraphPublicationQueue, - inventory_digest, - publication_inventory, -) - - -def source(tmp_path): - directory = tmp_path / "source" - directory.mkdir() - (directory / "graph.orrery.json").write_text('{"fixture":true}') - return directory, publication_inventory(directory, {"graph.orrery.json": "graph"}) - - -def test_jobs_survive_restart_without_run_registration_or_event_retention(tmp_path): - directory, inventory = source(tmp_path) - queue = GraphPublicationQueue(tmp_path / "jobs.sqlite3") - queue.enqueue(directory, inventory) - restarted = GraphPublicationQueue(queue.path) - job = restarted.claim() - assert isinstance(job, ClaimedGraphPublication) - assert job.inventory == inventory - assert "run_id" not in inventory - assert job.publication_id == inventory["publication_id"] - assert job.directory == directory.resolve() - assert restarted.claim() is None - - -def test_claim_is_a_frozen_record_and_expired_leases_can_be_reclaimed( - tmp_path, monkeypatch -): - from microcosm.build.telemetry_emitter_service.graph_publication import time - - directory, inventory = source(tmp_path) - queue = GraphPublicationQueue(tmp_path / "jobs.sqlite3") - queue.enqueue(directory, inventory) - clock = [1000.0] - monkeypatch.setattr(time, "time", lambda: clock[0]) - claimed = queue.claim(lease_seconds=5) - assert isinstance(claimed, ClaimedGraphPublication) - assert claimed.lease_until == 1005.0 - with pytest.raises(FrozenInstanceError): - claimed.lease_until = 0 - assert GraphPublicationQueue(queue.path).claim() is None - clock[0] = 1005.0 - replacement = GraphPublicationQueue(queue.path).claim() - assert replacement.publication_id == claimed.publication_id - assert replacement.lease_until > claimed.lease_until - - -def test_claim_respects_retry_delay_and_published_jobs_are_not_reclaimed( - tmp_path, monkeypatch -): - from microcosm.build.telemetry_emitter_service.graph_publication import time - - directory, inventory = source(tmp_path) - queue = GraphPublicationQueue(tmp_path / "jobs.sqlite3") - queue.enqueue(directory, inventory) - clock = [1000.0] - monkeypatch.setattr(time, "time", lambda: clock[0]) - claimed = queue.claim() - queue.finish( - claimed.publication_id, - error_code="publication_network_error", - expected_lease=claimed.lease_until, - ) - assert queue.claim() is None - clock[0] = 1002.0 - retried = queue.claim() - assert retried.publication_id == claimed.publication_id - queue.finish( - retried.publication_id, - url="https://runs.example/runs/published/", - expected_lease=retried.lease_until, - ) - assert queue.claim() is None - - -def test_queue_rejects_changed_bytes_and_conflicting_inventory(tmp_path): - directory, inventory = source(tmp_path) - queue = GraphPublicationQueue(tmp_path / "jobs.sqlite3") - (directory / "graph.orrery.json").write_text("changed") - with pytest.raises(ValueError, match="digest"): - queue.enqueue(directory, inventory) - with pytest.raises(ValueError): - publication_inventory(directory, {"private.h5": "graph"}) - - -def test_delivery_retries_missing_credentials_and_network_failure(tmp_path): - directory, inventory = source(tmp_path) - queue = GraphPublicationQueue(tmp_path / "jobs.sqlite3") - queue.enqueue(directory, inventory) - delivery = GraphPublicationDelivery( - queue, credential=lambda: ("missing", None), origin="https://runs.example" - ) - delivery.flush_once() - assert queue.receipt(inventory["publication_id"])["status"] == "pending" - queue.retry(inventory["publication_id"]) - client = httpx.Client( - transport=httpx.MockTransport( - lambda request: (_ for _ in ()).throw( - httpx.ConnectError("secret", request=request) - ) - ) - ) - GraphPublicationDelivery( - queue, - credential=lambda: ("ok", "session"), - origin="https://runs.example", - client=client, - ).flush_once() - receipt = queue.receipt(inventory["publication_id"]) - assert receipt["status"] == "pending" - assert "secret" not in json.dumps(receipt) - - -@pytest.mark.parametrize( - "upload_url", - [ - "https://store.blob.vercel-storage.com/upload?signed=credential", - "https://vercel.com/api/blob/?pathname=object&signed=credential", - ], -) -def test_delivery_uses_exact_bytes_and_does_not_forward_session_to_signed_upload( - tmp_path, - upload_url, -): - directory, inventory = source(tmp_path) - queue = GraphPublicationQueue(tmp_path / "jobs.sqlite3") - queue.enqueue(directory, inventory) - requests = [] - - def respond(request): - requests.append(request) - if request.url.path == "/api/uploads/": - return httpx.Response( - 200, - json={ - "uploads": [ - { - "name": "graph.orrery.json", - "url": upload_url, - "already_uploaded": False, - } - ] - }, - ) - if request.method == "PUT": - assert "authorization" not in request.headers - assert request.content == (directory / "graph.orrery.json").read_bytes() - return httpx.Response(200, json={}) - return httpx.Response( - 200, - json={ - "publication_id": inventory["publication_id"], - "inventory_sha256": inventory_digest(inventory), - }, - ) - - delivery = GraphPublicationDelivery( - queue, - credential=lambda: ("ok", "session"), - origin="https://runs.example", - client=httpx.Client(transport=httpx.MockTransport(respond)), - ) - delivery.flush_once() - receipt = queue.receipt(inventory["publication_id"]) - assert receipt["status"] == "published" - assert len(requests) == 3 - assert "signed=" not in json.dumps(receipt) - assert "session" not in json.dumps(receipt) - - -def test_queue_resumes_after_worker_exits_and_keeps_pending_jobs(tmp_path): - directory, inventory = source(tmp_path) - queue = GraphPublicationQueue(tmp_path / "jobs.sqlite3") - queue.enqueue(directory, inventory) - queue.claim(lease_seconds=0) - assert GraphPublicationQueue(queue.path).claim() is not None - - -def test_late_failed_attempt_cannot_replace_a_published_receipt(tmp_path): - directory, inventory = source(tmp_path) - queue = GraphPublicationQueue(tmp_path / "jobs.sqlite3") - queue.enqueue(directory, inventory) - queue.claim(lease_seconds=0) - replacement = GraphPublicationQueue(queue.path) - replacement.claim() - id_ = inventory["publication_id"] - replacement.finish(id_, url=f"https://runs.example/runs/{id_}/") - published = queue.receipt(id_) - queue.finish(id_, error_code="publication_network_error") - assert queue.receipt(id_) == published - assert json.loads((directory / "publication.status.json").read_bytes()) == published - - -def test_expired_worker_cannot_release_a_replacement_workers_lease(tmp_path): - directory, inventory = source(tmp_path) - queue = GraphPublicationQueue(tmp_path / "jobs.sqlite3") - queue.enqueue(directory, inventory) - expired = queue.claim(lease_seconds=0) - replacement = GraphPublicationQueue(queue.path) - current = replacement.claim() - id_ = inventory["publication_id"] - initial = queue.receipt(id_) - queue.finish( - id_, - error_code="publication_network_error", - expected_lease=expired.lease_until, - ) - assert queue.receipt(id_) == initial - assert queue.claim() is None - replacement.finish( - id_, - url=f"https://runs.example/runs/{id_}/", - expected_lease=current.lease_until, - ) - assert queue.receipt(id_)["status"] == "published" - - -def test_telemetry_pruning_cannot_remove_graph_jobs(tmp_path): - from microcosm.build.telemetry_emitter_service.spool import EventSpool - - directory, inventory = source(tmp_path) - queue = GraphPublicationQueue(tmp_path / "jobs.sqlite3") - queue.enqueue(directory, inventory) - EventSpool(queue.path).prune() - assert ( - GraphPublicationQueue(queue.path).receipt(inventory["publication_id"])["status"] - == "pending" - ) - - -def test_rejected_graph_session_is_refreshed_without_discarding_the_job(tmp_path): - directory, inventory = source(tmp_path) - queue = GraphPublicationQueue(tmp_path / "jobs.sqlite3") - queue.enqueue(directory, inventory) - invalidations = [] - delivery = GraphPublicationDelivery( - queue, - credential=lambda: ("ok", "expired"), - invalidate_credential=lambda: invalidations.append(True), - origin="https://runs.example", - client=httpx.Client( - transport=httpx.MockTransport(lambda request: httpx.Response(401)) - ), - ) - assert not delivery.flush_once() - assert invalidations == [True] - assert queue.receipt(inventory["publication_id"])["status"] == "pending" - - -def test_retry_cli_reuses_identity_and_leaves_hf_snapshot_unchanged( - tmp_path, monkeypatch -): - from microcosm.build import graph_publication_cli as cli - - directory, inventory = source(tmp_path) - (directory / "orrery.upload.json").write_text(json.dumps(inventory)) - snapshot = directory / "orrery.publication.json" - snapshot.write_text('{"status":"pending"}') - calls = [] - - class Delivery: - def __init__(self, queue, **kwargs): - self.queue = queue - - def flush_once(self, *, publication_id): - calls.append(publication_id) - self.queue.finish( - publication_id, url="https://runs.example/runs/published/" - ) - - monkeypatch.setattr(cli, "GraphPublicationDelivery", Delivery) - assert ( - cli.main( - [ - "--spool", - str(tmp_path / "jobs.sqlite3"), - "--publication-id", - inventory["publication_id"], - "--directory", - str(directory), - ] - ) - == 0 - ) - assert calls == [inventory["publication_id"]] - assert snapshot.read_text() == '{"status":"pending"}' - - -def test_deleted_preserved_directory_is_retained_without_stopping_delivery( - tmp_path, monkeypatch -): - import shutil - - from microcosm.build.telemetry_emitter_service.graph_publication import time - - clock = [1000.0] - monkeypatch.setattr(time, "time", lambda: clock[0]) - queue = GraphPublicationQueue(tmp_path / "jobs.sqlite3") - (tmp_path / "removed").mkdir() - (tmp_path / "kept").mkdir() - removed, removed_inventory = source(tmp_path / "removed") - queue.enqueue(removed, removed_inventory) - clock[0] = 1001.0 - kept, kept_inventory = source(tmp_path / "kept") - queue.enqueue(kept, kept_inventory) - shutil.rmtree(removed) - delivery = GraphPublicationDelivery( - queue, credential=lambda: ("missing", None), origin="https://runs.example" - ) - - assert delivery.flush_once() is False - receipt = queue.receipt(removed_inventory["publication_id"]) - assert receipt["status"] == "pending" - assert receipt["error_code"] == "preserved_files_missing" - assert receipt["attempts"] == 1 - # The failed job backs off, so the next claim reaches the other job. - assert queue.claim().publication_id == kept_inventory["publication_id"] - - -def test_status_file_failure_does_not_roll_back_the_queue_result(tmp_path): - directory, inventory = source(tmp_path) - queue = GraphPublicationQueue(tmp_path / "jobs.sqlite3") - queue.enqueue(directory, inventory) - claimed = queue.claim() - (directory / "publication.status.json").mkdir() - queue.finish( - claimed.publication_id, - url="https://runs.example/runs/published/", - expected_lease=claimed.lease_until, - ) - receipt = queue.receipt(claimed.publication_id) - assert receipt["status"] == "published" - assert list(directory.glob("*.tmp")) == [] - - -@pytest.mark.parametrize( - "origin", - ["http://runs.example", "https://runs.example/path", "https://user@runs.example"], -) -def test_delivery_rejects_non_origin_destinations(tmp_path, origin): - queue = GraphPublicationQueue(tmp_path / "jobs.sqlite3") - with pytest.raises(ValueError, match="service URL"): - GraphPublicationDelivery( - queue, credential=lambda: ("missing", None), origin=origin - ) diff --git a/packages/microcosm-build/tests/engine_free/shared/test_provider_adapter.py b/packages/microcosm-build/tests/engine_free/shared/test_provider_adapter.py new file mode 100644 index 0000000000..bb0b3f3558 --- /dev/null +++ b/packages/microcosm-build/tests/engine_free/shared/test_provider_adapter.py @@ -0,0 +1,157 @@ +"""Test Microcosm's adapter with provider APIs replaced, not hosted services.""" + +from types import SimpleNamespace + +import pytest + +from microcosm.build import telemetry_emitter as adapter + +START = adapter.LocalTelemetryEmitter.start.__func__ + + +def test_selects_both_workers_once_with_shared_path_and_build_identity( + monkeypatch, tmp_path +): + calls, messages, closed = [], [], [] + transport = SimpleNamespace(send=messages.append) + + def for_module(name): + assert name == "telemetry" + return transport + + handle = SimpleNamespace( + client=SimpleNamespace(for_module=for_module), close=lambda: closed.append(True) + ) + + def start(modules, **options): + calls.append((modules, options)) + return handle + + monkeypatch.setattr(adapter, "start_services", start) + monkeypatch.setattr(adapter, "runtime_identity", lambda: {"commit": "synthetic"}) + path = tmp_path / "jobs.sqlite3" + emitter = START( + adapter.LocalTelemetryEmitter, + run_id="run-one", + country_code="uk", + pipeline="full", + spool_path=path, + heartbeat_seconds=123, + ) + assert len(calls) == 1 + modules, options = calls[0] + assert set(modules) == {"telemetry", "orrery"} + assert all(m["spool_path"] == str(path) for m in modules.values()) + assert "registration" not in modules["orrery"] + assert modules["telemetry"]["registration"]["run_id"] == "run-one" + assert modules["telemetry"]["heartbeat_seconds"] == 123 + assert options["startup_timeout"] == 3 + assert messages[0]["details"] == {"identity": {"commit": "synthetic"}} + emitter.complete() + emitter.close() + assert closed == [True] + assert messages[-1]["status"] == "completed" + + +def test_disabled_telemetry_still_delegates_durable_graph_enqueue( + monkeypatch, tmp_path +): + calls = [] + + def publish(directory, inventory, **options): + calls.append((directory, inventory, options)) + assert options["available"]() is False + return {"status": "pending", "publication_id": "independent"} + + monkeypatch.setattr(adapter, "publish_graph", publish) + path = tmp_path / "jobs.sqlite3" + emitter = adapter.LocalTelemetryEmitter( + run=adapter.TelemetryRun("run-one", "uk", "full"), + transport=None, + spool_path=path, + ) + value = {"publication_id": "independent"} + receipt = emitter.publish_graph(tmp_path, value, wait_seconds=0) + assert receipt["status"] == "pending" + assert calls[0][0:2] == (tmp_path, value) + assert calls[0][2]["spool_path"] == path + assert calls[0][2]["wait_seconds"] == 0 + + +def test_start_failure_is_nonfatal_and_does_not_log_exception_payload( + monkeypatch, capsys, tmp_path +): + def fail(*args, **kwargs): + raise RuntimeError("private credential should not be logged") + + monkeypatch.setattr(adapter, "start_services", fail) + emitter = START( + adapter.LocalTelemetryEmitter, + run_id="run-one", + country_code="uk", + pipeline="full", + spool_path=tmp_path / "jobs.sqlite3", + ) + assert not emitter.available + warning = capsys.readouterr().err + assert "RuntimeError" in warning + assert "private credential" not in warning + + +@pytest.mark.parametrize("operation", ["complete", "close"]) +def test_default_test_emitter_uses_provider_lifecycle_in_memory( + fake_telemetry_emitters, operation +): + emitter = adapter.LocalTelemetryEmitter.start( + run_id="run-one", country_code="uk", pipeline="full" + ) + emitter.stage("one") + getattr(emitter, operation)() + assert not emitter.available + assert emitter.events[0]["stage_id"] == "one" + + +def test_fixture_intercepts_preimported_build_entrypoint( + monkeypatch, fake_telemetry_emitters +): + from microcosm.build.uk_runtime import rowwise_staging + from test_support.microcosm_build.telemetry import FakeTelemetryEmitter + + def fail(*args, **kwargs): + pytest.fail("test started a real provider") + + monkeypatch.setattr(adapter, "start_services", fail) + for start in ( + rowwise_staging.start_local_telemetry_emitter_service, + adapter.LocalTelemetryEmitter.start, + ): + emitter = start(run_id="fake", country_code="GB", pipeline="test") + assert isinstance(emitter, FakeTelemetryEmitter) + assert emitter in fake_telemetry_emitters + + +def test_fixture_isolates_cached_huggingface_credentials(tmp_path): + from pathlib import Path + + from huggingface_hub import constants, get_token + + assert Path(constants.HF_TOKEN_PATH).is_relative_to(tmp_path) + assert get_token() is None + + +@pytest.mark.parametrize("failure", ["default_spool_path", "runtime_identity"]) +def test_metadata_or_cache_failure_never_starts_a_service(monkeypatch, failure): + def fail(): + raise OSError("private setup detail") + + monkeypatch.setattr(adapter, failure, fail) + monkeypatch.setattr( + adapter, "start_services", lambda *a, **kw: pytest.fail("started after failure") + ) + emitter = START( + adapter.LocalTelemetryEmitter, + run_id="failed", + country_code="uk", + pipeline="full", + ) + assert not emitter.available diff --git a/packages/microcosm-build/tests/engine_free/shared/test_publication_workers.py b/packages/microcosm-build/tests/engine_free/shared/test_publication_workers.py deleted file mode 100644 index 71bfe1566b..0000000000 --- a/packages/microcosm-build/tests/engine_free/shared/test_publication_workers.py +++ /dev/null @@ -1,142 +0,0 @@ -"""Independent publication delivery in the existing local service.""" - -import socket -import tempfile -import threading -import time -from pathlib import Path - -from microcosm.build.telemetry_emitter_service.auth import CollectorSession -from microcosm.build.telemetry_emitter_service.main import build_parser -from microcosm.build.telemetry_emitter_service.runtime import EmitterService -from microcosm.build.telemetry_emitter_service.spool import EventSpool - - -def test_slow_graph_upload_does_not_block_telemetry_or_socket( - tmp_path, real_local_telemetry -): - entered, release, telemetry_worked = (threading.Event() for _ in range(3)) - - class GraphDelivery: - def flush_once(self): - entered.set() - release.wait(5) - - class TelemetryDelivery: - def flush_once(self): - telemetry_worked.set() - return False - - class Sampler: - def parent_alive(self): - return True - - def sample(self): - return {} - - path = Path(tempfile.mkdtemp(prefix="mc-pub-test-", dir="/tmp")) / "e.sock" - service = EmitterService( - socket_path=path, - registration={ - "run_id": "run", - "producer_id": "producer", - "country_code": "GB", - "pipeline": "test", - }, - spool=EventSpool(tmp_path / "events.sqlite3"), - delivery=TelemetryDelivery(), - graph_delivery=GraphDelivery(), - sampler=Sampler(), - heartbeat_seconds=60, - drain_seconds=0, - ) - thread = threading.Thread(target=service.run) - thread.start() - try: - assert entered.wait(3) - assert telemetry_worked.wait(3) - with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as peer: - peer.settimeout(0.5) - peer.connect(str(path)) - peer.sendall(b'{"action":"ping"}\n') - assert peer.recv(16) == b"ok\n" - finally: - service._stop.set() - release.set() - thread.join(3) - assert not thread.is_alive() - - -def test_graph_only_service_requires_no_run_registration(tmp_path): - args = build_parser().parse_args( - [ - "--socket", - str(tmp_path / "e.sock"), - "--spool", - str(tmp_path / "jobs.sqlite3"), - "--parent-pid", - "1", - ] - ) - assert args.registration_json is None - - -def test_both_workers_share_one_collector_exchange(): - calls = [] - - def exchange(*args): - calls.append(args) - time.sleep(0.01) - return 200, {"access_token": "synthetic-session", "expires_in": 3600} - - session = CollectorSession( - "https://collector.example", post=exchange, credential=lambda: "hf-synthetic" - ) - results = [] - threads = [ - threading.Thread(target=lambda: results.append(session.credential())) - for _ in range(2) - ] - for thread in threads: - thread.start() - for thread in threads: - thread.join(2) - assert results == [("ok", "synthetic-session")] * 2 - assert len(calls) == 1 - - -def test_graph_worker_survives_a_local_queue_error(tmp_path, capsys): - from sqlalchemy.exc import OperationalError - - attempts = [] - recovered = threading.Event() - - class GraphDelivery: - def flush_once(self): - attempts.append(None) - if len(attempts) == 1: - raise OperationalError("UPDATE", {}, Exception("database is locked")) - recovered.set() - return False - - service = EmitterService( - socket_path=tmp_path / "unused.sock", - registration=None, - spool=EventSpool(tmp_path / "events.sqlite3"), - delivery=None, - graph_delivery=GraphDelivery(), - sampler=None, - heartbeat_seconds=60, - drain_seconds=0, - ) - worker = threading.Thread( - target=service._graph_worker, args=(service.graph_delivery,) - ) - worker.start() - try: - assert recovered.wait(10) - finally: - service._stop.set() - worker.join(5) - assert not worker.is_alive() - assert "graph publication hit a local queue error" in capsys.readouterr().err diff --git a/packages/microcosm-build/tests/engine_free/shared/test_telemetry_emitter.py b/packages/microcosm-build/tests/engine_free/shared/test_telemetry_emitter.py deleted file mode 100644 index a2cb501161..0000000000 --- a/packages/microcosm-build/tests/engine_free/shared/test_telemetry_emitter.py +++ /dev/null @@ -1,970 +0,0 @@ -import json -import os -import socket -import tempfile -import threading -import time -from datetime import UTC, datetime, timedelta -from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer -from pathlib import Path -from types import SimpleNamespace - -import pytest -from alembic import command -from sqlalchemy import Column, Integer, MetaData, String, Table, create_engine -from sqlalchemy.orm import Session - -from microcosm.build.telemetry_emitter import ( - LocalTelemetryEmitter, - TelemetryRun, -) -from microcosm.build.telemetry_emitter_service import ( - CollectorDelivery, - EmitterService, - EventSpool, -) -from microcosm.build.telemetry_emitter_service import collector as collector_module -from microcosm.build.telemetry_emitter_service import resources as resources_module -from microcosm.build.telemetry_emitter_service import runtime as runtime_module -from microcosm.build.telemetry_emitter_service import spool as spool_module -from microcosm.build.telemetry_emitter_service.constants import ( - DRAIN_RETRY_SECONDS, - PRODUCTION_COLLECTOR_URL, -) -from microcosm.build.telemetry_emitter_service.database import ( - create_spool_engine, - sqlite_database_url, -) -from microcosm.build.telemetry_emitter_service.migrations import ( - alembic_config, - current_database_revision, - migration_head_revision, -) -from microcosm.build.telemetry_emitter_service.models import ( - SpoolModel, - TelemetryEventRecord, - TelemetryRunRecord, - serialized_json_length, -) -from microcosm.build.telemetry_protocol import ( - BUILD_COMPLETED_MESSAGE, - BUILD_STARTED_MESSAGE, - MAX_TELEMETRY_DETAILS_BYTES, -) -from microcosm.build.telemetry_sanitization import ( - sanitize_details, - sanitize_json, - sanitize_text, -) - - -def _recent_timestamp() -> str: - """A stored-row timestamp inside the spool's retention window. - - The spool prunes rows older than ``RETENTION_DAYS`` when it opens, so a - fixed calendar date in a migrated-spool fixture expires a week later. - """ - - return (datetime.now(UTC) - timedelta(minutes=5)).isoformat() - - -def _registration(run_id: str = "run-a") -> dict[str, object]: - return TelemetryRun( - run_id=run_id, - country_code="US", - pipeline="us_fiscal_refresh", - candidate_id="candidate-a", - producer_id="producer-a", - ).as_registration() - - -def _event(stage_id: str = "compile_targets") -> dict[str, object]: - return { - "timestamp": "2026-10-02T10:00:00+00:00", - "event_type": "stage", - "stage_id": stage_id, - "status": "started", - "message": "Compiling targets.", - "details": {"batches": 12}, - } - - -def test_development_collector_must_be_on_loopback() -> None: - assert collector_module._development_collector_url("http://127.0.0.1:8080") == ( - "http://127.0.0.1:8080" - ) - for value in ("https://collector.example", "http://192.0.2.1:8080"): - try: - collector_module._development_collector_url(value) - except ValueError as error: - assert "loopback" in str(error) - else: - raise AssertionError("non-loopback development collector was accepted") - - -def test_production_collector_cannot_be_replaced_by_environment( - tmp_path, monkeypatch -) -> None: - monkeypatch.setenv( - "MICROCOSM_TELEMETRY_COLLECTOR_URL", - "https://untrusted.example", - ) - - delivery = CollectorDelivery(EventSpool(tmp_path / "events.sqlite3")) - - assert delivery.collector_url == PRODUCTION_COLLECTOR_URL - - -def test_token_bearing_http_post_does_not_follow_redirects() -> None: - paths: list[str] = [] - - class Handler(BaseHTTPRequestHandler): - def do_POST(self): - paths.append(self.path) - self.send_response(307) - self.send_header("Location", "/credential-leak") - self.send_header("Content-Length", "0") - self.end_headers() - - def log_message(self, format, *args): - return - - server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) - server_thread = threading.Thread(target=server.serve_forever) - server_thread.start() - try: - status, _ = collector_module._http_post( - f"http://127.0.0.1:{server.server_port}/exchange", - {"run_id": "run-a"}, - "hf-private-token", - ) - finally: - server.shutdown() - server_thread.join(timeout=2) - server.server_close() - - assert status == 307 - assert paths == ["/exchange"] - - -def test_outbound_payload_redacts_credentials_and_tracebacks() -> None: - assert sanitize_text("Bearer hf_abcdefghijk") == "[redacted]" - assert sanitize_json( - { - "HF_TOKEN": "hf_abcdefghijk", - "traceback": "private stack", - "message": "credential=top-secret", - } - ) == { - "HF_TOKEN": "[redacted]", - "traceback": "[redacted]", - "message": "[redacted]", - } - bounded = sanitize_details( - {"done": 2, **{f"large_{index}": "x" * 2_000 for index in range(10)}} - ) - assert bounded["done"] == 2 - assert bounded["telemetry_details_truncated"] is True - assert len(json.dumps(bounded).encode()) <= MAX_TELEMETRY_DETAILS_BYTES - - -def test_event_spool_assigns_stable_sequences_and_acknowledges(tmp_path) -> None: - spool_path = tmp_path / "events.sqlite3" - spool = EventSpool(spool_path) - registration = _registration() - spool.register(registration) - - first = spool.append(registration, _event()) - second = spool.append(registration, _event("calibrate")) - - assert first["sequence"] == 1 - assert second["sequence"] == 2 - assert first["producer_id"] == "producer-a" - assert [event["sequence"] for event in spool.batch("run-a", "producer-a")] == [ - 1, - 2, - ] - - spool.acknowledge([first["event_id"]]) - assert [event["sequence"] for event in spool.batch("run-a", "producer-a")] == [2] - assert current_database_revision(spool_path) == migration_head_revision() - - -def test_alembic_schema_matches_sqlalchemy_models(tmp_path) -> None: - spool_path = tmp_path / "events.sqlite3" - EventSpool(spool_path) - engine = create_spool_engine(spool_path) - try: - with engine.begin() as connection: - with alembic_config(connection=connection) as config: - command.check(config) - finally: - engine.dispose() - - -def test_sequential_stage_updates_close_the_previous_stage() -> None: - emitter = LocalTelemetryEmitter( - run=TelemetryRun( - run_id="run-a", - country_code="US", - pipeline="us_fiscal_refresh", - ), - process=None, - socket_path=Path("/unused"), - runtime_dir=None, - ) - messages: list[dict[str, object]] = [] - emitter._send = messages.append - - emitter.transition_stage("load", message="Loading.") - emitter.transition_stage("compile", message="Compiling.") - emitter.transition_stage("compile", status="completed", batches=4) - - events = [message["event"] for message in messages] - assert [(event["stage_id"], event["status"]) for event in events] == [ - ("load", "started"), - ("load", "completed"), - ("compile", "started"), - ("compile", "completed"), - ] - - -def test_event_spool_keeps_repeated_run_producers_separate(tmp_path) -> None: - spool = EventSpool(tmp_path / "events.sqlite3") - first = _registration() - second = {**first, "producer_id": "producer-b"} - spool.register(first) - spool.register(second) - - spool.append(first, _event()) - spool.append(second, _event()) - - assert spool.batch("run-a", "producer-a")[0]["sequence"] == 1 - assert spool.batch("run-a", "producer-b")[0]["sequence"] == 1 - assert len(spool.pending_runs()) == 2 - - -def _recent_timestamp() -> str: - """A stored-row timestamp inside the spool's retention window on any date.""" - - return (datetime.now(UTC) - timedelta(hours=1)).isoformat() - - -def test_pre_eligibility_spool_is_not_uploaded_after_upgrade(tmp_path) -> None: - path = tmp_path / "events.sqlite3" - registration = _registration() - metadata = MetaData() - runs = Table( - "telemetry_runs", - metadata, - Column("run_id", String, primary_key=True), - Column("producer_id", String, primary_key=True), - Column("registration_json", String, nullable=False), - Column("next_sequence", Integer, nullable=False, default=1), - Column("updated_at", String, nullable=False), - ) - events = Table( - "telemetry_events", - metadata, - Column("event_id", String, primary_key=True), - Column("run_id", String, nullable=False), - Column("producer_id", String, nullable=False), - Column("sequence", Integer, nullable=False), - Column("payload_json", String, nullable=False), - Column("created_at", String, nullable=False), - ) - engine = create_engine(sqlite_database_url(path)) - metadata.create_all(engine) - with engine.begin() as connection: - connection.execute( - runs.insert().values( - run_id="run-a", - producer_id="producer-a", - registration_json=json.dumps(registration), - next_sequence=2, - updated_at=_recent_timestamp(), - ) - ) - connection.execute( - events.insert().values( - event_id="old-event", - run_id="run-a", - producer_id="producer-a", - sequence=1, - payload_json=json.dumps({"event_id": "old-event"}), - created_at=_recent_timestamp(), - ) - ) - engine.dispose() - - spool = EventSpool(path) - - assert spool.has_pending() - assert spool.pending_runs() == [] - assert current_database_revision(path) == migration_head_revision() - - -def test_current_pre_alembic_spool_is_adopted_without_losing_events(tmp_path) -> None: - path = tmp_path / "events.sqlite3" - registration = _registration() - engine = create_engine(sqlite_database_url(path)) - SpoolModel.metadata.create_all( - engine, - tables=[TelemetryRunRecord.__table__, TelemetryEventRecord.__table__], - ) - with Session(engine) as session, session.begin(): - session.add( - TelemetryRunRecord( - run_id="run-a", - producer_id="producer-a", - registration=registration, - next_sequence=2, - upload_state="pending", - local_only_reason=None, - updated_at=_recent_timestamp(), - ) - ) - session.add( - TelemetryEventRecord( - event_id="existing-event", - run_id="run-a", - producer_id="producer-a", - sequence=1, - payload={"event_id": "existing-event", "sequence": 1}, - created_at=_recent_timestamp(), - ) - ) - engine.dispose() - - spool = EventSpool(path) - - assert spool.pending_runs() == [registration] - assert spool.batch("run-a", "producer-a") == [ - {"event_id": "existing-event", "sequence": 1} - ] - assert current_database_revision(path) == migration_head_revision() - - -def test_event_spool_prunes_oldest_events_to_size_limit( - tmp_path, - monkeypatch, -) -> None: - spool = EventSpool(tmp_path / "events.sqlite3") - registration = _registration() - spool.register(registration) - first = spool.append(registration, _event("first")) - second = spool.append(registration, _event("second")) - monkeypatch.setattr( - spool_module, - "MAX_QUEUED_BYTES", - serialized_json_length(second), - ) - - spool.prune() - - assert first["event_id"] != second["event_id"] - assert spool.batch("run-a", "producer-a") == [second] - - -def test_collector_delivery_exchanges_hf_token_then_flushes( - tmp_path, monkeypatch -) -> None: - spool = EventSpool(tmp_path / "events.sqlite3") - registration = _registration() - spool.register(registration) - queued = spool.append(registration, _event()) - requests: list[tuple[str, dict[str, object], str]] = [] - - monkeypatch.setattr(collector_module, "_huggingface_token", lambda: "hf-secret") - - def fake_post(url, payload, token, *, timeout=5.0): - requests.append((url, payload, token)) - if url.endswith("/v1/auth/huggingface/exchange"): - return 200, {"access_token": "collector-token", "expires_in": 3600} - if url.endswith("/v1/runs"): - return 201, {"registered": True} - return 202, {"accepted": 1, "duplicates": 0} - - monkeypatch.setattr(collector_module, "_http_post", fake_post) - - delivery = CollectorDelivery( - spool, - development_collector_url="http://127.0.0.1:8080", - ) - assert delivery.flush_once() - assert requests[0][2] == "hf-secret" - assert requests[0][1] == {} - assert requests[1] == ( - "http://127.0.0.1:8080/v1/runs", - registration, - "collector-token", - ) - assert requests[2][2] == "collector-token" - assert requests[2][1] == {"events": [queued]} - assert not spool.has_pending() - - -def test_non_org_credential_keeps_event_local(tmp_path, monkeypatch, capsys) -> None: - spool = EventSpool(tmp_path / "events.sqlite3") - registration = _registration() - spool.register(registration) - spool.append(registration, _event()) - requests: list[dict[str, object]] = [] - monkeypatch.setattr(collector_module, "_huggingface_token", lambda: "hf-outsider") - - def reject(url, payload, token, *, timeout=5.0): - requests.append(payload) - return 403, {"detail": "not a member"} - - monkeypatch.setattr(collector_module, "_http_post", reject) - delivery = CollectorDelivery( - spool, - development_collector_url="http://127.0.0.1:8080", - ) - - assert not delivery.flush_once() - assert not delivery.flush_once() - assert spool.has_pending() - assert spool.pending_runs() == [] - assert requests == [{}] - assert capsys.readouterr().err.count("local-only for this run") == 1 - - -@pytest.mark.parametrize( - ("status", "retried"), [(422, False), (404, False), (429, True), (408, True)] -) -def test_a_settled_collector_rejection_goes_local_only_instead_of_retrying( - tmp_path, monkeypatch, capsys, status, retried -) -> None: - """A 4xx the collector will repeat (an event shape it does not accept, say) - must not wedge the queue: the run goes local-only with a reason and one - warning. A timeout or a rate limit still retries.""" - spool = EventSpool(tmp_path / "events.sqlite3") - registration = _registration() - spool.register(registration) - spool.append(registration, _event()) - monkeypatch.setattr(collector_module, "_huggingface_token", lambda: "hf-member") - - def fake_post(url, payload, token, *, timeout=5.0): - if url.endswith("/v1/auth/huggingface/exchange"): - return 200, {"access_token": "collector-token", "expires_in": 3600} - if url.endswith("/v1/runs"): - return 201, {"registered": True} - return status, {"detail": "rejected"} - - monkeypatch.setattr(collector_module, "_http_post", fake_post) - delivery = CollectorDelivery( - spool, development_collector_url="http://127.0.0.1:8080" - ) - assert not delivery.flush_once() - assert spool.has_pending() - if retried: - assert spool.pending_runs() == [registration] - assert "local-only" not in capsys.readouterr().err - return - assert spool.pending_runs() == [] - assert not delivery.flush_once() - err = capsys.readouterr().err - assert err.count("local-only for this run") == 1 - assert f"HTTP {status}" in err - - -def test_identity_provider_outage_keeps_events_eligible_for_retry( - tmp_path, monkeypatch -) -> None: - spool = EventSpool(tmp_path / "events.sqlite3") - registration = _registration() - spool.register(registration) - spool.append(registration, _event()) - monkeypatch.setattr(collector_module, "_huggingface_token", lambda: "hf-member") - monkeypatch.setattr( - collector_module, - "_http_post", - lambda *args, **kwargs: (503, {"detail": "temporarily unavailable"}), - ) - - assert not CollectorDelivery(spool).flush_once() - assert spool.pending_runs() == [registration] - - -def test_missing_credential_never_contacts_collector_and_stays_local_only( - tmp_path, monkeypatch -) -> None: - spool_path = tmp_path / "events.sqlite3" - spool = EventSpool(spool_path) - registration = _registration() - spool.register(registration) - spool.append(registration, _event()) - monkeypatch.setattr(collector_module, "_huggingface_token", lambda: None) - monkeypatch.setattr( - collector_module, - "_http_post", - lambda *args, **kwargs: (_ for _ in ()).throw( - AssertionError("network request") - ), - ) - - delivery = CollectorDelivery(spool) - assert not delivery.flush_once() - assert spool.pending_runs() == [] - - reopened = EventSpool(spool_path) - monkeypatch.setattr(collector_module, "_huggingface_token", lambda: "hf-later") - assert reopened.pending_runs() == [] - - -def test_resource_sampler_has_a_base_install_fallback(monkeypatch) -> None: - monkeypatch.setattr(resources_module, "psutil", None) - sampler = resources_module.ProcessTreeSampler(os.getpid()) - - assert sampler.parent_alive() - sample = sampler.sample() - assert sample.keys() == { - "cpu_user_seconds", - "cpu_system_seconds", - "rss_bytes", - "peak_rss_bytes", - } - assert all(value >= 0 for value in sample.values()) - - -def test_resource_sampler_keeps_reaped_child_cpu_monotonic(monkeypatch) -> None: - state = {"child_alive": True} - - class FakeProcess: - def __init__(self, pid): - self.pid = pid - - def create_time(self): - return float(self.pid) - - def children(self, recursive=False): - assert recursive is True - if self.pid == 10 and state["child_alive"]: - return [FakeProcess(11)] - return [] - - def cpu_times(self): - if self.pid == 10: - return SimpleNamespace( - user=10.0, - system=2.0, - children_user=100.0 if not state["child_alive"] else 0.0, - children_system=20.0 if not state["child_alive"] else 0.0, - ) - return SimpleNamespace( - user=80.0, - system=15.0, - children_user=0.0, - children_system=0.0, - ) - - def memory_info(self): - return SimpleNamespace(rss=100) - - monkeypatch.setattr( - resources_module, - "psutil", - SimpleNamespace(Process=FakeProcess, Error=OSError, STATUS_ZOMBIE="zombie"), - ) - sampler = resources_module.ProcessTreeSampler(10) - - before = sampler.sample() - state["child_alive"] = False - after = sampler.sample() - - assert before["cpu_user_seconds"] == 90.0 - assert before["cpu_system_seconds"] == 17.0 - assert after["cpu_user_seconds"] == 110.0 - assert after["cpu_system_seconds"] == 22.0 - - -def test_emitter_startup_timeout_terminates_and_reaps_service( - tmp_path, monkeypatch, real_local_telemetry -) -> None: - class FakeProcess: - def __init__(self): - self.terminated = False - self.waited = False - - def poll(self): - return None - - def terminate(self): - self.terminated = True - - def wait(self, timeout=None): - self.waited = True - return 0 - - process = FakeProcess() - monkeypatch.setattr( - "microcosm.build.telemetry_emitter.subprocess.Popen", - lambda *args, **kwargs: process, - ) - - emitter = LocalTelemetryEmitter.start( - run_id="startup-timeout", - country_code="US", - pipeline="test-pipeline", - development_collector_url="http://127.0.0.1:8080", - spool_path=tmp_path / "events.sqlite3", - startup_timeout_seconds=0, - ) - - assert not emitter.available - assert process.terminated - assert process.waited - - -class _FakeSampler: - def sample(self): - return { - "cpu_user_seconds": 1.0, - "cpu_system_seconds": 0.5, - "rss_bytes": 100, - "peak_rss_bytes": 120, - } - - def parent_alive(self): - return True - - -class _FakeDelivery: - def flush_once(self): - return False - - -def test_local_socket_acknowledges_after_durable_queue( - tmp_path, real_local_telemetry -) -> None: - socket_path = ( - Path(tempfile.mkdtemp(prefix="microcosm-test-", dir="/tmp")) / "e.sock" - ) - spool = EventSpool(tmp_path / "events.sqlite3") - registration = _registration() - emitter = EmitterService( - socket_path=socket_path, - registration=registration, - spool=spool, - delivery=_FakeDelivery(), - sampler=_FakeSampler(), - heartbeat_seconds=60, - drain_seconds=0, - ) - thread = threading.Thread(target=emitter.run) - thread.start() - deadline = time.monotonic() + 2 - while not socket_path.exists() and time.monotonic() < deadline: - time.sleep(0.01) - - with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as client: - client.connect(str(socket_path)) - client.sendall( - json.dumps({"action": "event", "event": _event()}).encode() + b"\n" - ) - assert client.recv(16) == b"ok\n" - - assert spool.batch("run-a", "producer-a")[0]["resources"]["rss_bytes"] == 100 - - with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as client: - client.connect(str(socket_path)) - client.sendall(b'{"action":"close"}\n') - assert client.recv(16) == b"ok\n" - thread.join(timeout=2) - assert not thread.is_alive() - - -def test_subprocess_exchanges_ambient_token_and_delivers_events( - tmp_path, monkeypatch, real_local_telemetry -) -> None: - requests: list[tuple[str, str, dict[str, object]]] = [] - - class Handler(BaseHTTPRequestHandler): - def do_POST(self): - length = int(self.headers.get("Content-Length", "0")) - payload = json.loads(self.rfile.read(length)) - requests.append((self.path, self.headers["Authorization"], payload)) - if self.path.endswith("/v1/auth/huggingface/exchange"): - response = {"access_token": "collector-token", "expires_in": 3600} - status = 200 - elif self.path == "/v1/runs": - response = {"registered": True} - status = 201 - else: - response = { - "accepted": len(payload["events"]), - "duplicates": 0, - } - status = 202 - body = json.dumps(response).encode() - self.send_response(status) - self.send_header("Content-Type", "application/json") - self.send_header("Content-Length", str(len(body))) - self.end_headers() - self.wfile.write(body) - - def log_message(self, format, *args): - return - - server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) - server_thread = threading.Thread(target=server.serve_forever) - server_thread.start() - monkeypatch.setenv("HF_TOKEN", "hf-ambient-test-token") - emitter = LocalTelemetryEmitter.start( - run_id="subprocess-run", - country_code="US", - pipeline="test-pipeline", - development_collector_url=f"http://127.0.0.1:{server.server_port}", - spool_path=tmp_path / "subprocess.sqlite3", - heartbeat_seconds=60, - ) - assert emitter.available - emitter.stage("compile", message="Started.") - emitter.complete() - assert emitter._process is not None - emitter._process.wait(timeout=10) - server.shutdown() - server_thread.join(timeout=2) - server.server_close() - - assert requests[0][0] == "/v1/auth/huggingface/exchange" - assert requests[0][1] == "Bearer hf-ambient-test-token" - assert requests[0][2] == {} - assert requests[1][0] == "/v1/runs" - assert requests[1][1] == "Bearer collector-token" - ingestion = [request for request in requests if request[0].endswith("/events")] - assert ingestion - events = [ - event - for _, authorization, payload in ingestion - for event in payload["events"] - if authorization == "Bearer collector-token" - ] - assert [event["sequence"] for event in events] == list(range(1, len(events) + 1)) - identity = events[0]["details"]["identity"] - assert identity["host"]["cpu_count"] is not None - assert "runtime" in identity - assert events[0]["message"] == BUILD_STARTED_MESSAGE - assert events[-1]["message"] == BUILD_COMPLETED_MESSAGE - assert events[-1]["status"] == "completed" - - -def test_failure_marks_active_stage_failed_before_run_failure(): - from test_support.microcosm_build.telemetry import FakeTelemetryEmitter - - emitter = FakeTelemetryEmitter(TelemetryRun("failed-build", "GB", "test")) - emitter.transition_stage("calibration") - emitter.fail(ValueError("invalid calibration")) - - assert [(event["event_type"], event["status"]) for event in emitter.events] == [ - ("stage", "started"), - ("stage", "failed"), - ("run", "failed"), - ] - assert emitter.events[1]["message"] == "invalid calibration" - assert emitter.events[1]["details"]["error_type"] == "ValueError" - assert emitter.events[2]["details"]["failed_during"] == "calibration" - assert not emitter.available - - -@pytest.mark.parametrize("failure_point", ["mkdtemp", "chmod", "cache_dir"]) -def test_early_startup_failure_returns_disabled_handle( - tmp_path, monkeypatch, capsys, real_local_telemetry, failure_point -): - from microcosm.build import telemetry_emitter as module - - runtime_dir = tmp_path / "runtime" - - def make_runtime(**kwargs): - runtime_dir.mkdir() - return str(runtime_dir) - - def fail(*args, **kwargs): - raise OSError("telemetry setup refused") - - monkeypatch.setattr(module.tempfile, "mkdtemp", make_runtime) - monkeypatch.setattr( - module.subprocess, - "Popen", - lambda *a, **kw: pytest.fail("spawned after failed setup"), - ) - if failure_point == "mkdtemp": - monkeypatch.setattr(module.tempfile, "mkdtemp", fail) - elif failure_point == "chmod": - monkeypatch.setattr(Path, "chmod", fail) - else: - monkeypatch.setattr(module, "_cache_dir", fail) - - emitter = LocalTelemetryEmitter.start( - run_id="failed-start", - country_code="GB", - pipeline="test", - development_collector_url="http://127.0.0.1:1", - ) - - assert not emitter.available - assert not runtime_dir.exists() - assert "warning" in capsys.readouterr().err.lower() - emitter.close() - - -def test_default_fixture_intercepts_preimported_entrypoint_and_class( - monkeypatch, fake_telemetry_emitters -): - from microcosm.build import telemetry_emitter as module - from microcosm.build.uk_runtime import rowwise_staging - from test_support.microcosm_build.telemetry import FakeTelemetryEmitter - - monkeypatch.setenv("HF_TOKEN", "hf-ambient-must-not-be-used") - monkeypatch.setattr( - module.subprocess, - "Popen", - lambda *a, **kw: pytest.fail("test spawned real emitter"), - ) - for start in ( - rowwise_staging.start_local_telemetry_emitter_service, - LocalTelemetryEmitter.start, - ): - emitter = start(run_id="fake", country_code="GB", pipeline="test") - assert isinstance(emitter, FakeTelemetryEmitter) - assert emitter in fake_telemetry_emitters - - -def test_default_fixture_isolates_cached_huggingface_credentials(tmp_path): - from huggingface_hub import constants, get_token - - assert Path(constants.HF_TOKEN_PATH).is_relative_to(tmp_path) - assert get_token() is None - - -@pytest.mark.parametrize("collector_url", [None, "https://collector.example"]) -def test_real_service_opt_in_requires_explicit_loopback( - real_local_telemetry, collector_url -): - with pytest.raises(ValueError, match="loopback"): - LocalTelemetryEmitter.start( - run_id="isolated", - country_code="GB", - pipeline="test", - development_collector_url=collector_url, - ) - - -def test_close_does_not_report_success(): - from test_support.microcosm_build.telemetry import FakeTelemetryEmitter - - emitter = FakeTelemetryEmitter(TelemetryRun("cleanup", "GB", "test")) - emitter.transition_stage("calibration") - emitter.close() - assert [(event["event_type"], event["status"]) for event in emitter.events] == [ - ("stage", "started") - ] - - -def test_failure_without_active_stage_only_reports_run_failure(): - from test_support.microcosm_build.telemetry import FakeTelemetryEmitter - - emitter = FakeTelemetryEmitter(TelemetryRun("failed", "GB", "test")) - emitter.fail(ValueError("before first stage"), failed_during="preflight") - assert [(event["event_type"], event["status"]) for event in emitter.events] == [ - ("run", "failed") - ] - assert emitter.events[0]["details"]["failed_during"] == "preflight" - - -@pytest.mark.parametrize("drain_seconds", [0.1, 1.0]) -def test_shutdown_retries_wait_after_stop_without_exceeding_deadline( - tmp_path, monkeypatch, drain_seconds -) -> None: - clock = SimpleNamespace(now=0.0) - attempts: list[float] = [] - - def flush_once() -> bool: - attempts.append(clock.now) - # Simulate a small amount of delivery work, so a broken busy loop - # still reaches its deadline without relying on real elapsed time. - clock.now += 0.01 - return False - - def sleep(seconds: float) -> None: - clock.now += seconds - - monkeypatch.setattr( - runtime_module, - "time", - SimpleNamespace(monotonic=lambda: clock.now, sleep=sleep), - ) - service = EmitterService( - socket_path=tmp_path / "unused.sock", - registration=_registration(), - spool=SimpleNamespace(has_deliverable=lambda: True), - delivery=SimpleNamespace(flush_once=flush_once), - sampler=_FakeSampler(), - heartbeat_seconds=60, - drain_seconds=drain_seconds, - ) - service._stop.set() - - service._drain() - - assert attempts - assert all( - later - earlier >= DRAIN_RETRY_SECONDS - for earlier, later in zip(attempts, attempts[1:], strict=False) - ) - assert clock.now == pytest.approx(drain_seconds) - - -def test_fallback_sampler_retains_reaped_child_cpu(tmp_path, monkeypatch) -> None: - proc_root = tmp_path / "proc" - - def write_stat( - pid: int, - parent_pid: int, - user_ticks: int, - system_ticks: int, - child_user_ticks: int = 0, - child_system_ticks: int = 0, - ) -> Path: - # Fields start at the state after the parenthesized process name. - fields = ["0"] * 22 - fields[0] = "S" - fields[1] = str(parent_pid) - fields[11:15] = map( - str, (user_ticks, system_ticks, child_user_ticks, child_system_ticks) - ) - fields[19] = "1000" - fields[21] = "4" - path = proc_root / str(pid) / "stat" - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(f"{pid} (build worker) {' '.join(fields)}") - return path - - write_stat(100, 1, 100, 200) - child_path = write_stat(101, 100, 500, 300) - monkeypatch.setattr(resources_module, "psutil", None) - monkeypatch.setattr( - resources_module, - "Path", - lambda path: proc_root / Path(path).relative_to("/proc"), - ) - monkeypatch.setattr( - resources_module, - "os", - SimpleNamespace( - sysconf=lambda name: {"SC_CLK_TCK": 100, "SC_PAGE_SIZE": 4096}[name] - ), - ) - sampler = resources_module.ProcessTreeSampler(100) - - before = sampler.sample() - child_path.unlink() - write_stat(100, 1, 100, 200, child_user_ticks=500, child_system_ticks=300) - after = sampler.sample() - - assert before["cpu_user_seconds"] == after["cpu_user_seconds"] == 6.0 - assert before["cpu_system_seconds"] == after["cpu_system_seconds"] == 5.0 - assert before["rss_bytes"] == 8 * 4096 - assert after["rss_bytes"] == 4 * 4096 - assert after["peak_rss_bytes"] == before["rss_bytes"] diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_orrery_publication.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_orrery_publication.py index 1e6b3c8da1..41e54f4549 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_orrery_publication.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_orrery_publication.py @@ -211,6 +211,24 @@ def broken(*args, **kwargs): ) +def test_no_emitter_delegates_publication_without_a_run_id(tmp_path, monkeypatch): + args = fixture(tmp_path) + args.publish_orrery = True + monkeypatch.setattr(rowwise_staging, "_ACTIVE_EMITTER", None) + calls = [] + + def publish(directory, inventory, **options): + assert "run_id" not in inventory + assert options == {"wait_seconds": 0} + calls.append(directory) + return {"status": "pending", "error_code": "credential_missing"} + + monkeypatch.setattr(orrery_publication, "publish_graph", publish) + receipt = orrery_publication.finalize_graph(args, manifest=None).receipt + assert receipt["status"] == "pending" + assert calls == [args.attempt_evidence / "orrery-publication"] + + @pytest.mark.parametrize( "explicit,local,disabled,expected", [ diff --git a/test_support/microcosm_build/telemetry.py b/test_support/microcosm_build/telemetry.py index 7ff55a2475..bd2e3b2652 100644 --- a/test_support/microcosm_build/telemetry.py +++ b/test_support/microcosm_build/telemetry.py @@ -13,16 +13,14 @@ class FakeTelemetryEmitter(LocalTelemetryEmitter): """Exercise real lifecycle methods without starting a service or socket.""" def __init__(self, run: TelemetryRun) -> None: - super().__init__( - run=run, - process=None, - socket_path=Path("/unused-test-telemetry-socket"), - runtime_dir=None, - ) self.messages: list[Mapping[str, Any]] = [] + super().__init__(run=run, transport=self) + + def send(self, payload: Mapping[str, Any]) -> None: + self.messages.append({"action": "event", "event": payload}) - def _send(self, payload: Mapping[str, Any]) -> None: - self.messages.append(payload) + def close(self) -> None: + self._closed = True def publish_graph( self, directory: Path, inventory: dict, *, wait_seconds: float = 30.0 diff --git a/uv.lock b/uv.lock index 8da37c6d36..1f65b8dfce 100644 --- a/uv.lock +++ b/uv.lock @@ -800,8 +800,6 @@ name = "microcosm-build" version = "0.1.0" source = { editable = "packages/microcosm-build" } dependencies = [ - { name = "alembic" }, - { name = "httpx" }, { name = "huggingface-hub" }, { name = "jsonschema" }, { name = "microcosm-calibrate" }, @@ -809,13 +807,15 @@ dependencies = [ { name = "microcosm-fit" }, { name = "microcosm-frame" }, { name = "microcosm-graph" }, + { name = "microcosm-provider-client" }, + { name = "microcosm-provider-orrery" }, + { name = "microcosm-provider-telemetry" }, { name = "numpy" }, { name = "pandas" }, { name = "pyarrow" }, { name = "pyyaml" }, { name = "referencing" }, { name = "scipy" }, - { name = "sqlalchemy" }, ] [package.optional-dependencies] @@ -842,10 +842,8 @@ dev = [ [package.metadata] requires-dist = [ - { name = "alembic", specifier = ">=1.13.3,<2" }, { name = "h5py", marker = "extra == 'uk'", specifier = ">=3" }, { name = "h5py", marker = "extra == 'us'", specifier = ">=3" }, - { name = "httpx", specifier = ">=0.27,<1" }, { name = "huggingface-hub", specifier = ">=0.20" }, { name = "jsonschema", specifier = ">=4.23,<5" }, { name = "microcosm-calibrate", editable = "packages/microcosm-calibrate" }, @@ -855,6 +853,9 @@ requires-dist = [ { name = "microcosm-fit", editable = "packages/microcosm-fit" }, { name = "microcosm-frame", editable = "packages/microcosm-frame" }, { name = "microcosm-graph", editable = "packages/microcosm-graph" }, + { name = "microcosm-provider-client", git = "https://github.com/PolicyEngine/microcosm-emitter?subdirectory=packages%2Fprovider-client&rev=06e47847983ee67ccdb6a596062251322d07f793" }, + { name = "microcosm-provider-orrery", git = "https://github.com/PolicyEngine/microcosm-emitter?subdirectory=packages%2Fprovider-orrery&rev=06e47847983ee67ccdb6a596062251322d07f793" }, + { name = "microcosm-provider-telemetry", git = "https://github.com/PolicyEngine/microcosm-emitter?subdirectory=packages%2Fprovider-telemetry&rev=06e47847983ee67ccdb6a596062251322d07f793" }, { name = "microunit", marker = "extra == 'us'", specifier = ">=0.1.0" }, { name = "numpy", specifier = ">=1.26" }, { name = "openpyxl", marker = "extra == 'uk'", specifier = ">=3.1" }, @@ -866,7 +867,6 @@ requires-dist = [ { name = "pyyaml", specifier = ">=6" }, { name = "referencing", specifier = ">=0.35,<1" }, { name = "scipy", specifier = ">=1.13" }, - { name = "sqlalchemy", specifier = ">=2,<3" }, { name = "tables", marker = "extra == 'uk'", specifier = ">=3" }, { name = "tables", marker = "extra == 'us'", specifier = ">=3" }, ] @@ -1081,6 +1081,47 @@ dev = [ { name = "pytest", specifier = ">=8" }, ] +[[package]] +name = "microcosm-provider-client" +version = "0.1.0" +source = { git = "https://github.com/PolicyEngine/microcosm-emitter?subdirectory=packages%2Fprovider-client&rev=06e47847983ee67ccdb6a596062251322d07f793#06e47847983ee67ccdb6a596062251322d07f793" } +dependencies = [ + { name = "psutil" }, +] + +[[package]] +name = "microcosm-provider-core" +version = "0.1.0" +source = { git = "https://github.com/PolicyEngine/microcosm-emitter?subdirectory=packages%2Fprovider-core&rev=06e47847983ee67ccdb6a596062251322d07f793#06e47847983ee67ccdb6a596062251322d07f793" } +dependencies = [ + { name = "alembic" }, + { name = "huggingface-hub" }, + { name = "sqlalchemy" }, +] + +[[package]] +name = "microcosm-provider-orrery" +version = "0.1.0" +source = { git = "https://github.com/PolicyEngine/microcosm-emitter?subdirectory=packages%2Fprovider-orrery&rev=06e47847983ee67ccdb6a596062251322d07f793#06e47847983ee67ccdb6a596062251322d07f793" } +dependencies = [ + { name = "httpx" }, + { name = "microcosm-provider-client" }, + { name = "microcosm-provider-core" }, +] + +[[package]] +name = "microcosm-provider-telemetry" +version = "0.1.0" +source = { git = "https://github.com/PolicyEngine/microcosm-emitter?subdirectory=packages%2Fprovider-telemetry&rev=06e47847983ee67ccdb6a596062251322d07f793#06e47847983ee67ccdb6a596062251322d07f793" } +dependencies = [ + { name = "alembic" }, + { name = "huggingface-hub" }, + { name = "microcosm-provider-client" }, + { name = "microcosm-provider-core" }, + { name = "psutil" }, + { name = "sqlalchemy" }, +] + [[package]] name = "microcosm-workspace" version = "0.0.0" From f8b8df21a4533c95082b0904ef482eb964bdcd98 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Mon, 12 Oct 2026 17:55:17 +0400 Subject: [PATCH 2/4] Fix issues from review: use one microcosm-emitter package --- README.md | 8 +- changelog.d/1152.changed.md | 2 +- changelog.d/1153-single-emitter.fixed.md | 1 + conftest.py | 17 +- docs/agent-guide.md | 4 +- docs/orrery-publication.md | 28 +- packages/microcosm-build/pyproject.toml | 11 +- .../src/microcosm/build/telemetry_emitter.py | 62 +++-- .../build/uk_runtime/orrery_publication.py | 23 +- .../build/us_runtime/worker_identity.py | 2 +- .../shared/test_emitter_adapter.py | 244 ++++++++++++++++++ .../shared/test_provider_adapter.py | 157 ----------- .../uk/test_uk_orrery_publication.py | 77 ++++++ test_support/microcosm_build/telemetry.py | 72 +++++- uv.lock | 61 +---- 15 files changed, 494 insertions(+), 275 deletions(-) create mode 100644 changelog.d/1153-single-emitter.fixed.md create mode 100644 packages/microcosm-build/tests/engine_free/shared/test_emitter_adapter.py delete mode 100644 packages/microcosm-build/tests/engine_free/shared/test_provider_adapter.py diff --git a/README.md b/README.md index 37a4bae9b3..427c0f5b89 100644 --- a/README.md +++ b/README.md @@ -51,10 +51,12 @@ immutable publication before Hugging Face artifact staging, and preserves durable retry jobs and receipt snapshots. See [graph publication](docs/orrery-publication.md) for configuration and retry. -Supported US and UK build commands start one local provider service with separate +Supported US and UK build commands start one local emitter service with separate telemetry and graph-publication workers. Microcosm supplies build metadata and -graph files; provider packages own authentication, durable queues and delivery. -This draft migration requires published provider packages before merge; see the +graph files; the required `microcosm-emitter` package owns authentication, +durable queues and delivery. Telemetry and graph publication are internal +libraries in that single package, not separate installations. +This draft migration requires a published emitter release before merge; see the [migration prerequisites](docs/orrery-publication.md#deferred-migration-prerequisites). The telemetry worker reports live progress to the hosted collector when the operator's existing Hugging Face login is accepted; otherwise it retains the events diff --git a/changelog.d/1152.changed.md b/changelog.d/1152.changed.md index 08a1bd1d38..0bb4082144 100644 --- a/changelog.d/1152.changed.md +++ b/changelog.d/1152.changed.md @@ -1 +1 @@ -Delegate local telemetry and durable Orrery graph delivery to microcosm-local-provider while retaining UK graph generation and artifact staging in Microcosm. +Delegate local telemetry and durable Orrery graph delivery to the single microcosm-emitter package while retaining UK graph generation and artifact staging in Microcosm. diff --git a/changelog.d/1153-single-emitter.fixed.md b/changelog.d/1153-single-emitter.fixed.md new file mode 100644 index 0000000000..fd2c787afa --- /dev/null +++ b/changelog.d/1153-single-emitter.fixed.md @@ -0,0 +1 @@ +Use one microcosm-emitter dependency and its socket-based graph APIs, preserve public graph files when the service is unavailable, and create one run-start event with the build identity. diff --git a/conftest.py b/conftest.py index 307a9bcad4..54b243bd96 100644 --- a/conftest.py +++ b/conftest.py @@ -73,9 +73,15 @@ def pytest_collection_modifyitems(items) -> None: def fake_telemetry_emitters(monkeypatch, tmp_path): """Default every workspace test to in-memory telemetry, never production.""" from huggingface_hub import constants as hf_constants + from microcosm_emitter.host import launcher from microcosm.build import telemetry_emitter - from test_support.microcosm_build.telemetry import FakeTelemetryEmitter + from microcosm.build.uk_runtime import orrery_publication + from test_support.microcosm_build.telemetry import ( + FakeTelemetryEmitter, + fake_publication_inventory, + fake_publish_graph, + ) for name in ("HF_TOKEN", "HUGGING_FACE_HUB_TOKEN", "HUGGINGFACE_TOKEN"): monkeypatch.delenv(name, raising=False) @@ -102,6 +108,15 @@ def start(_cls, **kwargs): monkeypatch.setattr( telemetry_emitter.LocalTelemetryEmitter, "start", classmethod(start) ) + monkeypatch.setattr( + orrery_publication, "publication_inventory", fake_publication_inventory + ) + monkeypatch.setattr(orrery_publication, "publish_graph", fake_publish_graph) + + def unexpected_start(*args, **options): + pytest.fail("A build test tried to start a real emitter service") + + monkeypatch.setattr(launcher, "start_services", unexpected_start) # Reset any cached handle left by a previously loaded build entrypoint. for module in tuple(sys.modules.values()): if isinstance( diff --git a/docs/agent-guide.md b/docs/agent-guide.md index e0aa80a485..1f9e195c85 100644 --- a/docs/agent-guide.md +++ b/docs/agent-guide.md @@ -15,7 +15,7 @@ the PEP 420 namespace `microcosm.`: `frame`, `fit`, `calibrate`, `build`, ## Commands UK build completion exports and stages Orrery graph evidence. Publication uses -an independent worker inside the external local provider and a durable queue; +an independent worker inside the external `microcosm-emitter` service and a durable queue; see [configuration and retry](orrery-publication.md). No new producer secret or environment variable is required. Keep graph export/publication tests independent of the hosted runs app and collector; use synthetic files and fake HTTP. @@ -98,7 +98,7 @@ These CI tests perform no browser rendering. Workspace tests use an in-memory telemetry emitter by default. The shared fixture isolates Hugging Face credentials and cache paths. Adapter tests replace -provider startup and publication APIs with fakes. Provider process, persistence +emitter startup, inventory and publication APIs with fakes. Emitter process, persistence and HTTP-delivery tests belong in microcosm-emitter; do not copy them into Microcosm. Build tests inspect `fake_telemetry_emitters` without creating sockets or service processes. diff --git a/docs/orrery-publication.md b/docs/orrery-publication.md index 4bf8ce7e38..7652f5b5d8 100644 --- a/docs/orrery-publication.md +++ b/docs/orrery-publication.md @@ -6,14 +6,14 @@ public-safe file inventory in the attempt directory, attempt publication before Hugging Face artifact staging, and register graph/evidence files and an `orrery.publication.json` receipt snapshot in the artifact manifest. -Microcosm's metadata adapter starts one provider service with two independent -workers: live telemetry and completed graph publication. The provider packages +Microcosm's metadata adapter starts one local emitter service with two independent +workers: live telemetry and completed graph publication. The `microcosm-emitter` package own sockets, authentication, persistence, retries and resource sampling. Microcosm retains graph generation, public-file selection and HF staging. Slow uploads cannot delay socket acknowledgement or telemetry delivery. -Publication IDs do not depend on telemetry run IDs. Jobs use SQLAlchemy and -Alembic in the existing local SQLite database. Pending jobs survive missing or +Publication IDs do not depend on telemetry run IDs. Jobs use SQLAlchemy ORM and +Alembic inside the emitter service's existing local SQLite database. Pending jobs survive missing or rejected credentials, process restarts and telemetry event pruning. Retry leases prevent a stale worker from replacing another worker's result. The exact source bytes must remain available until delivery completes. @@ -30,7 +30,7 @@ aggregate summaries can be published publicly. | Setting | Source and consumer | Provisioning | | --- | --- | --- | | `HF_TOKEN` / `HUGGINGFACE_TOKEN` | Existing operator credential, exchanged by the service for a short-lived collector session | Existing environment or Hugging Face login; never committed | -| Collector origin | Tracked `PRODUCTION_COLLECTOR_URL` in provider core | No producer variable or destination override | +| Collector origin | Tracked `PRODUCTION_COLLECTOR_URL` in the emitter's internal core library | No producer variable or destination override | | Runs origin | Tracked `DEFAULT_GRAPH_PUBLICATION_ORIGIN` | No new producer variable | | `XDG_CACHE_HOME` | Optional existing cache-location setting | Defaults to the user's cache directory | @@ -57,28 +57,30 @@ an upload or stage to Hugging Face. ## Retry ```bash -microcosm-provider-publish-graph --publication-id PUBLICATION_ID \ +microcosm-emitter-publish-graph --publication-id PUBLICATION_ID \ --directory /path/to/attempt/orrery-publication ``` The preserved directory contains `orrery.upload.json` and the exact files. Retries update a separate `publication.status.json`, never the original HF receipt snapshot. Pending jobs also resume when a subsequent local service -starts. The retry command comes from the provider package; Microcosm does not +starts. The retry command comes from the emitter package; Microcosm does not retain a second delivery implementation or a compatibility wrapper. ## Deferred migration prerequisites This adapter is implemented and tested in a draft PR, not merged or activated. -It depends on PolicyEngine/microcosm-emitter#4 (stacked on #2). -The draft uses exact Git source pins in `packages/microcosm-build/pyproject.toml` -and `uv.lock` because provider version 0.1.0 has not been released by this work. -Before merge, verify registry-owner configuration, release all four Python -packages, remove the draft source overrides, regenerate `uv.lock`, update its +It depends on PolicyEngine/microcosm-emitter#4. The draft uses one exact root +Git source pin in `packages/microcosm-build/pyproject.toml` and `uv.lock` for +that PR's single `microcosm-emitter` distribution; no registry release was +available when this draft was updated. +Before merge, merge emitter #4, verify registry-owner configuration, publish +one `microcosm-emitter` release containing those changes, replace the draft source +override with that released version, regenerate `uv.lock`, update its approved worker-identity digest and rerun tests and wheel checks. Merely building a Microcosm wheel does not make its unpublished dependencies installable. -The provider rejects unversioned databases without modification. Before switching +The emitter rejects unversioned databases without modification. Before switching an existing workstation, upgrade any old unversioned queue using the preceding Microcosm implementation. Versioned telemetry and graph jobs keep their existing IDs, eligibility, path and Alembic revision; do not delete a queue to migrate. diff --git a/packages/microcosm-build/pyproject.toml b/packages/microcosm-build/pyproject.toml index c3d1a7eeae..4ed4d66986 100644 --- a/packages/microcosm-build/pyproject.toml +++ b/packages/microcosm-build/pyproject.toml @@ -24,9 +24,7 @@ dependencies = [ "pyyaml>=6", "jsonschema>=4.23,<5", "referencing>=0.35,<1", - "microcosm-provider-client==0.1.0", - "microcosm-provider-telemetry==0.1.0", - "microcosm-provider-orrery==0.1.0", + "microcosm-emitter==0.1.0", ] [project.optional-dependencies] @@ -83,8 +81,5 @@ microcosm-fit = { workspace = true } microcosm-calibrate = { workspace = true } microcosm-diagnostics = { workspace = true } microcosm-data = { workspace = true } -# Draft-only source pins. Replace with published 0.1.0 distributions before merge. -microcosm-provider-client = { git = "https://github.com/PolicyEngine/microcosm-emitter", rev = "06e47847983ee67ccdb6a596062251322d07f793", subdirectory = "packages/provider-client" } -microcosm-provider-core = { git = "https://github.com/PolicyEngine/microcosm-emitter", rev = "06e47847983ee67ccdb6a596062251322d07f793", subdirectory = "packages/provider-core" } -microcosm-provider-telemetry = { git = "https://github.com/PolicyEngine/microcosm-emitter", rev = "06e47847983ee67ccdb6a596062251322d07f793", subdirectory = "packages/provider-telemetry" } -microcosm-provider-orrery = { git = "https://github.com/PolicyEngine/microcosm-emitter", rev = "06e47847983ee67ccdb6a596062251322d07f793", subdirectory = "packages/provider-orrery" } +# Draft-only root source pin. Replace with a published distribution before merge. +microcosm-emitter = { git = "https://github.com/PolicyEngine/microcosm-emitter", rev = "e522ced07d48d402e77fc63f30b97451c733204f" } diff --git a/packages/microcosm-build/src/microcosm/build/telemetry_emitter.py b/packages/microcosm-build/src/microcosm/build/telemetry_emitter.py index f8b91c6e3d..ff5a4f051f 100644 --- a/packages/microcosm-build/src/microcosm/build/telemetry_emitter.py +++ b/packages/microcosm-build/src/microcosm/build/telemetry_emitter.py @@ -1,21 +1,30 @@ -"""Microcosm metadata adapter for the independently packaged local provider.""" +"""Microcosm metadata adapter for the independently packaged local emitter.""" from __future__ import annotations import sys +from collections.abc import Mapping from pathlib import Path from typing import Any -from microcosm_provider_client.launcher import start_services -from microcosm_provider_orrery.client import default_spool_path, publish_graph -from microcosm_provider_orrery.contracts import PublicationInventory, PublicationReceipt -from microcosm_provider_telemetry.client import ( - LocalTelemetryEmitter as ProviderTelemetryEmitter, +from microcosm_emitter.host.launcher import start_services +from microcosm_emitter.orrery.client import ( + GraphPublicationClient, + default_spool_path, + publish_graph, ) -from microcosm_provider_telemetry.client import ( +from microcosm_emitter.orrery.contracts import ( + PublicationFileRole, + PublicationInventory, + PublicationReceipt, +) +from microcosm_emitter.telemetry.client import ( + LocalTelemetryEmitter as EmitterTelemetryClient, +) +from microcosm_emitter.telemetry.client import ( TelemetryRun, ) -from microcosm_provider_telemetry.constants import ( +from microcosm_emitter.telemetry.constants import ( DEFAULT_HEARTBEAT_SECONDS, SERVICE_START_WARNING, ) @@ -23,12 +32,14 @@ from microcosm.build.telemetry_identity import runtime_identity -class LocalTelemetryEmitter(ProviderTelemetryEmitter): - """Supply build identity and select both workers in one provider process.""" +class LocalTelemetryEmitter(EmitterTelemetryClient): + """Supply build identity and select both workers in one emitter process.""" - def __init__(self, *, run, transport, handle=None, spool_path=None): + def __init__(self, *, run, transport, handle=None, graph_client=None): super().__init__(run=run, transport=transport, handle=handle) - self._spool_path = Path(spool_path) if spool_path else None + self._graph_client = ( + graph_client if graph_client is not None else GraphPublicationClient() + ) @classmethod def start( @@ -53,7 +64,6 @@ def start( release_id=release_id, run_kind=run_kind, ) - path = None try: path = Path(spool_path) if spool_path else default_spool_path() identity = runtime_identity() @@ -66,6 +76,7 @@ def start( "telemetry": { **shared, "registration": run.as_registration(), + "identity": identity, "heartbeat_seconds": heartbeat_seconds, }, "orrery": shared, @@ -77,21 +88,19 @@ def start( SERVICE_START_WARNING.format(error_type=type(error).__name__), file=sys.stderr, ) - return cls(run=run, transport=None, spool_path=path) - emitter = cls( + return cls(run=run, transport=None) + return cls( run=run, transport=handle.client.for_module("telemetry"), handle=handle, - spool_path=path, + graph_client=GraphPublicationClient(handle.client.for_module("orrery")), ) - emitter.emit( - event_type="run", - stage_id="created", - status="started", - message="Build started", - details={"identity": identity}, - ) - return emitter + + def publication_inventory( + self, directory: Path, roles: Mapping[str, PublicationFileRole] + ) -> PublicationInventory | None: + """Ask the existing graph worker to measure selected public files.""" + return self._graph_client.inventory(directory, roles) def publish_graph( self, @@ -103,12 +112,11 @@ def publish_graph( return publish_graph( directory, inventory, - spool_path=self._spool_path, - available=lambda: self.available, + client=self._graph_client, wait_seconds=wait_seconds, ) def start_local_telemetry_emitter_service(**run: Any) -> LocalTelemetryEmitter: - """Construct the build adapter without importing provider service internals.""" + """Construct the build adapter without importing emitter service internals.""" return LocalTelemetryEmitter.start(**run) diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/orrery_publication.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/orrery_publication.py index 97a2f7480b..3cd1be5c4b 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/orrery_publication.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/orrery_publication.py @@ -15,8 +15,8 @@ from pathlib import Path from typing import Any, TypedDict -from microcosm_provider_orrery.client import publication_inventory, publish_graph -from microcosm_provider_orrery.contracts import ( +from microcosm_emitter.orrery.client import publication_inventory, publish_graph +from microcosm_emitter.orrery.contracts import ( PublicationFileRole, PublicationReceipt, ) @@ -153,7 +153,20 @@ def finalize_graph( preserved = Path(args.attempt_evidence) / "orrery-publication" for name in roles: materialize_bytes((output / name).read_bytes(), preserved / name) - inventory = publication_inventory(preserved, roles) + emitter = rowwise_staging.active_telemetry_emitter() + inventory = ( + publication_inventory(preserved, roles) + if emitter is None + else emitter.publication_inventory(preserved, roles) + ) + if inventory is None: + receipt.update( + status="failed", + error_code="local_service_unavailable", + preserved_directory=str(preserved), + recorded_execution=execution is not None, + ) + raise RuntimeError("Local graph inventory service is unavailable.") materialize_bytes(canonical_json(inventory), preserved / INVENTORY_NAME) materialize_bytes(canonical_json(inventory), output / INVENTORY_NAME) names = [*roles, INVENTORY_NAME] @@ -167,7 +180,6 @@ def finalize_graph( "recorded_execution": execution is not None, } if publication_enabled(args): - emitter = rowwise_staging.active_telemetry_emitter() if emitter is None: # Durable enqueue is still possible without a running # emitter; the retry command requires no telemetry run. @@ -181,8 +193,9 @@ def finalize_graph( except Exception: # No credential, exception text, paths to raw input data, or signed URL # is copied from an exception into public evidence or telemetry. + if receipt["status"] != "failed": + receipt["error_code"] = "graph_export_or_enqueue_failed" receipt["status"] = "failed" - receipt["error_code"] = "graph_export_or_enqueue_failed" materialize_bytes(canonical_json(receipt), output / RECEIPT_NAME) names.append(RECEIPT_NAME) if manifest is not None: diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/worker_identity.py b/packages/microcosm-build/src/microcosm/build/us_runtime/worker_identity.py index 2124fb6a40..c31e531123 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/worker_identity.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/worker_identity.py @@ -28,7 +28,7 @@ PRIMARY_QRF_WORKER_MODULE = "microcosm.build.us_runtime.puf_qrf_worker" PRIMARY_QRF_INTERPRETER_PLACEHOLDER = "{python_interpreter}" APPROVED_UV_LOCK_SHA256 = ( - "95346afaf219870b7428fea331f74533dfbc2eb7e9549d654fa968a39e023394" + "c5cb3c787c963b10e3502b3f00f393371886dcfa67d7feb13f521ff01f56b581" ) LEGACY_CAMPAIGN_UV_LOCK_SHA256 = ( "27f47e385cfa35e2644a37410d1804b361ad9aee123577551c8421547bda65ee" diff --git a/packages/microcosm-build/tests/engine_free/shared/test_emitter_adapter.py b/packages/microcosm-build/tests/engine_free/shared/test_emitter_adapter.py new file mode 100644 index 0000000000..dd51bf8f43 --- /dev/null +++ b/packages/microcosm-build/tests/engine_free/shared/test_emitter_adapter.py @@ -0,0 +1,244 @@ +"""Test Microcosm's adapter with emitter APIs replaced, not hosted services.""" + +from types import SimpleNamespace + +import pytest + +from microcosm.build import telemetry_emitter as adapter + +START = adapter.LocalTelemetryEmitter.start.__func__ + + +def test_selects_both_workers_once_with_shared_path_and_build_identity( + monkeypatch, tmp_path +): + calls, messages, queries, closed = [], [], [], [] + transport = SimpleNamespace(send=messages.append) + inventory = { + "version": 1, + "publication_id": "independent", + "files": [], + } + + def request(message, **options): + queries.append((message, options)) + if message["command"] == "inventory": + return {"inventory": inventory} + return { + "receipt": { + "version": 1, + "publication_id": "independent", + "status": "pending", + "error_code": None, + "attempts": 1, + } + } + + graph_transport = SimpleNamespace(request=request) + + def for_module(name): + return {"telemetry": transport, "orrery": graph_transport}[name] + + handle = SimpleNamespace( + client=SimpleNamespace(for_module=for_module), close=lambda: closed.append(True) + ) + + def start(modules, **options): + calls.append((modules, options)) + return handle + + monkeypatch.setattr(adapter, "start_services", start) + monkeypatch.setattr(adapter, "runtime_identity", lambda: {"commit": "synthetic"}) + path = tmp_path / "jobs.sqlite3" + emitter = START( + adapter.LocalTelemetryEmitter, + run_id="run-one", + country_code="uk", + pipeline="full", + spool_path=path, + heartbeat_seconds=123, + ) + assert len(calls) == 1 + modules, options = calls[0] + assert set(modules) == {"telemetry", "orrery"} + assert all(m["spool_path"] == str(path) for m in modules.values()) + assert "registration" not in modules["orrery"] + assert modules["telemetry"]["registration"]["run_id"] == "run-one" + assert modules["telemetry"]["heartbeat_seconds"] == 123 + assert modules["telemetry"]["identity"] == {"commit": "synthetic"} + assert options["startup_timeout"] == 3 + # The service emits one run-start event from its startup configuration. + # The build adapter must not send a second run-start request. + assert messages == [] + emitter.stage("stage-that-is-not-in-a-fixed-list") + assert messages[0]["stage_id"] == "stage-that-is-not-in-a-fixed-list" + assert emitter.publication_inventory(tmp_path, {}) == inventory + receipt = emitter.publish_graph(tmp_path, inventory, wait_seconds=0) + assert receipt["status"] == "pending" + assert [query[0]["command"] for query in queries] == ["inventory", "enqueue"] + assert not path.exists() + emitter.complete() + emitter.close() + assert closed == [True] + assert messages[-1]["command"] == "complete" + + +def test_disabled_startup_cannot_claim_persistence_or_start_a_replacement_service( + monkeypatch, tmp_path +): + monkeypatch.setattr( + adapter.GraphPublicationClient, + "start", + lambda **options: pytest.fail("started a replacement graph service"), + ) + emitter = adapter.LocalTelemetryEmitter( + run=adapter.TelemetryRun("run-one", "uk", "full"), + transport=None, + ) + value = {"version": 1, "publication_id": "independent", "files": []} + assert emitter.publication_inventory(tmp_path, {}) is None + receipt = emitter.publish_graph(tmp_path, value, wait_seconds=0) + assert receipt == { + "version": 1, + "publication_id": "independent", + "status": "failed", + "error_code": "local_service_unavailable", + } + assert list(tmp_path.iterdir()) == [] + + +def test_unavailable_graph_socket_returns_failure_without_writing_a_queue(tmp_path): + from microcosm_emitter.host.errors import LocalServiceError + + def unavailable(*args, **options): + raise LocalServiceError("socket unavailable") + + emitter = adapter.LocalTelemetryEmitter( + run=adapter.TelemetryRun("run-one", "uk", "full"), + transport=SimpleNamespace(send=lambda message: None), + graph_client=adapter.GraphPublicationClient( + SimpleNamespace(request=unavailable) + ), + ) + assert emitter.publication_inventory(tmp_path, {}) is None + receipt = emitter.publish_graph( + tmp_path, + {"version": 1, "publication_id": "independent", "files": []}, + wait_seconds=0, + ) + assert receipt["status"] == "failed" + assert receipt["error_code"] == "local_service_unavailable" + assert list(tmp_path.iterdir()) == [] + + +def test_start_failure_is_nonfatal_and_does_not_log_exception_payload( + monkeypatch, capsys, tmp_path +): + def fail(*args, **kwargs): + raise RuntimeError("private credential should not be logged") + + monkeypatch.setattr(adapter, "start_services", fail) + emitter = START( + adapter.LocalTelemetryEmitter, + run_id="run-one", + country_code="uk", + pipeline="full", + spool_path=tmp_path / "jobs.sqlite3", + ) + assert not emitter.available + warning = capsys.readouterr().err + assert "RuntimeError" in warning + assert "private credential" not in warning + + +@pytest.mark.parametrize("operation", ["complete", "close"]) +def test_default_test_emitter_uses_emitter_lifecycle_in_memory( + fake_telemetry_emitters, operation +): + emitter = adapter.LocalTelemetryEmitter.start( + run_id="run-one", country_code="uk", pipeline="full" + ) + emitter.stage("one") + getattr(emitter, operation)() + assert not emitter.available + assert emitter.events[0]["stage_id"] == "one" + + +def test_fixture_intercepts_preimported_build_entrypoint( + monkeypatch, fake_telemetry_emitters +): + from microcosm.build.uk_runtime import rowwise_staging + from test_support.microcosm_build.telemetry import FakeTelemetryEmitter + + def fail(*args, **kwargs): + pytest.fail("test started a real emitter") + + monkeypatch.setattr(adapter, "start_services", fail) + for start in ( + rowwise_staging.start_local_telemetry_emitter_service, + adapter.LocalTelemetryEmitter.start, + ): + emitter = start(run_id="fake", country_code="GB", pipeline="test") + assert isinstance(emitter, FakeTelemetryEmitter) + assert emitter in fake_telemetry_emitters + + +def test_fixture_isolates_cached_huggingface_credentials(tmp_path): + from pathlib import Path + + from huggingface_hub import constants, get_token + + assert Path(constants.HF_TOKEN_PATH).is_relative_to(tmp_path) + assert get_token() is None + + +@pytest.mark.parametrize("failure", ["default_spool_path", "runtime_identity"]) +def test_metadata_or_cache_failure_never_starts_a_service(monkeypatch, failure): + def fail(): + raise OSError("private setup detail") + + monkeypatch.setattr(adapter, failure, fail) + monkeypatch.setattr( + adapter, "start_services", lambda *a, **kw: pytest.fail("started after failure") + ) + emitter = START( + adapter.LocalTelemetryEmitter, + run_id="failed", + country_code="uk", + pipeline="full", + ) + assert not emitter.available + + +def test_graph_receipt_poll_uses_the_existing_module_client(monkeypatch, tmp_path): + from microcosm_emitter.orrery import client + + calls = [] + + def request(message, **options): + calls.append(message["command"]) + return { + "receipt": { + "version": 1, + "publication_id": "independent", + "status": ( + "pending" if message["command"] == "enqueue" else "published" + ), + "error_code": None, + "attempts": 0 if message["command"] == "enqueue" else 1, + } + } + + monkeypatch.setattr(client.time, "sleep", lambda duration: None) + emitter = adapter.LocalTelemetryEmitter( + run=adapter.TelemetryRun("run-one", "uk", "full"), + transport=SimpleNamespace(send=lambda message: None), + graph_client=adapter.GraphPublicationClient(SimpleNamespace(request=request)), + ) + receipt = emitter.publish_graph( + tmp_path, + {"version": 1, "publication_id": "independent", "files": []}, + wait_seconds=1, + ) + assert receipt["status"] == "published" + assert calls == ["enqueue", "receipt"] diff --git a/packages/microcosm-build/tests/engine_free/shared/test_provider_adapter.py b/packages/microcosm-build/tests/engine_free/shared/test_provider_adapter.py deleted file mode 100644 index bb0b3f3558..0000000000 --- a/packages/microcosm-build/tests/engine_free/shared/test_provider_adapter.py +++ /dev/null @@ -1,157 +0,0 @@ -"""Test Microcosm's adapter with provider APIs replaced, not hosted services.""" - -from types import SimpleNamespace - -import pytest - -from microcosm.build import telemetry_emitter as adapter - -START = adapter.LocalTelemetryEmitter.start.__func__ - - -def test_selects_both_workers_once_with_shared_path_and_build_identity( - monkeypatch, tmp_path -): - calls, messages, closed = [], [], [] - transport = SimpleNamespace(send=messages.append) - - def for_module(name): - assert name == "telemetry" - return transport - - handle = SimpleNamespace( - client=SimpleNamespace(for_module=for_module), close=lambda: closed.append(True) - ) - - def start(modules, **options): - calls.append((modules, options)) - return handle - - monkeypatch.setattr(adapter, "start_services", start) - monkeypatch.setattr(adapter, "runtime_identity", lambda: {"commit": "synthetic"}) - path = tmp_path / "jobs.sqlite3" - emitter = START( - adapter.LocalTelemetryEmitter, - run_id="run-one", - country_code="uk", - pipeline="full", - spool_path=path, - heartbeat_seconds=123, - ) - assert len(calls) == 1 - modules, options = calls[0] - assert set(modules) == {"telemetry", "orrery"} - assert all(m["spool_path"] == str(path) for m in modules.values()) - assert "registration" not in modules["orrery"] - assert modules["telemetry"]["registration"]["run_id"] == "run-one" - assert modules["telemetry"]["heartbeat_seconds"] == 123 - assert options["startup_timeout"] == 3 - assert messages[0]["details"] == {"identity": {"commit": "synthetic"}} - emitter.complete() - emitter.close() - assert closed == [True] - assert messages[-1]["status"] == "completed" - - -def test_disabled_telemetry_still_delegates_durable_graph_enqueue( - monkeypatch, tmp_path -): - calls = [] - - def publish(directory, inventory, **options): - calls.append((directory, inventory, options)) - assert options["available"]() is False - return {"status": "pending", "publication_id": "independent"} - - monkeypatch.setattr(adapter, "publish_graph", publish) - path = tmp_path / "jobs.sqlite3" - emitter = adapter.LocalTelemetryEmitter( - run=adapter.TelemetryRun("run-one", "uk", "full"), - transport=None, - spool_path=path, - ) - value = {"publication_id": "independent"} - receipt = emitter.publish_graph(tmp_path, value, wait_seconds=0) - assert receipt["status"] == "pending" - assert calls[0][0:2] == (tmp_path, value) - assert calls[0][2]["spool_path"] == path - assert calls[0][2]["wait_seconds"] == 0 - - -def test_start_failure_is_nonfatal_and_does_not_log_exception_payload( - monkeypatch, capsys, tmp_path -): - def fail(*args, **kwargs): - raise RuntimeError("private credential should not be logged") - - monkeypatch.setattr(adapter, "start_services", fail) - emitter = START( - adapter.LocalTelemetryEmitter, - run_id="run-one", - country_code="uk", - pipeline="full", - spool_path=tmp_path / "jobs.sqlite3", - ) - assert not emitter.available - warning = capsys.readouterr().err - assert "RuntimeError" in warning - assert "private credential" not in warning - - -@pytest.mark.parametrize("operation", ["complete", "close"]) -def test_default_test_emitter_uses_provider_lifecycle_in_memory( - fake_telemetry_emitters, operation -): - emitter = adapter.LocalTelemetryEmitter.start( - run_id="run-one", country_code="uk", pipeline="full" - ) - emitter.stage("one") - getattr(emitter, operation)() - assert not emitter.available - assert emitter.events[0]["stage_id"] == "one" - - -def test_fixture_intercepts_preimported_build_entrypoint( - monkeypatch, fake_telemetry_emitters -): - from microcosm.build.uk_runtime import rowwise_staging - from test_support.microcosm_build.telemetry import FakeTelemetryEmitter - - def fail(*args, **kwargs): - pytest.fail("test started a real provider") - - monkeypatch.setattr(adapter, "start_services", fail) - for start in ( - rowwise_staging.start_local_telemetry_emitter_service, - adapter.LocalTelemetryEmitter.start, - ): - emitter = start(run_id="fake", country_code="GB", pipeline="test") - assert isinstance(emitter, FakeTelemetryEmitter) - assert emitter in fake_telemetry_emitters - - -def test_fixture_isolates_cached_huggingface_credentials(tmp_path): - from pathlib import Path - - from huggingface_hub import constants, get_token - - assert Path(constants.HF_TOKEN_PATH).is_relative_to(tmp_path) - assert get_token() is None - - -@pytest.mark.parametrize("failure", ["default_spool_path", "runtime_identity"]) -def test_metadata_or_cache_failure_never_starts_a_service(monkeypatch, failure): - def fail(): - raise OSError("private setup detail") - - monkeypatch.setattr(adapter, failure, fail) - monkeypatch.setattr( - adapter, "start_services", lambda *a, **kw: pytest.fail("started after failure") - ) - emitter = START( - adapter.LocalTelemetryEmitter, - run_id="failed", - country_code="uk", - pipeline="full", - ) - assert not emitter.available diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_orrery_publication.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_orrery_publication.py index 41e54f4549..d35c21ea5e 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_orrery_publication.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_orrery_publication.py @@ -12,6 +12,7 @@ ) from microcosm.build.uk_runtime.orrery_contract import save_uk_graph_schema from microcosm.graph import Graph, Node, Owned, SourceRef, StructuralDelta +from test_support.microcosm_build.telemetry import fake_publication_inventory def fixture(tmp_path, **options): @@ -78,6 +79,8 @@ def test_publication_precedes_manifest_registration_and_receipt_is_a_snapshot( events = [] class Emitter: + publication_inventory = staticmethod(fake_publication_inventory) + def publish_graph(self, directory, inventory, *, wait_seconds): assert not (args.out / "orrery.publication.json").exists() events.append("publication_attempt") @@ -166,6 +169,8 @@ def test_interrupt_preserves_local_evidence_without_waiting_or_staging( waits = [] class Emitter: + publication_inventory = staticmethod(fake_publication_inventory) + def publish_graph(self, directory, inventory, *, wait_seconds): waits.append(wait_seconds) return { @@ -246,3 +251,75 @@ def test_publication_has_explicit_local_only_and_opt_out_controls( publish_orrery=explicit, staging_local_only=local, no_staging=disabled ) assert orrery_publication.publication_enabled(args) is expected + + +@pytest.mark.parametrize("active", [False, True]) +def test_unavailable_inventory_preserves_graph_without_claiming_a_queued_job( + tmp_path, monkeypatch, active +): + args = fixture(tmp_path) + args.publish_orrery = True + calls = [] + + def unavailable(directory, roles): + calls.append(directory) + return None + + def must_not_publish(*args, **options): + pytest.fail("publication attempted without a measured inventory") + + if active: + from types import SimpleNamespace + + emitter = SimpleNamespace( + publication_inventory=unavailable, publish_graph=must_not_publish + ) + monkeypatch.setattr(rowwise_staging, "_ACTIVE_EMITTER", emitter) + monkeypatch.setattr( + orrery_publication, + "publication_inventory", + lambda *a, **k: pytest.fail("started another inventory service"), + ) + else: + monkeypatch.setattr(rowwise_staging, "_ACTIVE_EMITTER", None) + monkeypatch.setattr(orrery_publication, "publication_inventory", unavailable) + monkeypatch.setattr(orrery_publication, "publish_graph", must_not_publish) + manifest = {"outputs": {}} + result = orrery_publication.finalize_graph(args, manifest=manifest) + assert result.receipt["status"] == "failed" + assert result.receipt["publication_id"] is None + assert result.receipt["error_code"] == "local_service_unavailable" + preserved = args.attempt_evidence / "orrery-publication" + assert calls == [preserved] + for name in ("graph.schema.json", "graph.orrery.json"): + assert (preserved / name).read_bytes() == (args.out / name).read_bytes() + assert f"orrery/{name}" in manifest["outputs"] + assert "orrery.upload.json" not in result.files + assert "orrery/orrery.publication.json" in manifest["outputs"] + + +def test_enqueue_failure_is_recorded_without_replacing_graph_exports( + tmp_path, monkeypatch +): + args = fixture(tmp_path) + args.publish_orrery = True + + class Emitter: + publication_inventory = staticmethod(fake_publication_inventory) + + def publish_graph(self, directory, inventory, *, wait_seconds): + return { + "version": 1, + "publication_id": inventory["publication_id"], + "status": "failed", + "error_code": "local_service_unavailable", + } + + monkeypatch.setattr(rowwise_staging, "_ACTIVE_EMITTER", Emitter()) + result = orrery_publication.finalize_graph(args, manifest=None) + assert result.receipt["status"] == "failed" + assert result.receipt["error_code"] == "local_service_unavailable" + assert result.receipt["publication_id"] + assert (args.out / "graph.orrery.json").is_file() + assert (args.out / "orrery.upload.json").is_file() + assert (args.out / "orrery.evidence-manifest.json").is_file() diff --git a/test_support/microcosm_build/telemetry.py b/test_support/microcosm_build/telemetry.py index bd2e3b2652..4693078473 100644 --- a/test_support/microcosm_build/telemetry.py +++ b/test_support/microcosm_build/telemetry.py @@ -1,30 +1,87 @@ -"""In-memory telemetry and isolated opt-in service startup for tests.""" +"""In-memory telemetry and synthetic graph requests for build tests.""" from __future__ import annotations +import hashlib +import uuid from collections.abc import Mapping from pathlib import Path from typing import Any +from microcosm_emitter.orrery.contracts import ( + PublicationFileRole, + PublicationInventory, + PublicationReceipt, +) +from microcosm_emitter.telemetry.service.lifecycle import ( + LifecycleState, + process_request, +) + from microcosm.build.telemetry_emitter import LocalTelemetryEmitter, TelemetryRun +def fake_publication_inventory( + directory: Path, roles: Mapping[str, PublicationFileRole] +) -> PublicationInventory: + """Measure only synthetic test files without starting an emitter service.""" + return { + "version": 1, + "publication_id": uuid.uuid4().hex, + "files": [ + { + "name": name, + "role": role, + "bytes": (directory / name).stat().st_size, + "sha256": hashlib.sha256((directory / name).read_bytes()).hexdigest(), + } + for name, role in sorted(roles.items()) + ], + } + + +def fake_publish_graph( + directory: Path, + inventory: PublicationInventory, + *, + wait_seconds: float = 30.0, +) -> PublicationReceipt: + """Return an in-memory receipt without persistence, credentials or HTTP.""" + return { + "version": 1, + "publication_id": inventory["publication_id"], + "status": "pending", + "error_code": None, + } + + class FakeTelemetryEmitter(LocalTelemetryEmitter): """Exercise real lifecycle methods without starting a service or socket.""" def __init__(self, run: TelemetryRun) -> None: self.messages: list[Mapping[str, Any]] = [] + self._lifecycle = LifecycleState() super().__init__(run=run, transport=self) def send(self, payload: Mapping[str, Any]) -> None: - self.messages.append({"action": "event", "event": payload}) + self._lifecycle, events = process_request(self._lifecycle, payload) + self.messages.extend({"action": "event", "event": event} for event in events) def close(self) -> None: self._closed = True + def publication_inventory( + self, directory: Path, roles: Mapping[str, PublicationFileRole] + ) -> PublicationInventory: + return fake_publication_inventory(directory, roles) + def publish_graph( - self, directory: Path, inventory: dict, *, wait_seconds: float = 30.0 - ) -> dict: + self, + directory: Path, + inventory: PublicationInventory, + *, + wait_seconds: float = 30.0, + ) -> PublicationReceipt: self.messages.append( { "action": "graph_publication", @@ -32,12 +89,7 @@ def publish_graph( "inventory": inventory, } ) - return { - "version": 1, - "publication_id": inventory["publication_id"], - "status": "pending", - "error_code": None, - } + return fake_publish_graph(directory, inventory, wait_seconds=wait_seconds) @property def events(self) -> list[Mapping[str, Any]]: diff --git a/uv.lock b/uv.lock index 1f65b8dfce..b30a79208b 100644 --- a/uv.lock +++ b/uv.lock @@ -804,12 +804,10 @@ dependencies = [ { name = "jsonschema" }, { name = "microcosm-calibrate" }, { name = "microcosm-diagnostics" }, + { name = "microcosm-emitter" }, { name = "microcosm-fit" }, { name = "microcosm-frame" }, { name = "microcosm-graph" }, - { name = "microcosm-provider-client" }, - { name = "microcosm-provider-orrery" }, - { name = "microcosm-provider-telemetry" }, { name = "numpy" }, { name = "pandas" }, { name = "pyarrow" }, @@ -850,12 +848,10 @@ requires-dist = [ { name = "microcosm-data", marker = "extra == 'uk'", editable = "packages/microcosm-data" }, { name = "microcosm-data", marker = "extra == 'us'", editable = "packages/microcosm-data" }, { name = "microcosm-diagnostics", editable = "packages/microcosm-diagnostics" }, + { name = "microcosm-emitter", git = "https://github.com/PolicyEngine/microcosm-emitter?rev=e522ced07d48d402e77fc63f30b97451c733204f" }, { name = "microcosm-fit", editable = "packages/microcosm-fit" }, { name = "microcosm-frame", editable = "packages/microcosm-frame" }, { name = "microcosm-graph", editable = "packages/microcosm-graph" }, - { name = "microcosm-provider-client", git = "https://github.com/PolicyEngine/microcosm-emitter?subdirectory=packages%2Fprovider-client&rev=06e47847983ee67ccdb6a596062251322d07f793" }, - { name = "microcosm-provider-orrery", git = "https://github.com/PolicyEngine/microcosm-emitter?subdirectory=packages%2Fprovider-orrery&rev=06e47847983ee67ccdb6a596062251322d07f793" }, - { name = "microcosm-provider-telemetry", git = "https://github.com/PolicyEngine/microcosm-emitter?subdirectory=packages%2Fprovider-telemetry&rev=06e47847983ee67ccdb6a596062251322d07f793" }, { name = "microunit", marker = "extra == 'us'", specifier = ">=0.1.0" }, { name = "numpy", specifier = ">=1.26" }, { name = "openpyxl", marker = "extra == 'uk'", specifier = ">=3.1" }, @@ -967,6 +963,18 @@ requires-dist = [{ name = "pydantic", specifier = ">=2.10,<3" }] [package.metadata.requires-dev] dev = [{ name = "pytest", specifier = ">=8" }] +[[package]] +name = "microcosm-emitter" +version = "0.1.0" +source = { git = "https://github.com/PolicyEngine/microcosm-emitter?rev=e522ced07d48d402e77fc63f30b97451c733204f#e522ced07d48d402e77fc63f30b97451c733204f" } +dependencies = [ + { name = "alembic" }, + { name = "httpx" }, + { name = "huggingface-hub" }, + { name = "psutil" }, + { name = "sqlalchemy" }, +] + [[package]] name = "microcosm-fit" version = "0.1.0" @@ -1081,47 +1089,6 @@ dev = [ { name = "pytest", specifier = ">=8" }, ] -[[package]] -name = "microcosm-provider-client" -version = "0.1.0" -source = { git = "https://github.com/PolicyEngine/microcosm-emitter?subdirectory=packages%2Fprovider-client&rev=06e47847983ee67ccdb6a596062251322d07f793#06e47847983ee67ccdb6a596062251322d07f793" } -dependencies = [ - { name = "psutil" }, -] - -[[package]] -name = "microcosm-provider-core" -version = "0.1.0" -source = { git = "https://github.com/PolicyEngine/microcosm-emitter?subdirectory=packages%2Fprovider-core&rev=06e47847983ee67ccdb6a596062251322d07f793#06e47847983ee67ccdb6a596062251322d07f793" } -dependencies = [ - { name = "alembic" }, - { name = "huggingface-hub" }, - { name = "sqlalchemy" }, -] - -[[package]] -name = "microcosm-provider-orrery" -version = "0.1.0" -source = { git = "https://github.com/PolicyEngine/microcosm-emitter?subdirectory=packages%2Fprovider-orrery&rev=06e47847983ee67ccdb6a596062251322d07f793#06e47847983ee67ccdb6a596062251322d07f793" } -dependencies = [ - { name = "httpx" }, - { name = "microcosm-provider-client" }, - { name = "microcosm-provider-core" }, -] - -[[package]] -name = "microcosm-provider-telemetry" -version = "0.1.0" -source = { git = "https://github.com/PolicyEngine/microcosm-emitter?subdirectory=packages%2Fprovider-telemetry&rev=06e47847983ee67ccdb6a596062251322d07f793#06e47847983ee67ccdb6a596062251322d07f793" } -dependencies = [ - { name = "alembic" }, - { name = "huggingface-hub" }, - { name = "microcosm-provider-client" }, - { name = "microcosm-provider-core" }, - { name = "psutil" }, - { name = "sqlalchemy" }, -] - [[package]] name = "microcosm-workspace" version = "0.0.0" From bd0ef2209691a8df6dbad6169cd04c731ae0ac49 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Mon, 12 Oct 2026 18:52:38 +0400 Subject: [PATCH 3/4] Fix issues from review: preserve emitter outcomes and cleanup ownership --- .../1153-outcome-compatibility.fixed.md | 1 + conftest.py | 4 +- docs/agent-guide.md | 6 + docs/orrery-publication.md | 7 +- packages/microcosm-build/pyproject.toml | 2 +- .../src/microcosm/build/telemetry_emitter.py | 25 ++- .../build/uk_runtime/full_build_cli.py | 1 + .../build/uk_runtime/rowwise_staging.py | 2 + .../build/us_runtime/worker_identity.py | 2 +- .../shared/test_emitter_adapter.py | 150 ++++++++++++++++++ .../engine_free/uk/test_uk_full_build_cli.py | 4 +- test_support/microcosm_build/telemetry.py | 10 +- uv.lock | 4 +- 13 files changed, 205 insertions(+), 13 deletions(-) create mode 100644 changelog.d/1153-outcome-compatibility.fixed.md diff --git a/changelog.d/1153-outcome-compatibility.fixed.md b/changelog.d/1153-outcome-compatibility.fixed.md new file mode 100644 index 0000000000..0bcb756462 --- /dev/null +++ b/changelog.d/1153-outcome-compatibility.fixed.md @@ -0,0 +1 @@ +Preserve validation-refused and classified-failure telemetry when using the external emitter, and keep its graph worker available until the build's final cleanup. diff --git a/conftest.py b/conftest.py index 54b243bd96..f434d79435 100644 --- a/conftest.py +++ b/conftest.py @@ -101,7 +101,9 @@ def start(_cls, **kwargs): run = telemetry_emitter.TelemetryRun( **{name: value for name, value in kwargs.items() if name in fields} ) - emitter = FakeTelemetryEmitter(run) + emitter = FakeTelemetryEmitter( + run, close_service_on_finish=kwargs.get("close_service_on_finish", True) + ) emitters.append(emitter) return emitter diff --git a/docs/agent-guide.md b/docs/agent-guide.md index 1f9e195c85..64cecc11c6 100644 --- a/docs/agent-guide.md +++ b/docs/agent-guide.md @@ -103,6 +103,12 @@ and HTTP-delivery tests belong in microcosm-emitter; do not copy them into Microcosm. Build tests inspect `fake_telemetry_emitters` without creating sockets or service processes. +The UK full-build caller sets `close_service_on_finish=False`: reporting an outcome +leaves the graph worker available, and that caller's existing `finally` cleanup +closes the combined service. Other callers close automatically when they finish. +Keep validation-refused outcomes and structured +failure codes intact when changing the adapter. + New commits to a PR cancel older unfinished CI runs for that same PR. Each main-push run has a unique concurrency group, so all main-push runs remain independent and can finish validating their merged changes. diff --git a/docs/orrery-publication.md b/docs/orrery-publication.md index 7652f5b5d8..8ca1a72409 100644 --- a/docs/orrery-publication.md +++ b/docs/orrery-publication.md @@ -8,10 +8,15 @@ Hugging Face artifact staging, and register graph/evidence files and an Microcosm's metadata adapter starts one local emitter service with two independent workers: live telemetry and completed graph publication. The `microcosm-emitter` package -own sockets, authentication, persistence, retries and resource sampling. +owns sockets, authentication, persistence, retries and resource sampling. Microcosm retains graph generation, public-file selection and HF staging. Slow uploads cannot delay socket acknowledgement or telemetry delivery. +UK full-build commands explicitly defer process shutdown when recording a completed, +validation-refused or failed telemetry outcome. Their final cleanup closes the combined +process after graph inventory and enqueue requests, so failure evidence uses the same +graph socket. Other callers close the process automatically when reporting their outcome. + Publication IDs do not depend on telemetry run IDs. Jobs use SQLAlchemy ORM and Alembic inside the emitter service's existing local SQLite database. Pending jobs survive missing or rejected credentials, process restarts and telemetry event pruning. Retry diff --git a/packages/microcosm-build/pyproject.toml b/packages/microcosm-build/pyproject.toml index 4ed4d66986..4f5b7cba96 100644 --- a/packages/microcosm-build/pyproject.toml +++ b/packages/microcosm-build/pyproject.toml @@ -82,4 +82,4 @@ microcosm-calibrate = { workspace = true } microcosm-diagnostics = { workspace = true } microcosm-data = { workspace = true } # Draft-only root source pin. Replace with a published distribution before merge. -microcosm-emitter = { git = "https://github.com/PolicyEngine/microcosm-emitter", rev = "e522ced07d48d402e77fc63f30b97451c733204f" } +microcosm-emitter = { git = "https://github.com/PolicyEngine/microcosm-emitter", rev = "7fef7c91641c9fc487e55d0e6b4717eb62a92376" } diff --git a/packages/microcosm-build/src/microcosm/build/telemetry_emitter.py b/packages/microcosm-build/src/microcosm/build/telemetry_emitter.py index ff5a4f051f..559e77838e 100644 --- a/packages/microcosm-build/src/microcosm/build/telemetry_emitter.py +++ b/packages/microcosm-build/src/microcosm/build/telemetry_emitter.py @@ -35,8 +35,21 @@ class LocalTelemetryEmitter(EmitterTelemetryClient): """Supply build identity and select both workers in one emitter process.""" - def __init__(self, *, run, transport, handle=None, graph_client=None): - super().__init__(run=run, transport=transport, handle=handle) + def __init__( + self, + *, + run, + transport, + handle=None, + graph_client=None, + close_service_on_finish: bool = True, + ): + super().__init__( + run=run, + transport=transport, + handle=handle, + close_service_on_finish=close_service_on_finish, + ) self._graph_client = ( graph_client if graph_client is not None else GraphPublicationClient() ) @@ -55,6 +68,7 @@ def start( heartbeat_seconds: float = DEFAULT_HEARTBEAT_SECONDS, startup_timeout_seconds: float = 3.0, spool_path: Path | str | None = None, + close_service_on_finish: bool = True, ) -> LocalTelemetryEmitter: run = TelemetryRun( run_id=run_id, @@ -88,12 +102,17 @@ def start( SERVICE_START_WARNING.format(error_type=type(error).__name__), file=sys.stderr, ) - return cls(run=run, transport=None) + return cls( + run=run, + transport=None, + close_service_on_finish=close_service_on_finish, + ) return cls( run=run, transport=handle.client.for_module("telemetry"), handle=handle, graph_client=GraphPublicationClient(handle.client.for_module("orrery")), + close_service_on_finish=close_service_on_finish, ) def publication_inventory( diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/full_build_cli.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/full_build_cli.py index 78e3107ab1..1ed2d9ec12 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/full_build_cli.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/full_build_cli.py @@ -2912,6 +2912,7 @@ def _run_with_telemetry( args, build_id=build_id, run_kind="dry_run" if args.dry_run else "calibration", + close_service_on_finish=False, ) emitter.transition_stage( "preflight", diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/rowwise_staging.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/rowwise_staging.py index e9a3be5de8..55c80a4565 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/rowwise_staging.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/rowwise_staging.py @@ -233,6 +233,7 @@ def start_telemetry_emitter( *, build_id: str, run_kind: str = "calibration", + close_service_on_finish: bool = True, ) -> LocalTelemetryEmitter: """Start or reuse the process-local emitter for one UK build attempt.""" @@ -253,6 +254,7 @@ def start_telemetry_emitter( pipeline=posture.pipeline, candidate_id=args.staging_candidate_id or build_id, run_kind=run_kind, + close_service_on_finish=close_service_on_finish, ) return _ACTIVE_EMITTER diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/worker_identity.py b/packages/microcosm-build/src/microcosm/build/us_runtime/worker_identity.py index c31e531123..e5fa3d1221 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/worker_identity.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/worker_identity.py @@ -28,7 +28,7 @@ PRIMARY_QRF_WORKER_MODULE = "microcosm.build.us_runtime.puf_qrf_worker" PRIMARY_QRF_INTERPRETER_PLACEHOLDER = "{python_interpreter}" APPROVED_UV_LOCK_SHA256 = ( - "c5cb3c787c963b10e3502b3f00f393371886dcfa67d7feb13f521ff01f56b581" + "4a135f2d8b389a0dd4120f40ad5670d5ad7aa364162f95eb7de0ab71a2615237" ) LEGACY_CAMPAIGN_UV_LOCK_SHA256 = ( "27f47e385cfa35e2644a37410d1804b361ad9aee123577551c8421547bda65ee" diff --git a/packages/microcosm-build/tests/engine_free/shared/test_emitter_adapter.py b/packages/microcosm-build/tests/engine_free/shared/test_emitter_adapter.py index dd51bf8f43..159ffb743c 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_emitter_adapter.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_emitter_adapter.py @@ -242,3 +242,153 @@ def request(message, **options): ) assert receipt["status"] == "published" assert calls == ["enqueue", "receipt"] + + +def test_adapter_assigns_host_cleanup_to_the_build_finally(monkeypatch): + configured = [] + + def initialize(self, **options): + configured.append(options) + + monkeypatch.setattr(adapter.EmitterTelemetryClient, "__init__", initialize) + adapter.LocalTelemetryEmitter( + run=adapter.TelemetryRun("synthetic", "uk", "full"), + transport=None, + close_service_on_finish=False, + ) + assert configured[0]["close_service_on_finish"] is False + + +@pytest.mark.parametrize("outcome", ["blocked", "failed", "completed"]) +def test_graph_publication_precedes_host_cleanup_after_recording_an_outcome( + monkeypatch, tmp_path, outcome +): + from test_support.microcosm_build.telemetry import FakeTelemetryEmitter + + operations, queries = [], [] + telemetry = FakeTelemetryEmitter(adapter.TelemetryRun("synthetic", "uk", "full")) + inventory = {"version": 1, "publication_id": "synthetic", "files": []} + + def request(message, **options): + operations.append(message["command"]) + queries.append(message) + if message["command"] == "inventory": + return {"inventory": inventory} + return { + "receipt": { + "version": 1, + "publication_id": "synthetic", + "status": "pending", + "attempts": 1, + "error_code": None, + } + } + + def for_module(name): + return { + "telemetry": telemetry, + "orrery": SimpleNamespace(request=request), + }[name] + + handle = SimpleNamespace( + client=SimpleNamespace(for_module=for_module), + close=lambda: operations.append("close"), + ) + monkeypatch.setattr(adapter, "start_services", lambda *a, **k: handle) + monkeypatch.setattr(adapter, "runtime_identity", lambda: {"commit": "synthetic"}) + emitter = START( + adapter.LocalTelemetryEmitter, + run_id="synthetic", + country_code="uk", + pipeline="full", + spool_path=tmp_path / "queue.sqlite3", + close_service_on_finish=False, + ) + emitter.transition_stage("solve") + if outcome == "blocked": + emitter.block( + phase="post_calibration", + blocking_gate_ids=["uk.local.fit"], + blocking_failure_count=3, + gate_statuses={"uk.local.fit": "FAIL"}, + ) + elif outcome == "failed": + emitter.fail( + ValueError("synthetic validation refusal"), + error_code="BUILD_REFUSED", + failure_class="refused", + failed_during="solve", + ) + else: + emitter.complete() + assert operations == [] + assert emitter.available + final_events = [event for event in telemetry.events if event["event_type"] == "run"] + assert [event["status"] for event in final_events] == [outcome] + if outcome == "blocked": + assert final_events[0]["details"] == { + "phase": "post_calibration", + "blocking_gate_ids": ["uk.local.fit"], + "blocking_failure_count": 3, + "gate_statuses": {"uk.local.fit": "FAIL"}, + } + assert telemetry.events[-2]["event_type"] == "stage" + assert telemetry.events[-2]["status"] == "completed" + elif outcome == "failed": + assert final_events[0]["details"]["error_code"] == "BUILD_REFUSED" + assert final_events[0]["details"]["failure_class"] == "refused" + assert final_events[0]["details"]["failed_during"] == "solve" + # A completed telemetry lifecycle must not prevent requests on the other + # module; the existing build finally owns physical service shutdown. + try: + assert emitter.publication_inventory(tmp_path, {}) == inventory + assert emitter.publish_graph(tmp_path, inventory, wait_seconds=0)["status"] == ( + "pending" + ) + finally: + emitter.close() + emitter.close() + assert [query["command"] for query in queries] == ["inventory", "enqueue"] + assert operations == ["inventory", "enqueue", "close"] + assert not emitter.available + assert ( + len([event for event in telemetry.events if event["event_type"] == "run"]) == 1 + ) + + +@pytest.mark.parametrize("outcome", ["complete", "fail"]) +def test_default_finish_releases_host_without_an_outer_build_cleanup(outcome): + closed = [] + emitter = adapter.LocalTelemetryEmitter( + run=adapter.TelemetryRun("synthetic", "us", "release"), + transport=SimpleNamespace(send=lambda message: None), + handle=SimpleNamespace(close=lambda: closed.append(True)), + ) + if outcome == "complete": + emitter.complete() + else: + emitter.fail(ValueError("synthetic failure"), error_code="BUILD_FAILED") + assert closed == [True] + assert not emitter.available + emitter.close() + assert closed == [True] + + +@pytest.mark.parametrize("outcome", ["block", "fail"]) +def test_outcome_on_disabled_transport_is_safe_and_cannot_restart_service( + monkeypatch, outcome +): + monkeypatch.setattr( + adapter, + "start_services", + lambda *a, **k: pytest.fail("outcome started a replacement service"), + ) + emitter = adapter.LocalTelemetryEmitter( + run=adapter.TelemetryRun("synthetic", "uk", "full"), transport=None + ) + if outcome == "block": + emitter.block(phase="preflight", blocking_gate_ids=[], blocking_failure_count=1) + else: + emitter.fail(ValueError("refused"), error_code="BUILD_REFUSED") + assert not emitter.available + emitter.close() diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_full_build_cli.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_full_build_cli.py index 9fc790b549..46547b9eb7 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_full_build_cli.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_full_build_cli.py @@ -689,7 +689,7 @@ def close(self): monkeypatch.setattr( cli, "start_telemetry_emitter", - lambda requested, *, build_id, run_kind: ( + lambda requested, *, build_id, run_kind, close_service_on_finish: ( events.append(("start", build_id, run_kind)) or FakeEmitter() ), ) @@ -733,7 +733,7 @@ def close(self): monkeypatch.setattr( cli, "start_telemetry_emitter", - lambda requested, *, build_id, run_kind: ( + lambda requested, *, build_id, run_kind, close_service_on_finish: ( events.append(("start", build_id, run_kind)) or FakeEmitter() ), ) diff --git a/test_support/microcosm_build/telemetry.py b/test_support/microcosm_build/telemetry.py index 4693078473..8cd80791be 100644 --- a/test_support/microcosm_build/telemetry.py +++ b/test_support/microcosm_build/telemetry.py @@ -58,10 +58,16 @@ def fake_publish_graph( class FakeTelemetryEmitter(LocalTelemetryEmitter): """Exercise real lifecycle methods without starting a service or socket.""" - def __init__(self, run: TelemetryRun) -> None: + def __init__( + self, run: TelemetryRun, *, close_service_on_finish: bool = True + ) -> None: self.messages: list[Mapping[str, Any]] = [] self._lifecycle = LifecycleState() - super().__init__(run=run, transport=self) + super().__init__( + run=run, + transport=self, + close_service_on_finish=close_service_on_finish, + ) def send(self, payload: Mapping[str, Any]) -> None: self._lifecycle, events = process_request(self._lifecycle, payload) diff --git a/uv.lock b/uv.lock index b30a79208b..ba51c0ad23 100644 --- a/uv.lock +++ b/uv.lock @@ -848,7 +848,7 @@ requires-dist = [ { name = "microcosm-data", marker = "extra == 'uk'", editable = "packages/microcosm-data" }, { name = "microcosm-data", marker = "extra == 'us'", editable = "packages/microcosm-data" }, { name = "microcosm-diagnostics", editable = "packages/microcosm-diagnostics" }, - { name = "microcosm-emitter", git = "https://github.com/PolicyEngine/microcosm-emitter?rev=e522ced07d48d402e77fc63f30b97451c733204f" }, + { name = "microcosm-emitter", git = "https://github.com/PolicyEngine/microcosm-emitter?rev=7fef7c91641c9fc487e55d0e6b4717eb62a92376" }, { name = "microcosm-fit", editable = "packages/microcosm-fit" }, { name = "microcosm-frame", editable = "packages/microcosm-frame" }, { name = "microcosm-graph", editable = "packages/microcosm-graph" }, @@ -966,7 +966,7 @@ dev = [{ name = "pytest", specifier = ">=8" }] [[package]] name = "microcosm-emitter" version = "0.1.0" -source = { git = "https://github.com/PolicyEngine/microcosm-emitter?rev=e522ced07d48d402e77fc63f30b97451c733204f#e522ced07d48d402e77fc63f30b97451c733204f" } +source = { git = "https://github.com/PolicyEngine/microcosm-emitter?rev=7fef7c91641c9fc487e55d0e6b4717eb62a92376#7fef7c91641c9fc487e55d0e6b4717eb62a92376" } dependencies = [ { name = "alembic" }, { name = "httpx" }, From 3cc562b3622999e7b35d87bd28b7d6a85f30348d Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Tue, 13 Oct 2026 02:42:42 +0400 Subject: [PATCH 4/4] Fix issues from review: pin emitter concurrency fixes --- changelog.d/1153-emitter-concurrency.fixed.md | 1 + packages/microcosm-build/pyproject.toml | 2 +- .../src/microcosm/build/us_runtime/worker_identity.py | 2 +- uv.lock | 4 ++-- 4 files changed, 5 insertions(+), 4 deletions(-) create mode 100644 changelog.d/1153-emitter-concurrency.fixed.md diff --git a/changelog.d/1153-emitter-concurrency.fixed.md b/changelog.d/1153-emitter-concurrency.fixed.md new file mode 100644 index 0000000000..8a90cff51a --- /dev/null +++ b/changelog.d/1153-emitter-concurrency.fixed.md @@ -0,0 +1 @@ +Use the emitter revision that prevents graph file requests from delaying telemetry acknowledgements and prevents older graph status writes from replacing newer results. diff --git a/packages/microcosm-build/pyproject.toml b/packages/microcosm-build/pyproject.toml index 4f5b7cba96..0f40e28b58 100644 --- a/packages/microcosm-build/pyproject.toml +++ b/packages/microcosm-build/pyproject.toml @@ -82,4 +82,4 @@ microcosm-calibrate = { workspace = true } microcosm-diagnostics = { workspace = true } microcosm-data = { workspace = true } # Draft-only root source pin. Replace with a published distribution before merge. -microcosm-emitter = { git = "https://github.com/PolicyEngine/microcosm-emitter", rev = "7fef7c91641c9fc487e55d0e6b4717eb62a92376" } +microcosm-emitter = { git = "https://github.com/PolicyEngine/microcosm-emitter", rev = "12624a118db255405a0ce7397d53b3181deaf49a" } diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/worker_identity.py b/packages/microcosm-build/src/microcosm/build/us_runtime/worker_identity.py index e5fa3d1221..e890126dd1 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/worker_identity.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/worker_identity.py @@ -28,7 +28,7 @@ PRIMARY_QRF_WORKER_MODULE = "microcosm.build.us_runtime.puf_qrf_worker" PRIMARY_QRF_INTERPRETER_PLACEHOLDER = "{python_interpreter}" APPROVED_UV_LOCK_SHA256 = ( - "4a135f2d8b389a0dd4120f40ad5670d5ad7aa364162f95eb7de0ab71a2615237" + "75ffb31c9b711a288e7de3e8f0442047b5aafcdd146c7fd95219f4cc1ac769ae" ) LEGACY_CAMPAIGN_UV_LOCK_SHA256 = ( "27f47e385cfa35e2644a37410d1804b361ad9aee123577551c8421547bda65ee" diff --git a/uv.lock b/uv.lock index ba51c0ad23..fa69800b45 100644 --- a/uv.lock +++ b/uv.lock @@ -848,7 +848,7 @@ requires-dist = [ { name = "microcosm-data", marker = "extra == 'uk'", editable = "packages/microcosm-data" }, { name = "microcosm-data", marker = "extra == 'us'", editable = "packages/microcosm-data" }, { name = "microcosm-diagnostics", editable = "packages/microcosm-diagnostics" }, - { name = "microcosm-emitter", git = "https://github.com/PolicyEngine/microcosm-emitter?rev=7fef7c91641c9fc487e55d0e6b4717eb62a92376" }, + { name = "microcosm-emitter", git = "https://github.com/PolicyEngine/microcosm-emitter?rev=12624a118db255405a0ce7397d53b3181deaf49a" }, { name = "microcosm-fit", editable = "packages/microcosm-fit" }, { name = "microcosm-frame", editable = "packages/microcosm-frame" }, { name = "microcosm-graph", editable = "packages/microcosm-graph" }, @@ -966,7 +966,7 @@ dev = [{ name = "pytest", specifier = ">=8" }] [[package]] name = "microcosm-emitter" version = "0.1.0" -source = { git = "https://github.com/PolicyEngine/microcosm-emitter?rev=7fef7c91641c9fc487e55d0e6b4717eb62a92376#7fef7c91641c9fc487e55d0e6b4717eb62a92376" } +source = { git = "https://github.com/PolicyEngine/microcosm-emitter?rev=12624a118db255405a0ce7397d53b3181deaf49a#12624a118db255405a0ce7397d53b3181deaf49a" } dependencies = [ { name = "alembic" }, { name = "httpx" },