diff --git a/README.md b/README.md index 820921a1f3..427c0f5b89 100644 --- a/README.md +++ b/README.md @@ -51,8 +51,14 @@ immutable publication before Hugging Face artifact staging, and preserves durable retry jobs and receipt snapshots. See [graph publication](docs/orrery-publication.md) for configuration and retry. -Supported US and UK build commands always start the local telemetry emitter -service. It reports live progress to the hosted collector when the operator's +Supported US and UK build commands start one local emitter service with separate +telemetry and graph-publication workers. Microcosm supplies build metadata and +graph files; the required `microcosm-emitter` package owns authentication, +durable queues and delivery. Telemetry and graph publication are internal +libraries in that single package, not separate installations. +This draft migration requires a published emitter release before merge; see the +[migration prerequisites](docs/orrery-publication.md#deferred-migration-prerequisites). +The telemetry worker reports live progress to the hosted collector when the operator's existing Hugging Face login is accepted; otherwise it retains the events locally and the build continues. This live event path is independent of the staging run files described below. diff --git a/changelog.d/1152-publication-types.changed.md b/changelog.d/1152-publication-types.changed.md new file mode 100644 index 0000000000..3449f91a60 --- /dev/null +++ b/changelog.d/1152-publication-types.changed.md @@ -0,0 +1 @@ +Use the emitter package's explicit publication contracts and pin the deferred adapter to the typed implementation. diff --git a/changelog.d/1152.changed.md b/changelog.d/1152.changed.md new file mode 100644 index 0000000000..0bb4082144 --- /dev/null +++ b/changelog.d/1152.changed.md @@ -0,0 +1 @@ +Delegate local telemetry and durable Orrery graph delivery to the single microcosm-emitter package while retaining UK graph generation and artifact staging in Microcosm. diff --git a/changelog.d/1153-emitter-concurrency.fixed.md b/changelog.d/1153-emitter-concurrency.fixed.md new file mode 100644 index 0000000000..8a90cff51a --- /dev/null +++ b/changelog.d/1153-emitter-concurrency.fixed.md @@ -0,0 +1 @@ +Use the emitter revision that prevents graph file requests from delaying telemetry acknowledgements and prevents older graph status writes from replacing newer results. diff --git a/changelog.d/1153-outcome-compatibility.fixed.md b/changelog.d/1153-outcome-compatibility.fixed.md new file mode 100644 index 0000000000..0bcb756462 --- /dev/null +++ b/changelog.d/1153-outcome-compatibility.fixed.md @@ -0,0 +1 @@ +Preserve validation-refused and classified-failure telemetry when using the external emitter, and keep its graph worker available until the build's final cleanup. diff --git a/changelog.d/1153-single-emitter.fixed.md b/changelog.d/1153-single-emitter.fixed.md new file mode 100644 index 0000000000..fd2c787afa --- /dev/null +++ b/changelog.d/1153-single-emitter.fixed.md @@ -0,0 +1 @@ +Use one microcosm-emitter dependency and its socket-based graph APIs, preserve public graph files when the service is unavailable, and create one run-start event with the build identity. diff --git a/conftest.py b/conftest.py index 0e09de4aad..f434d79435 100644 --- a/conftest.py +++ b/conftest.py @@ -10,7 +10,6 @@ import importlib.util import sys from pathlib import Path -from urllib.parse import urlsplit import pytest @@ -74,10 +73,15 @@ def pytest_collection_modifyitems(items) -> None: def fake_telemetry_emitters(monkeypatch, tmp_path): """Default every workspace test to in-memory telemetry, never production.""" from huggingface_hub import constants as hf_constants + from microcosm_emitter.host import launcher from microcosm.build import telemetry_emitter - from microcosm.build.telemetry_emitter_service import collector - from test_support.microcosm_build.telemetry import FakeTelemetryEmitter + from microcosm.build.uk_runtime import orrery_publication + from test_support.microcosm_build.telemetry import ( + FakeTelemetryEmitter, + fake_publication_inventory, + fake_publish_graph, + ) for name in ("HF_TOKEN", "HUGGING_FACE_HUB_TOKEN", "HUGGINGFACE_TOKEN"): monkeypatch.delenv(name, raising=False) @@ -97,13 +101,24 @@ def start(_cls, **kwargs): run = telemetry_emitter.TelemetryRun( **{name: value for name, value in kwargs.items() if name in fields} ) - emitter = FakeTelemetryEmitter(run) + emitter = FakeTelemetryEmitter( + run, close_service_on_finish=kwargs.get("close_service_on_finish", True) + ) emitters.append(emitter) return emitter monkeypatch.setattr( telemetry_emitter.LocalTelemetryEmitter, "start", classmethod(start) ) + monkeypatch.setattr( + orrery_publication, "publication_inventory", fake_publication_inventory + ) + monkeypatch.setattr(orrery_publication, "publish_graph", fake_publish_graph) + + def unexpected_start(*args, **options): + pytest.fail("A build test tried to start a real emitter service") + + monkeypatch.setattr(launcher, "start_services", unexpected_start) # Reset any cached handle left by a previously loaded build entrypoint. for module in tuple(sys.modules.values()): if isinstance( @@ -112,38 +127,6 @@ def start(_cls, **kwargs): ): monkeypatch.setattr(module, "_ACTIVE_EMITTER", None) - real_post = collector._http_post - - def local_post(url, *args, **kwargs): - parsed = urlsplit(url) - collector._development_collector_url(f"{parsed.scheme}://{parsed.netloc}") - return real_post(url, *args, **kwargs) - - monkeypatch.setattr(collector, "_http_post", local_post) yield emitters for emitter in emitters: emitter.close() - - -@pytest.fixture -def real_local_telemetry(monkeypatch, fake_telemetry_emitters): - """Permit actual startup only with an explicit loopback collector.""" - from microcosm.build import telemetry_emitter - from microcosm.build.telemetry_emitter_service.collector import ( - _development_collector_url, - ) - - def start(cls, **kwargs): - _development_collector_url(kwargs.get("development_collector_url") or "") - return _REAL_TELEMETRY_START(cls, **kwargs) - - monkeypatch.setattr( - telemetry_emitter.LocalTelemetryEmitter, "start", classmethod(start) - ) - - -# Save the underlying method before per-test patches so imported aliases and -# class references all share the same default fake and explicit local opt-in. -from microcosm.build.telemetry_emitter import LocalTelemetryEmitter # noqa: E402 - -_REAL_TELEMETRY_START = LocalTelemetryEmitter.start.__func__ diff --git a/docs/agent-guide.md b/docs/agent-guide.md index d121fe37f0..64cecc11c6 100644 --- a/docs/agent-guide.md +++ b/docs/agent-guide.md @@ -15,7 +15,7 @@ the PEP 420 namespace `microcosm.`: `frame`, `fit`, `calibrate`, `build`, ## Commands UK build completion exports and stages Orrery graph evidence. Publication uses -an independent worker inside the existing local service and a durable queue; +an independent worker inside the external `microcosm-emitter` service and a durable queue; see [configuration and retry](orrery-publication.md). No new producer secret or environment variable is required. Keep graph export/publication tests independent of the hosted runs app and collector; use synthetic files and fake HTTP. @@ -97,11 +97,17 @@ uses the same pinned parser for groups, statuses, activities and artifacts. These CI tests perform no browser rendering. Workspace tests use an in-memory telemetry emitter by default. The shared -fixture isolates Hugging Face credentials and cache paths and blocks collector -HTTP requests outside loopback. Tests of actual emitter startup must request -`real_local_telemetry` and supply an explicit loopback development collector; -that opt-in retains credential isolation. Other build tests can inspect -`fake_telemetry_emitters` without creating sockets or service processes. +fixture isolates Hugging Face credentials and cache paths. Adapter tests replace +emitter startup, inventory and publication APIs with fakes. Emitter process, persistence +and HTTP-delivery tests belong in microcosm-emitter; do not copy them +into Microcosm. Build tests inspect `fake_telemetry_emitters` without creating +sockets or service processes. + +The UK full-build caller sets `close_service_on_finish=False`: reporting an outcome +leaves the graph worker available, and that caller's existing `finally` cleanup +closes the combined service. Other callers close automatically when they finish. +Keep validation-refused outcomes and structured +failure codes intact when changing the adapter. New commits to a PR cancel older unfinished CI runs for that same PR. Each main-push run has a unique concurrency group, so all main-push runs diff --git a/docs/orrery-publication.md b/docs/orrery-publication.md index c7e6c65d96..8ca1a72409 100644 --- a/docs/orrery-publication.md +++ b/docs/orrery-publication.md @@ -6,13 +6,19 @@ public-safe file inventory in the attempt directory, attempt publication before Hugging Face artifact staging, and register graph/evidence files and an `orrery.publication.json` receipt snapshot in the artifact manifest. -The existing local service has two independent workers: live telemetry and -completed graph publication. Slow uploads cannot delay socket acknowledgement -or telemetry delivery. This change does not introduce another generic runtime, -rename the telemetry package, or add compatibility wrappers. +Microcosm's metadata adapter starts one local emitter service with two independent +workers: live telemetry and completed graph publication. The `microcosm-emitter` package +owns sockets, authentication, persistence, retries and resource sampling. +Microcosm retains graph generation, public-file selection and HF staging. +Slow uploads cannot delay socket acknowledgement or telemetry delivery. -Publication IDs do not depend on telemetry run IDs. Jobs use SQLAlchemy and -Alembic in the existing local SQLite database. Pending jobs survive missing or +UK full-build commands explicitly defer process shutdown when recording a completed, +validation-refused or failed telemetry outcome. Their final cleanup closes the combined +process after graph inventory and enqueue requests, so failure evidence uses the same +graph socket. Other callers close the process automatically when reporting their outcome. + +Publication IDs do not depend on telemetry run IDs. Jobs use SQLAlchemy ORM and +Alembic inside the emitter service's existing local SQLite database. Pending jobs survive missing or rejected credentials, process restarts and telemetry event pruning. Retry leases prevent a stale worker from replacing another worker's result. The exact source bytes must remain available until delivery completes. @@ -29,7 +35,7 @@ aggregate summaries can be published publicly. | Setting | Source and consumer | Provisioning | | --- | --- | --- | | `HF_TOKEN` / `HUGGINGFACE_TOKEN` | Existing operator credential, exchanged by the service for a short-lived collector session | Existing environment or Hugging Face login; never committed | -| Collector origin | Tracked `PRODUCTION_COLLECTOR_URL` in the existing service | No producer variable or destination override | +| Collector origin | Tracked `PRODUCTION_COLLECTOR_URL` in the emitter's internal core library | No producer variable or destination override | | Runs origin | Tracked `DEFAULT_GRAPH_PUBLICATION_ORIGIN` | No new producer variable | | `XDG_CACHE_HOME` | Optional existing cache-location setting | Defaults to the user's cache directory | @@ -56,15 +62,33 @@ an upload or stage to Hugging Face. ## Retry ```bash -microcosm-publish-graph --publication-id PUBLICATION_ID \ +microcosm-emitter-publish-graph --publication-id PUBLICATION_ID \ --directory /path/to/attempt/orrery-publication ``` The preserved directory contains `orrery.upload.json` and the exact files. Retries update a separate `publication.status.json`, never the original HF receipt snapshot. Pending jobs also resume when a subsequent local service -starts. A job whose preserved directory has been removed stays pending with -error code `preserved_files_missing` until its files are restored to the same -directory; it does not stop delivery of other jobs. Moving this -service into the `microcosm-emitter` package is a separate, deferred consumer -change; this implementation does not require that package. +starts. The retry command comes from the emitter package; Microcosm does not +retain a second delivery implementation or a compatibility wrapper. + +## Deferred migration prerequisites + +This adapter is implemented and tested in a draft PR, not merged or activated. +It depends on PolicyEngine/microcosm-emitter#4. The draft uses one exact root +Git source pin in `packages/microcosm-build/pyproject.toml` and `uv.lock` for +that PR's single `microcosm-emitter` distribution; no registry release was +available when this draft was updated. +Before merge, merge emitter #4, verify registry-owner configuration, publish +one `microcosm-emitter` release containing those changes, replace the draft source +override with that released version, regenerate `uv.lock`, update its +approved worker-identity digest and rerun tests and wheel checks. Merely building +a Microcosm wheel does not make its unpublished dependencies installable. + +The emitter rejects unversioned databases without modification. Before switching +an existing workstation, upgrade any old unversioned queue using the preceding +Microcosm implementation. Versioned telemetry and graph jobs keep their existing +IDs, eligibility, path and Alembic revision; do not delete a queue to migrate. +No new runtime variable, credential, hosted service or infrastructure is required +by this migration. The previously required hosted authorization/API deployments +remain prerequisites for remote publication. diff --git a/packages/microcosm-build/pyproject.toml b/packages/microcosm-build/pyproject.toml index ed07ea12da..0f40e28b58 100644 --- a/packages/microcosm-build/pyproject.toml +++ b/packages/microcosm-build/pyproject.toml @@ -7,12 +7,10 @@ requires-python = ">=3.13" license = { text = "MIT" } authors = [{ name = "PolicyEngine" }] dependencies = [ - "alembic>=1.13.3,<2", "numpy>=1.26", "pandas>=2", "scipy>=1.13", "huggingface-hub>=0.20", - "httpx>=0.27,<1", "microcosm-frame>=0.1,<0.2", "microcosm-graph>=0.1,<0.2", "microcosm-fit>=0.1,<0.2", @@ -26,7 +24,7 @@ dependencies = [ "pyyaml>=6", "jsonschema>=4.23,<5", "referencing>=0.35,<1", - "sqlalchemy>=2,<3", + "microcosm-emitter==0.1.0", ] [project.optional-dependencies] @@ -51,7 +49,6 @@ us = [ uk = ["policyengine-uk>=2.123.0", "microcosm-data>=0.1,<0.2", "h5py>=3", "tables>=3", "openpyxl>=3.1"] [project.scripts] -microcosm-publish-graph = "microcosm.build.graph_publication_cli:main" microcosm-build-uk = "microcosm.build.uk_runtime.full_build_cli:main" microcosm-export-us-l0-refit-h5 = "microcosm.build.us_runtime.l0_refit_export:main" @@ -84,3 +81,5 @@ microcosm-fit = { workspace = true } microcosm-calibrate = { workspace = true } microcosm-diagnostics = { workspace = true } microcosm-data = { workspace = true } +# Draft-only root source pin. Replace with a published distribution before merge. +microcosm-emitter = { git = "https://github.com/PolicyEngine/microcosm-emitter", rev = "12624a118db255405a0ce7397d53b3181deaf49a" } diff --git a/packages/microcosm-build/src/microcosm/build/graph_publication_cli.py b/packages/microcosm-build/src/microcosm/build/graph_publication_cli.py deleted file mode 100644 index 89318d838a..0000000000 --- a/packages/microcosm-build/src/microcosm/build/graph_publication_cli.py +++ /dev/null @@ -1,58 +0,0 @@ -"""Retry preserved graph uploads without registering a telemetry run.""" - -from __future__ import annotations - -import argparse -import json -from pathlib import Path - -from microcosm.build.telemetry_emitter import default_spool_path -from microcosm.build.telemetry_emitter_service.auth import ( - CollectorSession, - validated_origin, -) -from microcosm.build.telemetry_emitter_service.constants import PRODUCTION_COLLECTOR_URL -from microcosm.build.telemetry_emitter_service.graph_publication import ( - DEFAULT_GRAPH_PUBLICATION_ORIGIN, - GraphPublicationDelivery, - GraphPublicationQueue, -) - - -def main(argv: list[str] | None = None) -> int: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--spool", type=Path, default=default_spool_path()) - parser.add_argument("--publication-id", required=True) - parser.add_argument( - "--directory", - type=Path, - help="Optional preserved bundle containing orrery.upload.json; re-enqueues after moving machines.", - ) - parser.add_argument("--origin", default=DEFAULT_GRAPH_PUBLICATION_ORIGIN) - args = parser.parse_args(argv) - queue = GraphPublicationQueue(args.spool) - if args.directory is not None: - inventory = json.loads((args.directory / "orrery.upload.json").read_text()) - if inventory["publication_id"] != args.publication_id: - parser.error("The preserved inventory has a different publication ID.") - queue.enqueue(args.directory, inventory) - if queue.receipt(args.publication_id) is None: - parser.error("Publication job not found; supply its preserved directory.") - queue.retry(args.publication_id) - session = CollectorSession(validated_origin(PRODUCTION_COLLECTOR_URL)) - delivery = GraphPublicationDelivery( - queue, - credential=session.credential, - origin=args.origin, - invalidate_credential=session.invalidate, - ) - # Explicit retries process this job only, not unrelated pending jobs. - delivery.flush_once(publication_id=args.publication_id) - receipt = queue.receipt(args.publication_id) - assert receipt is not None, "A queued graph publication must have a receipt." - print(json.dumps(receipt, sort_keys=True)) - return 0 if receipt["status"] == "published" else 1 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/packages/microcosm-build/src/microcosm/build/graph_publication_types.py b/packages/microcosm-build/src/microcosm/build/graph_publication_types.py deleted file mode 100644 index a0f03f6a8c..0000000000 --- a/packages/microcosm-build/src/microcosm/build/graph_publication_types.py +++ /dev/null @@ -1,60 +0,0 @@ -"""Explicit data contracts for immutable graph publication.""" - -from dataclasses import dataclass -from pathlib import Path -from typing import Literal, NotRequired, TypedDict - -type PublicationFileRole = Literal[ - "graph", - "schema", - "index", - "declaration", - "manifest", - "binding", - "summaries", - "upstream", -] - - -type GraphJobStatus = Literal["pending", "published"] - - -class PublicationFile(TypedDict): - """One public file, identified by its exact bytes and allowed role.""" - - name: str - role: PublicationFileRole - bytes: int - sha256: str - - -class PublicationInventory(TypedDict): - """The versioned file inventory submitted to the publication API.""" - - version: Literal[1] - publication_id: str - files: list[PublicationFile] - - -class PublicationReceipt(TypedDict): - """Local publication result, including retry and build-snapshot fields.""" - - version: Literal[1] - publication_id: str | None - status: Literal["pending", "published", "skipped", "failed", "no_evidence"] - error_code: str | None - url: NotRequired[str | None] - attempts: NotRequired[int] - inventory_sha256: NotRequired[str] - preserved_directory: NotRequired[str] - recorded_execution: NotRequired[bool] - - -@dataclass(frozen=True) -class ClaimedGraphPublication: - """A job reserved until lease_until; the timestamp also identifies its lease.""" - - publication_id: str - directory: Path - inventory: PublicationInventory - lease_until: float diff --git a/packages/microcosm-build/src/microcosm/build/telemetry_emitter.py b/packages/microcosm-build/src/microcosm/build/telemetry_emitter.py index 336b846fed..559e77838e 100644 --- a/packages/microcosm-build/src/microcosm/build/telemetry_emitter.py +++ b/packages/microcosm-build/src/microcosm/build/telemetry_emitter.py @@ -1,147 +1,58 @@ -"""Best-effort client for the local telemetry emitter service. - -The build process only writes small messages to a private local socket. A -separate process owns retry, authentication, resource sampling, and network -I/O so collector availability cannot delay or fail a dataset build. -""" +"""Microcosm metadata adapter for the independently packaged local emitter.""" from __future__ import annotations -import json -import os -import socket -import subprocess import sys -import tempfile -import time -import uuid from collections.abc import Mapping -from dataclasses import dataclass, field -from datetime import UTC, datetime from pathlib import Path from typing import Any -from microcosm.build.graph_publication_types import ( +from microcosm_emitter.host.launcher import start_services +from microcosm_emitter.orrery.client import ( + GraphPublicationClient, + default_spool_path, + publish_graph, +) +from microcosm_emitter.orrery.contracts import ( + PublicationFileRole, PublicationInventory, PublicationReceipt, ) -from microcosm.build.telemetry_emitter_constants import ( - DEFAULT_HEARTBEAT_SECONDS, - DEFAULT_SEND_TIMEOUT_SECONDS, - DEFAULT_STARTUP_TIMEOUT_SECONDS, - INVALID_ACKNOWLEDGEMENT_ERROR, - NO_LOCAL_SOCKET_WARNING, - QUEUE_WARNING, - RUNTIME_DIRECTORY_PREFIX, - SERVICE_NOT_READY_WARNING, - SERVICE_PROCESS_EXIT_TIMEOUT_SECONDS, - SERVICE_READY_TIMEOUT_SECONDS, - SERVICE_START_WARNING, - SOCKET_FILENAME, - STARTUP_POLL_SECONDS, - TELEMETRY_CACHE_PARTS, - TELEMETRY_SERVICE_MODULE, - TELEMETRY_SPOOL_FILENAME, - TEMPORARY_DIRECTORY_ALIAS, +from microcosm_emitter.telemetry.client import ( + LocalTelemetryEmitter as EmitterTelemetryClient, ) -from microcosm.build.telemetry_identity import runtime_identity -from microcosm.build.telemetry_protocol import ( - ACTION_CLOSE, - ACTION_EVENT, - BUILD_BLOCKED_MESSAGE, - BUILD_COMPLETED_MESSAGE, - BUILD_STARTED_MESSAGE, - CALIBRATION_EVENT_KIND, - EVENT_TYPE_CALIBRATION, - EVENT_TYPE_PROGRESS, - EVENT_TYPE_RUN, - EVENT_TYPE_STAGE, - LOCAL_ACKNOWLEDGEMENT_OK, - LOCAL_ACKNOWLEDGEMENT_READ_BYTES, - LOCAL_MESSAGE_DELIMITER, - LOCAL_PING_MESSAGE, - MAX_TELEMETRY_MESSAGE_CHARS, - SEQUENTIAL_STATUS_MAP, - STAGE_BLOCKED, - STAGE_CALIBRATING, - STAGE_COMPLETE, - STAGE_CREATED, - STAGE_FAILED, - STATUS_BLOCKED, - STATUS_COMPLETED, - STATUS_FAILED, - STATUS_PROGRESS, - STATUS_STARTED, - TelemetryEventType, - TelemetryStatus, +from microcosm_emitter.telemetry.client import ( + TelemetryRun, ) -from microcosm.build.telemetry_sanitization import ( - sanitize_details, - sanitize_text, +from microcosm_emitter.telemetry.constants import ( + DEFAULT_HEARTBEAT_SECONDS, + SERVICE_START_WARNING, ) - -def _now() -> str: - return datetime.now(UTC).isoformat() - - -def _cache_dir() -> Path: - configured = os.environ.get("XDG_CACHE_HOME", "").strip() - root = Path(configured).expanduser() if configured else Path.home() / ".cache" - return root.joinpath(*TELEMETRY_CACHE_PARTS) - - -def default_spool_path() -> Path: - """The local SQLite queue shared by telemetry events and graph publication.""" - return _cache_dir() / TELEMETRY_SPOOL_FILENAME - - -@dataclass(frozen=True) -class TelemetryRun: - """Identity registered with the hosted collector for one build.""" - - run_id: str - country_code: str - pipeline: str - candidate_id: str | None = None - release_id: str | None = None - run_kind: str = "build" - producer_id: str = field(default_factory=lambda: uuid.uuid4().hex) - - def as_registration(self) -> dict[str, Any]: - return { - "run_id": self.run_id, - "producer_id": self.producer_id, - "country_code": self.country_code, - "pipeline": self.pipeline, - "candidate_id": self.candidate_id, - "release_id": self.release_id, - "run_kind": self.run_kind, - } +from microcosm.build.telemetry_identity import runtime_identity -class LocalTelemetryEmitter: - """Non-blocking handle owned by the instrumented build process.""" +class LocalTelemetryEmitter(EmitterTelemetryClient): + """Supply build identity and select both workers in one emitter process.""" def __init__( self, *, - run: TelemetryRun, - process: subprocess.Popen[bytes] | None, - socket_path: Path | None, - runtime_dir: Path | None, - send_timeout_seconds: float = DEFAULT_SEND_TIMEOUT_SECONDS, - spool_path: Path | None = None, - ) -> None: - self.run = run - self._process = process - self._socket_path = socket_path - self._runtime_dir = runtime_dir - self._send_timeout_seconds = send_timeout_seconds - self._spool_path = spool_path - self._closed = False - self._warned = False - self._transition_stage: str | None = None + run, + transport, + handle=None, + graph_client=None, + close_service_on_finish: bool = True, + ): + super().__init__( + run=run, + transport=transport, + handle=handle, + close_service_on_finish=close_service_on_finish, + ) + self._graph_client = ( + graph_client if graph_client is not None else GraphPublicationClient() + ) @classmethod def start( @@ -155,11 +66,10 @@ def start( run_kind: str = "build", development_collector_url: str | None = None, heartbeat_seconds: float = DEFAULT_HEARTBEAT_SECONDS, - startup_timeout_seconds: float = DEFAULT_STARTUP_TIMEOUT_SECONDS, + startup_timeout_seconds: float = 3.0, spool_path: Path | str | None = None, + close_service_on_finish: bool = True, ) -> LocalTelemetryEmitter: - """Start the service, returning a harmless disabled handle on failure.""" - run = TelemetryRun( run_id=run_id, country_code=country_code, @@ -168,301 +78,48 @@ def start( release_id=release_id, run_kind=run_kind, ) - if not hasattr(socket, "AF_UNIX"): - print(NO_LOCAL_SOCKET_WARNING, file=sys.stderr) - return cls(run=run, process=None, socket_path=None, runtime_dir=None) - - runtime_dir: Path | None = None - socket_path: Path | None = None - process: subprocess.Popen[bytes] | None = None try: - # macOS limits AF_UNIX paths to roughly 100 bytes. Its default - # temporary directory is already long, so use the short system alias. - temporary_root = ( - TEMPORARY_DIRECTORY_ALIAS - if TEMPORARY_DIRECTORY_ALIAS.is_dir() - else None - ) - runtime_dir = Path( - tempfile.mkdtemp(prefix=RUNTIME_DIRECTORY_PREFIX, dir=temporary_root) - ) - runtime_dir.chmod(0o700) - socket_path = runtime_dir / SOCKET_FILENAME - queue_path = Path(spool_path) if spool_path else default_spool_path() - command = [ - sys.executable, - "-m", - TELEMETRY_SERVICE_MODULE, - "--socket", - str(socket_path), - "--spool", - str(queue_path), - "--registration-json", - json.dumps(run.as_registration(), separators=(",", ":")), - "--parent-pid", - str(os.getpid()), - "--heartbeat-seconds", - str(heartbeat_seconds), - ] - if development_collector_url is not None: - command.extend( - ["--development-collector-url", development_collector_url] - ) - process = subprocess.Popen( - command, - stdin=subprocess.DEVNULL, - stdout=subprocess.DEVNULL, - start_new_session=True, + path = Path(spool_path) if spool_path else default_spool_path() + identity = runtime_identity() + shared = { + "spool_path": str(path), + "development_collector_url": development_collector_url, + } + handle = start_services( + { + "telemetry": { + **shared, + "registration": run.as_registration(), + "identity": identity, + "heartbeat_seconds": heartbeat_seconds, + }, + "orrery": shared, + }, + startup_timeout=startup_timeout_seconds, ) - deadline = time.monotonic() + startup_timeout_seconds - while time.monotonic() < deadline: - if socket_path.exists() and _service_ready(socket_path): - emitter = cls( - run=run, - process=process, - socket_path=socket_path, - runtime_dir=runtime_dir, - spool_path=queue_path, - ) - emitter.emit( - event_type=EVENT_TYPE_RUN, - stage_id=STAGE_CREATED, - status=STATUS_STARTED, - message=BUILD_STARTED_MESSAGE, - details={"identity": runtime_identity()}, - ) - return emitter - if process.poll() is not None: - break - time.sleep(STARTUP_POLL_SECONDS) except Exception as error: print( - SERVICE_START_WARNING.format( - error_type=type(error).__name__, - error=error, - ), + SERVICE_START_WARNING.format(error_type=type(error).__name__), file=sys.stderr, ) - else: - print(SERVICE_NOT_READY_WARNING, file=sys.stderr) - if process is not None: - _terminate_process(process) - try: - if socket_path is not None: - socket_path.unlink(missing_ok=True) - if runtime_dir is not None: - runtime_dir.rmdir() - except OSError: - pass + return cls( + run=run, + transport=None, + close_service_on_finish=close_service_on_finish, + ) return cls( run=run, - process=None, - socket_path=None, - runtime_dir=runtime_dir, - spool_path=Path(spool_path) if spool_path else None, + transport=handle.client.for_module("telemetry"), + handle=handle, + graph_client=GraphPublicationClient(handle.client.for_module("orrery")), + close_service_on_finish=close_service_on_finish, ) - @property - def available(self) -> bool: - return self._socket_path is not None and not self._closed - - def emit( - self, - *, - event_type: TelemetryEventType, - status: TelemetryStatus, - stage_id: str | None = None, - message: str | None = None, - details: Mapping[str, Any] | None = None, - ) -> None: - """Queue one event locally, never raising into the build.""" - - if not self.available: - return - self._send( - { - "action": ACTION_EVENT, - "event": { - "timestamp": _now(), - "event_type": event_type, - "stage_id": stage_id, - "status": status, - "message": ( - sanitize_text(message, limit=MAX_TELEMETRY_MESSAGE_CHARS) - if message - else None - ), - "details": sanitize_details(details or {}), - }, - } - ) - - def stage( - self, - stage_id: str, - *, - status: TelemetryStatus = STATUS_STARTED, - message: str | None = None, - **details: Any, - ) -> None: - self.emit( - event_type=EVENT_TYPE_STAGE, - stage_id=stage_id, - status=status, - message=message, - details=details, - ) - - def transition_stage( - self, - stage_id: str, - *, - status: str = "running", - message: str | None = None, - **details: Any, - ) -> None: - """Translate a sequential stage update into explicit lifecycle events.""" - - collector_status = SEQUENTIAL_STATUS_MAP.get(status, STATUS_PROGRESS) - if collector_status == STATUS_STARTED: - if self._transition_stage == stage_id: - self.stage( - stage_id, - status=STATUS_PROGRESS, - message=message, - **details, - ) - return - self._close_transition_stage() - self._transition_stage = stage_id - elif self._transition_stage == stage_id: - self._transition_stage = None - else: - self._close_transition_stage() - self.stage( - stage_id, - status=collector_status, - message=message, - **details, - ) - - def progress( - self, - stage_id: str, - *, - done: int, - total: int, - unit: str | None = None, - **details: Any, - ) -> None: - self.emit( - event_type=EVENT_TYPE_PROGRESS, - stage_id=stage_id, - status=STATUS_PROGRESS, - details={"done": done, "total": total, "unit": unit, **details}, - ) - - def calibration_progress(self, event: Mapping[str, Any]) -> None: - if event.get("kind") != CALIBRATION_EVENT_KIND: - return - self.emit( - event_type=EVENT_TYPE_CALIBRATION, - stage_id=STAGE_CALIBRATING, - status=STATUS_PROGRESS, - details=event, - ) - - def transition_calibration_progress(self, event: Mapping[str, Any]) -> None: - """Enter the sequential calibration stage, then report one epoch.""" - - if event.get("kind") != CALIBRATION_EVENT_KIND: - return - if self._transition_stage != STAGE_CALIBRATING: - self.transition_stage(STAGE_CALIBRATING) - self.calibration_progress(event) - - def fail( - self, - error: BaseException, - *, - failed_during: str | None = None, - failure_class: str = "build_failure", - error_code: str | None = None, - ) -> None: - failed_stage = failed_during or self._transition_stage - message = str(error)[:MAX_TELEMETRY_MESSAGE_CHARS] - details = { - "error_type": type(error).__name__, - "failure_class": failure_class, - "failed_during": failed_stage, - } - if error_code is not None: - details["error_code"] = error_code - self._close_transition_stage(status=STATUS_FAILED, message=message, **details) - self.emit( - event_type=EVENT_TYPE_RUN, - stage_id=STAGE_FAILED, - status=STATUS_FAILED, - message=message, - details=details, - ) - self.close() - - def block( - self, - *, - phase: str, - blocking_gate_ids: list[str] | tuple[str, ...], - blocking_failure_count: int | None = None, - gate_statuses: Mapping[str, str] | None = None, - ) -> None: - """End the run as ``blocked``: the gates refused its candidate. - - A dedicated run event rather than a stage transition, whose status map - would read an unknown ``blocked`` as progress. The open stage itself - finished; it closes as completed.""" - - gate_ids = [str(gate_id) for gate_id in blocking_gate_ids] - details: dict[str, Any] = { - "phase": phase, - "blocking_failure_count": ( - len(gate_ids) - if blocking_failure_count is None - else blocking_failure_count - ), - "blocking_gate_ids": gate_ids, - } - if gate_statuses is not None: - details["gate_statuses"] = { - str(gate_id): str(status) for gate_id, status in gate_statuses.items() - } - self._close_transition_stage() - self.emit( - event_type=EVENT_TYPE_RUN, - stage_id=STAGE_BLOCKED, - status=STATUS_BLOCKED, - message=BUILD_BLOCKED_MESSAGE.format(phase=phase), - details=details, - ) - self.close() - - def complete(self) -> None: - self._close_transition_stage() - self.emit( - event_type=EVENT_TYPE_RUN, - stage_id=STAGE_COMPLETE, - status=STATUS_COMPLETED, - message=BUILD_COMPLETED_MESSAGE, - ) - self.close() - - def close(self) -> None: - """Ask the service to flush in the background, then release the handle.""" - - if self._closed: - return - self._send({"action": ACTION_CLOSE}) - self._closed = True + def publication_inventory( + self, directory: Path, roles: Mapping[str, PublicationFileRole] + ) -> PublicationInventory | None: + """Ask the existing graph worker to measure selected public files.""" + return self._graph_client.inventory(directory, roles) def publish_graph( self, @@ -471,98 +128,14 @@ def publish_graph( *, wait_seconds: float = 30.0, ) -> PublicationReceipt: - """Preserve a job first, then wait a bounded time for its first result. - - Unlike telemetry, missing credentials never discard this job. Enqueue - directly into the service's shared SQLite database so a socket timeout - or unavailable service cannot lose the publication request. - """ - from microcosm.build.telemetry_emitter_service.graph_publication import ( - GraphPublicationQueue, + return publish_graph( + directory, + inventory, + client=self._graph_client, + wait_seconds=wait_seconds, ) - queue = GraphPublicationQueue(self._spool_path or default_spool_path()) - queue.enqueue(directory, inventory) - deadline = time.monotonic() + max(0, wait_seconds) - while self.available and time.monotonic() < deadline: - receipt = queue.receipt(inventory["publication_id"]) - assert receipt is not None, ( - "A queued graph publication must have a receipt." - ) - if receipt.get("attempts", 0) or receipt["status"] == "published": - return receipt - time.sleep(0.1) - receipt = queue.receipt(inventory["publication_id"]) - assert receipt is not None, "A queued graph publication must have a receipt." - return receipt - - def _close_transition_stage( - self, - *, - status: TelemetryStatus = STATUS_COMPLETED, - message: str | None = None, - **details: Any, - ) -> None: - if self._transition_stage is None: - return - stage_id = self._transition_stage - self._transition_stage = None - self.stage(stage_id, status=status, message=message, **details) - - def _send(self, payload: Mapping[str, Any]) -> None: - if self._socket_path is None: - return - try: - with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as client: - client.settimeout(self._send_timeout_seconds) - client.connect(str(self._socket_path)) - client.sendall( - json.dumps(payload, separators=(",", ":"), allow_nan=False).encode() - + LOCAL_MESSAGE_DELIMITER - ) - acknowledgement = client.recv(LOCAL_ACKNOWLEDGEMENT_READ_BYTES) - if acknowledgement != LOCAL_ACKNOWLEDGEMENT_OK: - raise OSError(INVALID_ACKNOWLEDGEMENT_ERROR) - except Exception as error: - if not self._warned: - print( - QUEUE_WARNING.format(error_type=type(error).__name__), - file=sys.stderr, - ) - self._warned = True - - -def start_local_telemetry_emitter_service( - **run: Any, -) -> LocalTelemetryEmitter: - """Named construction seam for build entrypoints and tests.""" +def start_local_telemetry_emitter_service(**run: Any) -> LocalTelemetryEmitter: + """Construct the build adapter without importing emitter service internals.""" return LocalTelemetryEmitter.start(**run) - - -def _service_ready(socket_path: Path) -> bool: - try: - with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as client: - client.settimeout(SERVICE_READY_TIMEOUT_SECONDS) - client.connect(str(socket_path)) - client.sendall(LOCAL_PING_MESSAGE) - return ( - client.recv(LOCAL_ACKNOWLEDGEMENT_READ_BYTES) - == LOCAL_ACKNOWLEDGEMENT_OK - ) - except OSError: - return False - - -def _terminate_process(process: subprocess.Popen[bytes]) -> None: - """Terminate and reap an emitter service that did not become usable.""" - - try: - if process.poll() is None: - process.terminate() - process.wait(timeout=SERVICE_PROCESS_EXIT_TIMEOUT_SECONDS) - except subprocess.TimeoutExpired: - process.kill() - process.wait(timeout=SERVICE_PROCESS_EXIT_TIMEOUT_SECONDS) - except OSError: - pass diff --git a/packages/microcosm-build/src/microcosm/build/telemetry_emitter_constants.py b/packages/microcosm-build/src/microcosm/build/telemetry_emitter_constants.py deleted file mode 100644 index 57a477f82b..0000000000 --- a/packages/microcosm-build/src/microcosm/build/telemetry_emitter_constants.py +++ /dev/null @@ -1,38 +0,0 @@ -"""Configuration and user-facing messages for the telemetry emitter client.""" - -from __future__ import annotations - -from pathlib import Path -from typing import Final - -TELEMETRY_SERVICE_MODULE: Final = "microcosm.build.telemetry_emitter_service" -TELEMETRY_CACHE_PARTS: Final = ("microcosm", "telemetry") -TELEMETRY_SPOOL_FILENAME: Final = "events.sqlite3" -RUNTIME_DIRECTORY_PREFIX: Final = "microcosm-telemetry-" -SOCKET_FILENAME: Final = "emitter.sock" -TEMPORARY_DIRECTORY_ALIAS: Final = Path("/tmp") - -DEFAULT_SEND_TIMEOUT_SECONDS: Final = 0.2 -DEFAULT_HEARTBEAT_SECONDS: Final = 60.0 -DEFAULT_STARTUP_TIMEOUT_SECONDS: Final = 3.0 -STARTUP_POLL_SECONDS: Final = 0.02 -SERVICE_READY_TIMEOUT_SECONDS: Final = 0.1 -SERVICE_PROCESS_EXIT_TIMEOUT_SECONDS: Final = 1.0 - -NO_LOCAL_SOCKET_WARNING: Final = ( - "warning: Microcosm telemetry is unavailable because this platform has no " - "local Unix sockets." -) -SERVICE_START_WARNING: Final = ( - "warning: the local telemetry emitter service could not start: " - "{error_type}: {error}" -) -SERVICE_NOT_READY_WARNING: Final = ( - "warning: the local telemetry emitter service did not become ready; " - "the build will continue without hosted telemetry." -) -QUEUE_WARNING: Final = ( - "warning: the local telemetry emitter service could not queue an update " - "({error_type}); the build will continue." -) -INVALID_ACKNOWLEDGEMENT_ERROR: Final = "invalid acknowledgement" diff --git a/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/__init__.py b/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/__init__.py deleted file mode 100644 index ef5335561f..0000000000 --- a/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/__init__.py +++ /dev/null @@ -1,17 +0,0 @@ -"""Local service that queues and delivers Microcosm telemetry.""" - -from microcosm.build.telemetry_emitter_service.collector import CollectorDelivery -from microcosm.build.telemetry_emitter_service.constants import ( - PRODUCTION_COLLECTOR_URL, -) -from microcosm.build.telemetry_emitter_service.resources import ProcessTreeSampler -from microcosm.build.telemetry_emitter_service.runtime import EmitterService -from microcosm.build.telemetry_emitter_service.spool import EventSpool - -__all__ = [ - "CollectorDelivery", - "EmitterService", - "EventSpool", - "PRODUCTION_COLLECTOR_URL", - "ProcessTreeSampler", -] diff --git a/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/__main__.py b/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/__main__.py deleted file mode 100644 index 2981038314..0000000000 --- a/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/__main__.py +++ /dev/null @@ -1,5 +0,0 @@ -"""Module execution entry point for the telemetry emitter service.""" - -from microcosm.build.telemetry_emitter_service.main import main - -raise SystemExit(main()) diff --git a/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/alembic/env.py b/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/alembic/env.py deleted file mode 100644 index 2cb2edcc25..0000000000 --- a/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/alembic/env.py +++ /dev/null @@ -1,28 +0,0 @@ -"""Alembic environment for the local telemetry spool.""" - -from alembic import context -from sqlalchemy.engine import Connection - -from microcosm.build.telemetry_emitter_service.models import SpoolModel - -_MISSING_CONNECTION_ERROR = ( - "telemetry spool migrations require a programmatically supplied connection" -) - - -def run_migrations() -> None: - """Run migrations on the connection supplied by the emitter service.""" - - connection = context.config.attributes.get("connection") - if not isinstance(connection, Connection): - raise RuntimeError(_MISSING_CONNECTION_ERROR) - context.configure( - connection=connection, - target_metadata=SpoolModel.metadata, - render_as_batch=True, - ) - with context.begin_transaction(): - context.run_migrations() - - -run_migrations() diff --git a/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/alembic/script.py.mako b/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/alembic/script.py.mako deleted file mode 100644 index f059f7642d..0000000000 --- a/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/alembic/script.py.mako +++ /dev/null @@ -1,31 +0,0 @@ -"""${message} - -Revision ID: ${up_revision} -Revises: ${down_revision | comma,n} -Create Date: ${create_date} -""" - -from __future__ import annotations - -from collections.abc import Sequence - -import sqlalchemy as sa -from alembic import op -${imports if imports else ""} - -revision: str = ${repr(up_revision)} -down_revision: str | None = ${repr(down_revision)} -branch_labels: str | Sequence[str] | None = ${repr(branch_labels)} -depends_on: str | Sequence[str] | None = ${repr(depends_on)} - - -def upgrade() -> None: - """Apply this schema revision.""" - - ${upgrades if upgrades else "pass"} - - -def downgrade() -> None: - """Reverse this schema revision.""" - - ${downgrades if downgrades else "pass"} diff --git a/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/alembic/versions/v20261007_01_initial_spool.py b/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/alembic/versions/v20261007_01_initial_spool.py deleted file mode 100644 index 87290fd653..0000000000 --- a/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/alembic/versions/v20261007_01_initial_spool.py +++ /dev/null @@ -1,140 +0,0 @@ -"""Create or adopt the telemetry spool schema. - -Revision ID: 20261007_01 -Revises: -""" - -from __future__ import annotations - -from collections.abc import Sequence - -import sqlalchemy as sa -from alembic import op - -revision: str = "20261007_01" -down_revision: str | None = None -branch_labels: str | Sequence[str] | None = None -depends_on: str | Sequence[str] | None = None - -_RUNS_TABLE = "telemetry_runs" -_EVENTS_TABLE = "telemetry_events" -_EVENTS_INDEX = "telemetry_events_run_sequence" -_PENDING_UPLOAD_STATE = "pending" -_LOCAL_ONLY_UPLOAD_STATE = "local_only" -_PRE_ELIGIBILITY_REASON = "created_before_upload_eligibility" - - -def _create_runs_table() -> None: - op.create_table( - _RUNS_TABLE, - sa.Column("run_id", sa.Text(), nullable=False), - sa.Column("producer_id", sa.Text(), nullable=False), - sa.Column("registration_json", sa.Text(), nullable=False), - sa.Column( - "next_sequence", - sa.Integer(), - nullable=False, - server_default="1", - ), - sa.Column( - "upload_state", - sa.Text(), - nullable=False, - server_default=_PENDING_UPLOAD_STATE, - ), - sa.Column("local_only_reason", sa.Text(), nullable=True), - sa.Column("updated_at", sa.Text(), nullable=False), - sa.PrimaryKeyConstraint("run_id", "producer_id"), - ) - - -def _adopt_runs_table(inspector: sa.Inspector) -> None: - column_names = {column["name"] for column in inspector.get_columns(_RUNS_TABLE)} - added_upload_state = "upload_state" not in column_names - if added_upload_state: - op.add_column( - _RUNS_TABLE, - sa.Column( - "upload_state", - sa.Text(), - nullable=False, - server_default=_PENDING_UPLOAD_STATE, - ), - ) - added_local_only_reason = "local_only_reason" not in column_names - if added_local_only_reason: - op.add_column( - _RUNS_TABLE, - sa.Column("local_only_reason", sa.Text(), nullable=True), - ) - - runs = sa.table( - _RUNS_TABLE, - sa.column("upload_state", sa.Text()), - sa.column("local_only_reason", sa.Text()), - ) - if added_upload_state: - op.execute(sa.update(runs).values(upload_state=_LOCAL_ONLY_UPLOAD_STATE)) - if added_upload_state or added_local_only_reason: - op.execute( - sa.update(runs) - .where(runs.c.upload_state == _LOCAL_ONLY_UPLOAD_STATE) - .values(local_only_reason=_PRE_ELIGIBILITY_REASON) - ) - - -def _create_events_table() -> None: - op.create_table( - _EVENTS_TABLE, - sa.Column("event_id", sa.Text(), nullable=False), - sa.Column("run_id", sa.Text(), nullable=False), - sa.Column("producer_id", sa.Text(), nullable=False), - sa.Column("sequence", sa.Integer(), nullable=False), - sa.Column("payload_json", sa.Text(), nullable=False), - sa.Column("created_at", sa.Text(), nullable=False), - sa.ForeignKeyConstraint( - ["run_id", "producer_id"], - [f"{_RUNS_TABLE}.run_id", f"{_RUNS_TABLE}.producer_id"], - name="fk_telemetry_events_run", - ), - sa.PrimaryKeyConstraint("event_id"), - sa.UniqueConstraint( - "run_id", - "producer_id", - "sequence", - name="uq_telemetry_events_run_producer_sequence", - ), - ) - - -def upgrade() -> None: - """Create the current schema or adopt a pre-Alembic spool.""" - - inspector = sa.inspect(op.get_bind()) - table_names = set(inspector.get_table_names()) - if _RUNS_TABLE not in table_names: - _create_runs_table() - else: - _adopt_runs_table(inspector) - - inspector = sa.inspect(op.get_bind()) - table_names = set(inspector.get_table_names()) - if _EVENTS_TABLE not in table_names: - _create_events_table() - - inspector = sa.inspect(op.get_bind()) - index_names = {index["name"] for index in inspector.get_indexes(_EVENTS_TABLE)} - if _EVENTS_INDEX not in index_names: - op.create_index( - _EVENTS_INDEX, - _EVENTS_TABLE, - ["run_id", "producer_id", "sequence"], - ) - - -def downgrade() -> None: - """Remove the local spool schema.""" - - op.drop_index(_EVENTS_INDEX, table_name=_EVENTS_TABLE) - op.drop_table(_EVENTS_TABLE) - op.drop_table(_RUNS_TABLE) diff --git a/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/alembic/versions/v20261008_02_graph_publication.py b/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/alembic/versions/v20261008_02_graph_publication.py deleted file mode 100644 index 14ae0318a6..0000000000 --- a/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/alembic/versions/v20261008_02_graph_publication.py +++ /dev/null @@ -1,27 +0,0 @@ -"""Add durable graph publication jobs without changing event retention.""" - -import sqlalchemy as sa -from alembic import op - -revision = "20261008_02" -down_revision = "20261007_01" -branch_labels = None -depends_on = None - - -def upgrade() -> None: - op.create_table( - "graph_publication_jobs", - sa.Column("publication_id", sa.Text(), primary_key=True), - sa.Column("inventory", sa.Text(), nullable=False), - sa.Column("directory", sa.Text(), nullable=False), - sa.Column("status", sa.Text(), nullable=False), - sa.Column("attempts", sa.Integer(), nullable=False), - sa.Column("next_attempt_at", sa.Float(), nullable=False), - sa.Column("lease_until", sa.Float(), nullable=False), - sa.Column("receipt", sa.Text(), nullable=False), - ) - - -def downgrade() -> None: - op.drop_table("graph_publication_jobs") diff --git a/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/auth.py b/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/auth.py deleted file mode 100644 index 205fef3dd5..0000000000 --- a/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/auth.py +++ /dev/null @@ -1,180 +0,0 @@ -"""HF-to-collector session shared by telemetry and graph publication.""" - -from __future__ import annotations - -import json -import os -import threading -import time -import urllib.error -import urllib.request -from collections.abc import Callable, Mapping -from http import HTTPStatus -from typing import Any, Literal, NamedTuple -from urllib.parse import urlsplit - -from huggingface_hub import get_token - -from microcosm.build.telemetry_emitter_service.constants import ( - COLLECTOR_RESPONSE_TOO_LARGE_ERROR, - DEFAULT_TOKEN_LIFETIME_SECONDS, - DEVELOPMENT_COLLECTOR_LOOPBACK_ERROR, - HTTP_TIMEOUT_SECONDS, - HTTP_USER_AGENT, - LOOPBACK_HOSTS, - MAX_HTTP_RESPONSE_BYTES, - MINIMUM_TOKEN_LIFETIME_SECONDS, - SERVICE_ORIGIN_FORMAT_ERROR, - SERVICE_ORIGIN_HTTPS_ERROR, - TOKEN_EXCHANGE_PATH, - TOKEN_REFRESH_MARGIN_SECONDS, -) - - -class _NoRedirectHandler(urllib.request.HTTPRedirectHandler): - """Keep bearer credentials on the explicitly configured origin.""" - - def redirect_request(self, req, fp, code, msg, headers, newurl): - return None - - -type CredentialStatus = Literal["ok", "missing", "rejected", "unavailable"] - - -class CollectorCredential(NamedTuple): - """A session token, or the reason none is available for this attempt.""" - - status: CredentialStatus - token: str | None - - -def validated_origin(value: str, *, allow_loopback_http: bool = False) -> str: - """Return an HTTPS origin, or plain HTTP only for an allowed loopback host.""" - parsed = urlsplit(value) - loopback = parsed.hostname in LOOPBACK_HOSTS - valid_scheme = parsed.scheme == "https" or ( - allow_loopback_http and loopback and parsed.scheme == "http" - ) - if not parsed.hostname or not valid_scheme: - raise ValueError(SERVICE_ORIGIN_HTTPS_ERROR) - if ( - parsed.username - or parsed.password - or parsed.query - or parsed.fragment - or parsed.path not in {"", "/"} - ): - raise ValueError(SERVICE_ORIGIN_FORMAT_ERROR) - return value.rstrip("/") - - -def _development_collector_url(value: str) -> str: - parsed = urlsplit(value) - if parsed.hostname not in LOOPBACK_HOSTS: - raise ValueError(DEVELOPMENT_COLLECTOR_LOOPBACK_ERROR) - return validated_origin(value, allow_loopback_http=True) - - -def _decode_response(body: bytes) -> dict[str, Any]: - try: - response = json.loads(body) if body else {} - except (json.JSONDecodeError, UnicodeDecodeError): - return {} - return response if isinstance(response, dict) else {} - - -def _http_post( - url: str, - payload: Mapping[str, Any], - bearer_token: str, - *, - timeout: float = HTTP_TIMEOUT_SECONDS, -) -> tuple[int, dict[str, Any]]: - request = urllib.request.Request( - url, - data=json.dumps(payload, separators=(",", ":")).encode(), - headers={ - "Authorization": f"Bearer {bearer_token}", - "Content-Type": "application/json", - "User-Agent": HTTP_USER_AGENT, - }, - method="POST", - ) - try: - opener = urllib.request.build_opener(_NoRedirectHandler) - with opener.open(request, timeout=timeout) as response: - body = response.read(MAX_HTTP_RESPONSE_BYTES + 1) - if len(body) > MAX_HTTP_RESPONSE_BYTES: - raise OSError(COLLECTOR_RESPONSE_TOO_LARGE_ERROR) - return response.status, _decode_response(body) - except urllib.error.HTTPError as error: - body = error.read(MAX_HTTP_RESPONSE_BYTES + 1) - if len(body) > MAX_HTTP_RESPONSE_BYTES: - return error.code, {} - return error.code, _decode_response(body) - - -type HttpPost = Callable[[str, Mapping[str, Any], str], tuple[int, dict[str, Any]]] - - -def _huggingface_token() -> str | None: - return ( - os.environ.get("HF_TOKEN", "").strip() - or os.environ.get("HUGGINGFACE_TOKEN", "").strip() - or get_token() - ) - - -class CollectorSession: - """One thread-safe short-lived credential shared by independent publishers.""" - - def __init__( - self, - origin: str, - *, - post: HttpPost | None = None, - credential: Callable[[], str | None] | None = None, - ) -> None: - self.origin = origin - self._post = post or _http_post - self._credential = credential or _huggingface_token - self._session_token: tuple[str, float] | None = None - self._lock = threading.Lock() - - def invalidate(self) -> None: - with self._lock: - self._session_token = None - - def credential(self) -> CollectorCredential: - with self._lock: - if ( - self._session_token is not None - and self._session_token[1] - > time.monotonic() + TOKEN_REFRESH_MARGIN_SECONDS - ): - return CollectorCredential("ok", self._session_token[0]) - hf_token = self._credential() - if not hf_token: - return CollectorCredential("missing", None) - try: - status, response = self._post( - self.origin + TOKEN_EXCHANGE_PATH, {}, hf_token - ) - except (OSError, TimeoutError): - return CollectorCredential("unavailable", None) - if status == HTTPStatus.OK and isinstance( - response.get("access_token"), str - ): - try: - expires_in = max( - MINIMUM_TOKEN_LIFETIME_SECONDS, - int(response.get("expires_in", DEFAULT_TOKEN_LIFETIME_SECONDS)), - ) - except (ValueError, TypeError): - return CollectorCredential("unavailable", None) - token = response["access_token"] - self._session_token = (token, time.monotonic() + expires_in) - return CollectorCredential("ok", token) - if status in {HTTPStatus.UNAUTHORIZED, HTTPStatus.FORBIDDEN}: - return CollectorCredential("rejected", None) - return CollectorCredential("unavailable", None) diff --git a/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/collector.py b/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/collector.py deleted file mode 100644 index 8bc19a3b73..0000000000 --- a/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/collector.py +++ /dev/null @@ -1,191 +0,0 @@ -"""Best-effort telemetry delivery; shared authentication lives outside this component.""" - -from __future__ import annotations - -import sys -import time -from collections.abc import Mapping -from http import HTTPStatus -from typing import Any - -from microcosm.build.telemetry_emitter_service.auth import ( - CollectorCredential, - CollectorSession, - _development_collector_url, - _http_post, - _huggingface_token, - validated_origin, -) -from microcosm.build.telemetry_emitter_service.constants import ( - INITIAL_RETRY_SECONDS, - LOCAL_ONLY_MISSING_CREDENTIAL, - LOCAL_ONLY_REJECTED_COLLECTOR_AUTHORIZATION, - LOCAL_ONLY_REJECTED_CREDENTIAL, - LOCAL_ONLY_REJECTED_EVENTS, - LOCAL_ONLY_REJECTED_REGISTRATION, - MAX_RETRY_SECONDS, - NO_CREDENTIAL_MESSAGE, - PRODUCTION_COLLECTOR_URL, - REJECTED_CREDENTIAL_MESSAGE, - REJECTED_EVENTS_MESSAGE, - RUN_EVENTS_PATH_TEMPLATE, - RUN_REGISTRATION_PATH, -) -from microcosm.build.telemetry_emitter_service.spool import EventSpool - -#: Client errors worth retrying: a timeout and a rate limit pass with time. -#: Every other 4xx is the collector's settled answer to the same request, so -#: retrying it only wedges the queue (a payload shape it does not accept, say). -_RETRYABLE_CLIENT_ERRORS = frozenset( - {HTTPStatus.REQUEST_TIMEOUT, HTTPStatus.TOO_MANY_REQUESTS} -) - - -def _permanent_client_error(status: int) -> bool: - return 400 <= int(status) < 500 and status not in _RETRYABLE_CLIENT_ERRORS - - -def _registration_key(registration: Mapping[str, Any]) -> str: - return f"{registration['run_id']}:{registration['producer_id']}" - - -class CollectorDelivery: - """Authenticate queued runs and deliver idempotent event batches.""" - - def __init__( - self, - spool: EventSpool, - *, - development_collector_url: str | None = None, - session: CollectorSession | None = None, - ) -> None: - self.collector_url = ( - _development_collector_url(development_collector_url) - if development_collector_url is not None - else validated_origin(PRODUCTION_COLLECTOR_URL) - ) - self.spool = spool - self.session = session or CollectorSession( - self.collector_url, - post=lambda *args: _http_post(*args), - credential=lambda: _huggingface_token(), - ) - self._registered: set[str] = set() - self._warned_no_token = False - self._warned_denied: set[str] = set() - self._next_attempt_at = 0.0 - self._retry_seconds = INITIAL_RETRY_SECONDS - - def flush_once(self) -> bool: - """Attempt one delivery pass without waiting for retry deadlines.""" - - if time.monotonic() < self._next_attempt_at: - return False - made_progress = False - for registration in self.spool.pending_runs(): - run_id = str(registration["run_id"]) - registration_key = _registration_key(registration) - token = self._collector_token(registration) - if token is None: - continue - if registration_key not in self._registered: - if not self._register(registration, token): - continue - events = self.spool.batch(run_id, str(registration["producer_id"])) - if not events: - continue - try: - status, _ = _http_post( - self.collector_url + RUN_EVENTS_PATH_TEMPLATE.format(run_id=run_id), - {"events": events}, - token, - ) - except (OSError, TimeoutError): - self._defer_retry() - continue - if status == HTTPStatus.ACCEPTED: - self.spool.acknowledge([event["event_id"] for event in events]) - made_progress = True - self._retry_seconds = INITIAL_RETRY_SECONDS - elif status == HTTPStatus.UNAUTHORIZED: - self.session.invalidate() - elif status == HTTPStatus.FORBIDDEN: - self._make_local_only( - registration, - LOCAL_ONLY_REJECTED_COLLECTOR_AUTHORIZATION, - ) - elif _permanent_client_error(status): - self._make_local_only( - registration, - LOCAL_ONLY_REJECTED_EVENTS, - message=REJECTED_EVENTS_MESSAGE.format(status=int(status)), - ) - else: - self._defer_retry() - return made_progress - - def _collector_token(self, registration: Mapping[str, Any]) -> str | None: - result, token = self.session_credential() - if result == "missing": - if not self._warned_no_token: - print(NO_CREDENTIAL_MESSAGE, file=sys.stderr, flush=True) - self._warned_no_token = True - self._make_local_only(registration, LOCAL_ONLY_MISSING_CREDENTIAL) - elif result == "rejected": - self._make_local_only(registration, LOCAL_ONLY_REJECTED_CREDENTIAL) - elif result != "ok": - self._defer_retry() - return token - - def session_credential(self) -> CollectorCredential: - """Compatibility accessor; authentication has no telemetry queue policy.""" - return self.session.credential() - - def _register(self, registration: Mapping[str, Any], token: str) -> bool: - registration_key = _registration_key(registration) - try: - status, _ = _http_post( - self.collector_url + RUN_REGISTRATION_PATH, - registration, - token, - ) - except (OSError, TimeoutError): - self._defer_retry() - return False - if status == HTTPStatus.CREATED: - self._registered.add(registration_key) - self._retry_seconds = INITIAL_RETRY_SECONDS - return True - if status == HTTPStatus.UNAUTHORIZED: - self.session.invalidate() - elif status in {HTTPStatus.FORBIDDEN, HTTPStatus.CONFLICT} or ( - _permanent_client_error(status) - ): - self._make_local_only(registration, LOCAL_ONLY_REJECTED_REGISTRATION) - else: - self._defer_retry() - return False - - def _make_local_only( - self, - registration: Mapping[str, Any], - reason: str, - *, - message: str = REJECTED_CREDENTIAL_MESSAGE, - ) -> None: - run_id = str(registration["run_id"]) - producer_id = str(registration["producer_id"]) - registration_key = _registration_key(registration) - self.spool.make_local_only(run_id, producer_id, reason) - if reason != LOCAL_ONLY_MISSING_CREDENTIAL: - self._warn_denied(registration_key, message) - - def _defer_retry(self) -> None: - self._next_attempt_at = time.monotonic() + self._retry_seconds - self._retry_seconds = min(MAX_RETRY_SECONDS, self._retry_seconds * 2) - - def _warn_denied(self, registration_key: str, message: str) -> None: - if registration_key in self._warned_denied: - return - print(message, file=sys.stderr, flush=True) - self._warned_denied.add(registration_key) diff --git a/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/constants.py b/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/constants.py deleted file mode 100644 index bd7559467b..0000000000 --- a/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/constants.py +++ /dev/null @@ -1,75 +0,0 @@ -"""Configuration and protocol values for the telemetry emitter service.""" - -from __future__ import annotations - -from typing import Final - -RETENTION_DAYS: Final = 7 -MAX_QUEUED_BYTES: Final = 100 * 1024 * 1024 -BATCH_SIZE: Final = 100 -PRODUCTION_COLLECTOR_URL: Final = ( - "https://microcosm-telemetry-389282473430.us-central1.run.app" -) - -LOOPBACK_HOSTS: Final = frozenset({"localhost", "127.0.0.1", "::1"}) -TOKEN_EXCHANGE_PATH: Final = "/v1/auth/huggingface/exchange" -RUN_REGISTRATION_PATH: Final = "/v1/runs" -RUN_EVENTS_PATH_TEMPLATE: Final = "/v1/runs/{run_id}/events" -HTTP_USER_AGENT: Final = "microcosm-telemetry-emitter/1" -MAX_HTTP_RESPONSE_BYTES: Final = 1_048_576 -HTTP_TIMEOUT_SECONDS: Final = 5.0 - -UPLOAD_STATE_PENDING: Final = "pending" -UPLOAD_STATE_LOCAL_ONLY: Final = "local_only" -LOCAL_ONLY_MISSING_CREDENTIAL: Final = "missing_huggingface_credential" -LOCAL_ONLY_REJECTED_CREDENTIAL: Final = "huggingface_credential_rejected" -LOCAL_ONLY_REJECTED_REGISTRATION: Final = "run_registration_rejected" -LOCAL_ONLY_REJECTED_COLLECTOR_AUTHORIZATION: Final = "collector_authorization_rejected" -LOCAL_ONLY_REJECTED_EVENTS: Final = "collector_rejected_events" -LOCAL_ONLY_PRE_ELIGIBILITY: Final = "created_before_upload_eligibility" - -NO_CREDENTIAL_MESSAGE: Final = ( - "Microcosm telemetry is local-only: no ambient Hugging Face credential was " - "found. The dataset build will continue." -) -REJECTED_CREDENTIAL_MESSAGE: Final = ( - "Microcosm telemetry is local-only for this run: the ambient Hugging Face " - "credential was not accepted as a PolicyEngine organization member. The " - "dataset build will continue." -) -REJECTED_EVENTS_MESSAGE: Final = ( - "Microcosm telemetry is local-only for this run: the collector rejected its " - "events (HTTP {status}), so they will not be retried. The dataset build will " - "continue." -) -SERVICE_ORIGIN_HTTPS_ERROR: Final = "service URL must be an HTTPS origin" -SERVICE_ORIGIN_FORMAT_ERROR: Final = ( - "service URL must be an origin without credentials or path data" -) -DEVELOPMENT_COLLECTOR_LOOPBACK_ERROR: Final = ( - "development collector URL must use a loopback address" -) -COLLECTOR_RESPONSE_TOO_LARGE_ERROR: Final = "collector response exceeds 1 MiB" - -DEFAULT_TOKEN_LIFETIME_SECONDS: Final = 3_600 -MINIMUM_TOKEN_LIFETIME_SECONDS: Final = 60 -TOKEN_REFRESH_MARGIN_SECONDS: Final = 30 -INITIAL_RETRY_SECONDS: Final = 1.0 -MAX_RETRY_SECONDS: Final = 60.0 - -DATABASE_TIMEOUT_SECONDS: Final = 5 -PRUNE_INTERVAL_SECONDS: Final = 60.0 - -DEFAULT_HEARTBEAT_SECONDS: Final = 60.0 -DEFAULT_DRAIN_SECONDS: Final = 15.0 -MINIMUM_HEARTBEAT_SECONDS: Final = 1.0 -SOCKET_LISTEN_BACKLOG: Final = 16 -SOCKET_ACCEPT_TIMEOUT_SECONDS: Final = 0.5 -SOCKET_CONNECTION_TIMEOUT_SECONDS: Final = 0.25 -WORKER_INTERVAL_SECONDS: Final = 1.0 -DRAIN_RETRY_SECONDS: Final = 0.5 - -EVENT_OBJECT_ERROR: Final = "event must be an object" -UNSUPPORTED_ACTION_ERROR: Final = "unsupported local telemetry action" -LOCAL_MESSAGE_TOO_LARGE_ERROR: Final = "local telemetry message exceeds 1 MiB" -FAILURE_CLASS_UNEXPECTED_PROCESS_EXIT: Final = "unexpected_process_exit" diff --git a/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/database.py b/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/database.py deleted file mode 100644 index 75d32d96ba..0000000000 --- a/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/database.py +++ /dev/null @@ -1,34 +0,0 @@ -"""SQLAlchemy engine and session construction for the telemetry spool.""" - -from __future__ import annotations - -from pathlib import Path - -from sqlalchemy import URL, create_engine -from sqlalchemy.engine import Engine -from sqlalchemy.orm import Session, sessionmaker - -from microcosm.build.telemetry_emitter_service.constants import ( - DATABASE_TIMEOUT_SECONDS, -) - - -def sqlite_database_url(path: Path | str) -> URL: - """Return a SQLAlchemy URL for a local SQLite spool path.""" - - return URL.create("sqlite+pysqlite", database=str(Path(path))) - - -def create_spool_engine(path: Path | str) -> Engine: - """Create the SQLAlchemy engine used by one emitter service process.""" - - return create_engine( - sqlite_database_url(path), - connect_args={"timeout": DATABASE_TIMEOUT_SECONDS}, - ) - - -def create_spool_session_factory(engine: Engine) -> sessionmaker[Session]: - """Create short-lived ORM sessions bound to the spool engine.""" - - return sessionmaker(engine, expire_on_commit=False) diff --git a/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/graph_publication.py b/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/graph_publication.py deleted file mode 100644 index f1f95ed137..0000000000 --- a/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/graph_publication.py +++ /dev/null @@ -1,414 +0,0 @@ -"""Durable, retryable immutable graph uploads owned by the emitter service. - -Graph bytes stay in preserved files, never in event details. Pending graph jobs -are not subject to the telemetry event queue's time or size pruning. -""" - -from __future__ import annotations - -import hashlib -import json -import re -import time -import uuid -from collections.abc import Callable, Mapping -from pathlib import Path -from urllib.parse import urlsplit - -import httpx -from sqlalchemy import select, update - -from microcosm.build.graph_publication_types import ( - ClaimedGraphPublication, - PublicationFile, - PublicationFileRole, - PublicationInventory, - PublicationReceipt, -) -from microcosm.build.telemetry_emitter_service.auth import ( - CollectorCredential, - validated_origin, -) -from microcosm.build.telemetry_emitter_service.database import ( - create_spool_engine, - create_spool_session_factory, -) -from microcosm.build.telemetry_emitter_service.migrations import upgrade_spool_database -from microcosm.build.telemetry_emitter_service.models import GraphPublicationJob - -MAX_GRAPH_FILE_BYTES = 64 * 1024 * 1024 -MAX_GRAPH_TOTAL_BYTES = 512 * 1024 * 1024 -GRAPH_PUBLICATION_WAIT_SECONDS = 30.0 -GRAPH_PUBLICATION_LEASE_SECONDS = 300.0 -GRAPH_PUBLICATION_STATUS_FILENAME = "publication.status.json" -GRAPH_WORKER_ERROR_MESSAGE = ( - "Microcosm graph publication hit a local queue error; pending jobs remain " - "queued for a later attempt." -) -DEFAULT_GRAPH_PUBLICATION_ORIGIN = "https://microcosm-runs.vercel.app" -GRAPH_PUBLICATION_ROLES = { - "graph": r"graph\.orrery\.json", - "schema": r"graph\.schema\.json", - "index": r"execution\.evidence\.json", - "declaration": r"evidence-[A-Za-z0-9_-]+-graph\.json", - "manifest": r"evidence-[A-Za-z0-9_-]+-manifest\.json", - "binding": r"evidence-[A-Za-z0-9_-]+-binding\.json", - "summaries": r"evidence-[A-Za-z0-9_-]+-summaries\.json", - "upstream": r"upstream-[a-f0-9]{64}\.json", -} - - -def _digest(payload: bytes) -> str: - return hashlib.sha256(payload).hexdigest() - - -def inventory_digest(inventory: PublicationInventory) -> str: - """Digest the versioned field order shared with the publication API.""" - ordered: PublicationInventory = { - "version": inventory["version"], - "publication_id": inventory["publication_id"], - "files": [ - { - "name": item["name"], - "role": item["role"], - "bytes": item["bytes"], - "sha256": item["sha256"], - } - for item in sorted(inventory["files"], key=lambda file: file["name"]) - ], - } - return _digest( - json.dumps(ordered, separators=(",", ":"), ensure_ascii=True).encode() - ) - - -def publication_inventory( - directory: Path, - roles: Mapping[str, PublicationFileRole], - *, - publication_id: str | None = None, -) -> PublicationInventory: - """Hash an explicit allowlist of files; publication identity is not a run ID.""" - files: list[PublicationFile] = [] - for name, role in sorted(roles.items()): - if role not in GRAPH_PUBLICATION_ROLES or not re.fullmatch( - GRAPH_PUBLICATION_ROLES[role], name - ): - raise ValueError("Ineligible graph publication filename or role.") - path = Path(directory) / name - if path.is_symlink() or not path.is_file(): - raise ValueError( - "Graph publication files must be regular, preserved files." - ) - size = path.stat().st_size - if not 0 < size <= MAX_GRAPH_FILE_BYTES: - raise ValueError("Graph publication file exceeds its size limit.") - files.append( - { - "name": name, - "role": role, - "bytes": size, - "sha256": _digest(path.read_bytes()), - } - ) - if ( - not files - or len(files) > 256 - or sum(f["bytes"] for f in files) > MAX_GRAPH_TOTAL_BYTES - or sum(f["role"] == "graph" for f in files) != 1 - ): - raise ValueError("Invalid graph publication inventory.") - id_ = publication_id or uuid.uuid4().hex - if not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_-]{0,127}", id_): - raise ValueError("Invalid graph publication ID.") - return {"version": 1, "publication_id": id_, "files": files} - - -def _write_status(directory: Path, receipt: PublicationReceipt) -> None: - path = directory / GRAPH_PUBLICATION_STATUS_FILENAME - temporary = path.with_suffix(f".{uuid.uuid4().hex}.tmp") - try: - temporary.write_text(json.dumps(receipt, sort_keys=True) + "\n") - temporary.replace(path) - except OSError: - temporary.unlink(missing_ok=True) - - -class GraphPublicationQueue: - """SQLite job records with leases; retained until publication or operator action.""" - - def __init__(self, path: Path) -> None: - self.path = Path(path) - self.path.parent.mkdir(parents=True, exist_ok=True) - self._engine = create_spool_engine(self.path) - upgrade_spool_database(self._engine) - self._sessions = create_spool_session_factory(self._engine) - - def enqueue(self, directory: Path, inventory: PublicationInventory) -> None: - """Durably register exact bytes without a telemetry run or credential.""" - directory = Path(directory).resolve() - expected = publication_inventory( - directory, - {f["name"]: f["role"] for f in inventory["files"]}, - publication_id=inventory["publication_id"], - ) - if expected != inventory: - raise ValueError("Graph publication digest mismatch.") - id_ = inventory["publication_id"] - with self._sessions.begin() as session: - existing = session.get(GraphPublicationJob, id_) - if existing is not None: - if existing.inventory != inventory or existing.directory != str( - directory - ): - raise ValueError( - "Graph publication identity conflicts with a queued job." - ) - return - session.add( - GraphPublicationJob( - publication_id=id_, - inventory=inventory, - directory=str(directory), - status="pending", - attempts=0, - next_attempt_at=0, - lease_until=0, - receipt={ - "version": 1, - "publication_id": id_, - "status": "pending", - "error_code": None, - }, - ) - ) - - def claim( - self, - *, - lease_seconds: float = GRAPH_PUBLICATION_LEASE_SECONDS, - publication_id: str | None = None, - ) -> ClaimedGraphPublication | None: - """Reserve one pending job whose retry delay and previous lease expired. - - A conditional database update allows only one current lease. Workers - that exit leave their jobs eligible again after lease expiry; completion - uses the returned lease timestamp to reject outdated workers. - """ - now = time.time() - with self._sessions.begin() as session: - statement = select(GraphPublicationJob.publication_id) - if publication_id is not None: - statement = statement.where( - GraphPublicationJob.publication_id == publication_id - ) - id_ = session.scalar( - statement.where( - GraphPublicationJob.status == "pending", - GraphPublicationJob.next_attempt_at <= now, - GraphPublicationJob.lease_until <= now, - ) - # Earliest-due first, so a job that keeps failing cannot - # repeatedly displace other pending publications. - .order_by(GraphPublicationJob.next_attempt_at) - .limit(1) - ) - if id_ is None: - return None - claimed_id = session.scalar( - update(GraphPublicationJob) - .where( - GraphPublicationJob.publication_id == id_, - GraphPublicationJob.status == "pending", - GraphPublicationJob.lease_until <= now, - ) - .values(lease_until=now + lease_seconds) - .returning(GraphPublicationJob.publication_id) - ) - if claimed_id is None: - return None - job = session.get(GraphPublicationJob, id_) - assert job is not None, ( - "A claimed publication must exist in its transaction." - ) - return ClaimedGraphPublication( - publication_id=id_, - directory=Path(job.directory), - inventory=job.inventory, - lease_until=job.lease_until, - ) - - def receipt(self, id_: str) -> PublicationReceipt | None: - """Read the latest local result; old HF receipt snapshots remain unchanged.""" - with self._sessions() as session: - job = session.get(GraphPublicationJob, id_) - return None if job is None else job.receipt - - def finish( - self, - id_: str, - *, - error_code: str | None = None, - url: str | None = None, - expected_lease: float | None = None, - ) -> None: - with self._sessions.begin() as session: - statement = update(GraphPublicationJob).where( - GraphPublicationJob.publication_id == id_, - GraphPublicationJob.status == "pending", - ) - if expected_lease is not None: - statement = statement.where( - GraphPublicationJob.lease_until == expected_lease - ) - job = session.scalar( - statement.values( - attempts=GraphPublicationJob.attempts + 1, - status="published" if url else "pending", - lease_until=0, - ).returning(GraphPublicationJob) - ) - # Completion is conditional in SQL, not a read-then-write check: - # expired workers cannot release another lease or replace success. - if job is None: - return - job.next_attempt_at = time.time() + min(300, 2 ** min(job.attempts, 8)) - receipt: PublicationReceipt = { - "version": 1, - "publication_id": id_, - "status": job.status, - "error_code": error_code, - "url": url, - "attempts": job.attempts, - "inventory_sha256": inventory_digest(job.inventory), - } - job.receipt = receipt - directory = Path(job.directory) - # The queue row is authoritative. This separate latest-result file (not - # the build/HF receipt) is a convenience copy, written only after the - # commit: a deleted preserved directory must not roll back the result. - _write_status(directory, receipt) - - def retry(self, id_: str | None = None) -> None: - """Make preserved pending jobs eligible again, including missing credentials.""" - statement = update(GraphPublicationJob).where( - GraphPublicationJob.status == "pending" - ) - if id_ is not None: - statement = statement.where(GraphPublicationJob.publication_id == id_) - with self._sessions.begin() as session: - session.execute(statement.values(next_attempt_at=0)) - - -class GraphPublicationDelivery: - """Upload exact files with scoped signed URLs, then seal the publication.""" - - def __init__( - self, - queue: GraphPublicationQueue, - *, - credential: Callable[[], CollectorCredential], - origin: str, - client: httpx.Client | None = None, - invalidate_credential: Callable[[], None] | None = None, - ) -> None: - self.queue = queue - self.credential = credential - self.origin = validated_origin(origin, allow_loopback_http=True) - self.client = client or httpx.Client(timeout=60, follow_redirects=False) - self.invalidate_credential = invalidate_credential - - def flush_once(self, *, publication_id: str | None = None) -> bool: - """Try one job; failures retain it without changing telemetry eligibility.""" - job = self.queue.claim(publication_id=publication_id) - if job is None: - return False - id_, inventory, directory = ( - job.publication_id, - job.inventory, - job.directory, - ) - error = "publication_unavailable" - lease = job.lease_until - if not directory.is_dir(): - # Retained until an operator restores the files to this directory; - # the retry backoff lets other pending jobs proceed meanwhile. - self.queue.finish( - id_, error_code="preserved_files_missing", expected_lease=lease - ) - return False - try: - status, token = self.credential() - if token is None: - self.queue.finish( - id_, error_code=f"credential_{status}", expected_lease=lease - ) - return False - current = publication_inventory( - directory, - {f["name"]: f["role"] for f in inventory["files"]}, - publication_id=id_, - ) - if current != inventory: - raise ValueError("preserved_bytes_changed") - headers = {"Authorization": f"Bearer {token}"} - response = self.client.post( - self.origin + "/api/uploads/", json=inventory, headers=headers - ) - response.raise_for_status() - body = response.json() - uploads = body.get("uploads") - if not isinstance(uploads, list): - raise ValueError("invalid_upload_response") - files = {f["name"]: f for f in inventory["files"]} - for upload in uploads: - name = upload.get("name") - if name not in files: - raise ValueError("invalid_upload_response") - if upload.get("already_uploaded"): - continue - url = urlsplit(upload["url"]) - if ( - url.scheme != "https" - or not ( - (url.hostname == "vercel.com" and url.path == "/api/blob/") - or url.hostname == "blob.vercel-storage.com" - or (url.hostname or "").endswith(".blob.vercel-storage.com") - ) - or url.username - or url.password - ): - raise ValueError("invalid_upload_url") - with (directory / name).open("rb") as stream: - result = self.client.put( - upload["url"], - content=stream, - headers={ - "Content-Type": "application/json", - "Content-Length": str(files[name]["bytes"]), - }, - ) - result.raise_for_status() - response = self.client.post( - self.origin + "/api/publications/", json=inventory, headers=headers - ) - response.raise_for_status() - result = response.json() - if result.get("publication_id") != id_ or result.get( - "inventory_sha256" - ) != inventory_digest(inventory): - raise ValueError("invalid_publication_response") - self.queue.finish( - id_, url=f"{self.origin}/runs/{id_}/", expected_lease=lease - ) - return True - except httpx.HTTPStatusError as exception: - error = f"publication_http_{exception.response.status_code}" - if exception.response.status_code == 401 and self.invalidate_credential: - self.invalidate_credential() - except httpx.HTTPError: - error = "publication_network_error" - except (OSError, ValueError, KeyError, TypeError): - error = "publication_invalid_evidence_or_response" - self.queue.finish(id_, error_code=error, expected_lease=lease) - return False diff --git a/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/main.py b/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/main.py deleted file mode 100644 index 51d18fa742..0000000000 --- a/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/main.py +++ /dev/null @@ -1,66 +0,0 @@ -"""Command-line construction for the telemetry emitter service.""" - -from __future__ import annotations - -import argparse -import json -from pathlib import Path - -from microcosm.build.telemetry_emitter_service.collector import CollectorDelivery -from microcosm.build.telemetry_emitter_service.constants import ( - DEFAULT_HEARTBEAT_SECONDS, -) -from microcosm.build.telemetry_emitter_service.graph_publication import ( - DEFAULT_GRAPH_PUBLICATION_ORIGIN, - GraphPublicationDelivery, - GraphPublicationQueue, -) -from microcosm.build.telemetry_emitter_service.resources import ProcessTreeSampler -from microcosm.build.telemetry_emitter_service.runtime import EmitterService -from microcosm.build.telemetry_emitter_service.spool import EventSpool - - -def build_parser() -> argparse.ArgumentParser: - """Create the service command-line parser.""" - - parser = argparse.ArgumentParser() - parser.add_argument("--socket", type=Path, required=True) - parser.add_argument("--spool", type=Path, required=True) - parser.add_argument("--development-collector-url") - parser.add_argument("--registration-json", help="Omit for graph-only delivery.") - parser.add_argument("--parent-pid", type=int, required=True) - parser.add_argument( - "--heartbeat-seconds", - type=float, - default=DEFAULT_HEARTBEAT_SECONDS, - ) - return parser - - -def main(argv: list[str] | None = None) -> int: - """Run the telemetry emitter service until the build disconnects.""" - - args = build_parser().parse_args(argv) - registration = ( - json.loads(args.registration_json) if args.registration_json else None - ) - spool = EventSpool(args.spool) - delivery = CollectorDelivery( - spool, development_collector_url=args.development_collector_url - ) - service = EmitterService( - socket_path=args.socket, - registration=registration, - spool=spool, - delivery=delivery, - graph_delivery=GraphPublicationDelivery( - GraphPublicationQueue(args.spool), - credential=delivery.session.credential, - invalidate_credential=delivery.session.invalidate, - origin=DEFAULT_GRAPH_PUBLICATION_ORIGIN, - ), - sampler=ProcessTreeSampler(args.parent_pid), - heartbeat_seconds=args.heartbeat_seconds, - ) - service.run() - return 0 diff --git a/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/migrations.py b/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/migrations.py deleted file mode 100644 index 8446a5a2f8..0000000000 --- a/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/migrations.py +++ /dev/null @@ -1,65 +0,0 @@ -"""Programmatic Alembic migration entry points for the telemetry spool.""" - -from __future__ import annotations - -import fcntl -from collections.abc import Iterator -from contextlib import contextmanager -from importlib.resources import as_file, files -from pathlib import Path - -from alembic import command -from alembic.config import Config -from alembic.migration import MigrationContext -from alembic.script import ScriptDirectory -from sqlalchemy.engine import Connection, Engine - -from microcosm.build.telemetry_emitter_service.database import create_spool_engine - -_MIGRATION_TARGET = "head" - - -@contextmanager -def alembic_config( - *, - connection: Connection | None = None, -) -> Iterator[Config]: - """Yield Alembic configuration with migrations on the filesystem.""" - - migration_resources = files(__package__).joinpath("alembic") - with as_file(migration_resources) as migration_directory: - config = Config() - config.set_main_option("script_location", str(migration_directory)) - if connection is not None: - config.attributes["connection"] = connection - yield config - - -def upgrade_spool_database(engine: Engine) -> None: - """Apply every committed spool migration to an engine.""" - - # Concurrent build processes share this spool and must serialize upgrades. - path = Path(engine.url.database) - with path.with_suffix(".migration.lock").open("a") as lock: - fcntl.flock(lock, fcntl.LOCK_EX) - with engine.begin() as connection: - with alembic_config(connection=connection) as config: - command.upgrade(config, _MIGRATION_TARGET) - - -def current_database_revision(path: Path | str) -> str | None: - """Return the Alembic revision recorded by one spool database.""" - - engine = create_spool_engine(path) - try: - with engine.connect() as connection: - return MigrationContext.configure(connection).get_current_revision() - finally: - engine.dispose() - - -def migration_head_revision() -> str | None: - """Return the single current head from the packaged migration history.""" - - with alembic_config() as config: - return ScriptDirectory.from_config(config).get_current_head() diff --git a/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/models.py b/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/models.py deleted file mode 100644 index 610576a94f..0000000000 --- a/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/models.py +++ /dev/null @@ -1,152 +0,0 @@ -"""SQLAlchemy ORM models for the local telemetry spool.""" - -from __future__ import annotations - -import json -from typing import Any - -from sqlalchemy import ( - Float, - ForeignKeyConstraint, - Index, - Integer, - Text, - UniqueConstraint, -) -from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column, relationship -from sqlalchemy.types import TypeDecorator - -from microcosm.build.graph_publication_types import ( - GraphJobStatus, - PublicationInventory, - PublicationReceipt, -) -from microcosm.build.telemetry_emitter_service.constants import ( - UPLOAD_STATE_PENDING, -) - - -class JsonObjectText(TypeDecorator[dict[str, Any]]): - """Store JSON objects in the spool's existing text-column format.""" - - impl = Text - cache_ok = True - - def process_bind_param( - self, - value: dict[str, Any] | None, - dialect, - ) -> str | None: - if value is None: - return None - return _serialize_json_object(value) - - def process_result_value( - self, - value: str | None, - dialect, - ) -> dict[str, Any] | None: - if value is None: - return None - decoded = json.loads(value) - if not isinstance(decoded, dict): - raise ValueError("telemetry spool JSON must contain an object") - return decoded - - -def serialized_json_length(value: dict[str, Any]) -> int: - """Return the stored character count for one mapped JSON object.""" - - return len(_serialize_json_object(value)) - - -def _serialize_json_object(value: dict[str, Any]) -> str: - return json.dumps( - value, - separators=(",", ":"), - sort_keys=True, - allow_nan=False, - ) - - -class SpoolModel(DeclarativeBase): - """Declarative base for the local telemetry spool.""" - - -class TelemetryRunRecord(SpoolModel): - """One telemetry producer registered for one build run.""" - - __tablename__ = "telemetry_runs" - - run_id: Mapped[str] = mapped_column(Text, primary_key=True) - producer_id: Mapped[str] = mapped_column(Text, primary_key=True) - registration: Mapped[dict[str, Any]] = mapped_column( - "registration_json", - JsonObjectText, - nullable=False, - ) - next_sequence: Mapped[int] = mapped_column(Integer, nullable=False, default=1) - upload_state: Mapped[str] = mapped_column( - Text, - nullable=False, - default=UPLOAD_STATE_PENDING, - ) - local_only_reason: Mapped[str | None] = mapped_column(Text) - updated_at: Mapped[str] = mapped_column(Text, nullable=False) - events: Mapped[list[TelemetryEventRecord]] = relationship( - back_populates="run", - cascade="all, delete-orphan", - ) - - -class TelemetryEventRecord(SpoolModel): - """One ordered telemetry event awaiting collector acknowledgement.""" - - __tablename__ = "telemetry_events" - __table_args__ = ( - UniqueConstraint( - "run_id", - "producer_id", - "sequence", - name="uq_telemetry_events_run_producer_sequence", - ), - ForeignKeyConstraint( - ["run_id", "producer_id"], - ["telemetry_runs.run_id", "telemetry_runs.producer_id"], - name="fk_telemetry_events_run", - ), - Index( - "telemetry_events_run_sequence", - "run_id", - "producer_id", - "sequence", - ), - ) - - event_id: Mapped[str] = mapped_column(Text, primary_key=True) - run_id: Mapped[str] = mapped_column(Text, nullable=False) - producer_id: Mapped[str] = mapped_column(Text, nullable=False) - sequence: Mapped[int] = mapped_column(Integer, nullable=False) - payload: Mapped[dict[str, Any]] = mapped_column( - "payload_json", - JsonObjectText, - nullable=False, - ) - created_at: Mapped[str] = mapped_column(Text, nullable=False) - run: Mapped[TelemetryRunRecord] = relationship(back_populates="events") - - -class GraphPublicationJob(SpoolModel): - """Durable artifact delivery, independent of event retention and run IDs.""" - - __tablename__ = "graph_publication_jobs" - publication_id: Mapped[str] = mapped_column(Text, primary_key=True) - inventory: Mapped[PublicationInventory] = mapped_column( - JsonObjectText, nullable=False - ) - directory: Mapped[str] = mapped_column(Text, nullable=False) - status: Mapped[GraphJobStatus] = mapped_column(Text, nullable=False) - attempts: Mapped[int] = mapped_column(Integer, nullable=False) - next_attempt_at: Mapped[float] = mapped_column(Float, nullable=False) - lease_until: Mapped[float] = mapped_column(Float, nullable=False) - receipt: Mapped[PublicationReceipt] = mapped_column(JsonObjectText, nullable=False) diff --git a/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/resources.py b/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/resources.py deleted file mode 100644 index f8ede94ee7..0000000000 --- a/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/resources.py +++ /dev/null @@ -1,137 +0,0 @@ -"""Process-tree resource sampling for telemetry events.""" - -from __future__ import annotations - -import os -from pathlib import Path -from typing import Any - -try: - import psutil -except ModuleNotFoundError: # Base installs use the standard-library fallback. - psutil = None - - -def _empty_sample(peak_rss: int) -> dict[str, Any]: - return { - "cpu_user_seconds": 0.0, - "cpu_system_seconds": 0.0, - "rss_bytes": 0, - "peak_rss_bytes": peak_rss, - } - - -class ProcessTreeSampler: - """Collect cumulative CPU and resident memory for a build process tree.""" - - def __init__(self, parent_pid: int) -> None: - self.parent_pid = parent_pid - self._peak_rss = 0 - self._parent_create_time = self._create_time() - - def _create_time(self) -> float | str | None: - if psutil is not None: - try: - return psutil.Process(self.parent_pid).create_time() - except (psutil.Error, OSError): - return None - stat_path = Path(f"/proc/{self.parent_pid}/stat") - try: - return stat_path.read_text().rsplit(")", 1)[1].split()[19] - except (OSError, IndexError): - return None - - def parent_alive(self) -> bool: - """Return whether the sampled process still has its original identity.""" - - if psutil is not None: - try: - process = psutil.Process(self.parent_pid) - return ( - self._parent_create_time is not None - and process.create_time() == self._parent_create_time - and process.is_running() - and process.status() != psutil.STATUS_ZOMBIE - ) - except (psutil.Error, OSError): - return False - try: - os.kill(self.parent_pid, 0) - except OSError: - return False - current = self._create_time() - # /proc supplies a process-start tick that protects against PID reuse. - # On platforms without it, existence is the best standard-library test. - return self._parent_create_time is None or current == self._parent_create_time - - def sample(self) -> dict[str, Any]: - """Sample the parent and every currently visible child process.""" - - if psutil is None: - return self._fallback_sample() - processes = [] - try: - parent = psutil.Process(self.parent_pid) - processes = [parent, *parent.children(recursive=True)] - except (psutil.Error, OSError): - pass - user = 0.0 - system = 0.0 - rss = 0 - for process in processes: - try: - cpu = process.cpu_times() - user += float(cpu.user) + float(getattr(cpu, "children_user", 0.0)) - system += float(cpu.system) + float( - getattr(cpu, "children_system", 0.0) - ) - rss += int(process.memory_info().rss) - except (psutil.Error, OSError): - continue - self._peak_rss = max(self._peak_rss, rss) - return { - "cpu_user_seconds": user, - "cpu_system_seconds": system, - "rss_bytes": rss, - "peak_rss_bytes": self._peak_rss, - } - - def _fallback_sample(self) -> dict[str, Any]: - """Sample Linux process trees when the country engine is not installed.""" - - process_rows: dict[int, tuple[int, float, float, int]] = {} - try: - clock_ticks = float(os.sysconf("SC_CLK_TCK")) - page_size = int(os.sysconf("SC_PAGE_SIZE")) - except (AttributeError, OSError, ValueError): - return _empty_sample(self._peak_rss) - for stat_path in Path("/proc").glob("[0-9]*/stat"): - try: - pid = int(stat_path.parent.name) - fields = stat_path.read_text().rsplit(")", 1)[1].split() - process_rows[pid] = ( - int(fields[1]), - (int(fields[11]) + int(fields[13])) / clock_ticks, - (int(fields[12]) + int(fields[14])) / clock_ticks, - int(fields[21]) * page_size, - ) - except (OSError, ValueError, IndexError): - continue - selected = {self.parent_pid} - changed = True - while changed: - changed = False - for pid, (parent, *_rest) in process_rows.items(): - if parent in selected and pid not in selected: - selected.add(pid) - changed = True - user = sum(process_rows[pid][1] for pid in selected if pid in process_rows) - system = sum(process_rows[pid][2] for pid in selected if pid in process_rows) - rss = sum(process_rows[pid][3] for pid in selected if pid in process_rows) - self._peak_rss = max(self._peak_rss, rss) - return { - "cpu_user_seconds": user, - "cpu_system_seconds": system, - "rss_bytes": rss, - "peak_rss_bytes": self._peak_rss, - } diff --git a/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/runtime.py b/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/runtime.py deleted file mode 100644 index 3a95be8157..0000000000 --- a/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/runtime.py +++ /dev/null @@ -1,264 +0,0 @@ -"""Existing local service with independent telemetry and graph workers.""" - -from __future__ import annotations - -import json -import os -import socket -import sys -import threading -import time -from collections.abc import Mapping -from pathlib import Path -from typing import Any - -from microcosm.build.telemetry_emitter_service.collector import CollectorDelivery -from microcosm.build.telemetry_emitter_service.constants import ( - DEFAULT_DRAIN_SECONDS, - DRAIN_RETRY_SECONDS, - EVENT_OBJECT_ERROR, - FAILURE_CLASS_UNEXPECTED_PROCESS_EXIT, - LOCAL_MESSAGE_TOO_LARGE_ERROR, - MINIMUM_HEARTBEAT_SECONDS, - SOCKET_ACCEPT_TIMEOUT_SECONDS, - SOCKET_CONNECTION_TIMEOUT_SECONDS, - SOCKET_LISTEN_BACKLOG, - UNSUPPORTED_ACTION_ERROR, - WORKER_INTERVAL_SECONDS, -) -from microcosm.build.telemetry_emitter_service.graph_publication import ( - GRAPH_WORKER_ERROR_MESSAGE, - GraphPublicationDelivery, -) -from microcosm.build.telemetry_emitter_service.resources import ProcessTreeSampler -from microcosm.build.telemetry_emitter_service.spool import EventSpool -from microcosm.build.telemetry_emitter_service.timestamps import utc_now -from microcosm.build.telemetry_protocol import ( - ACTION_CLOSE, - ACTION_EVENT, - ACTION_PING, - EVENT_TYPE_HEARTBEAT, - EVENT_TYPE_RUN, - LOCAL_ACKNOWLEDGEMENT_ERROR, - LOCAL_ACKNOWLEDGEMENT_OK, - LOCAL_MESSAGE_DELIMITER, - LOCAL_SOCKET_READ_BYTES, - MAX_LOCAL_MESSAGE_BYTES, - STAGE_BLOCKED, - STAGE_COMPLETE, - STAGE_CREATED, - STAGE_FAILED, - STATUS_FAILED, - STATUS_PROGRESS, - UNEXPECTED_PROCESS_EXIT_MESSAGE, -) - - -def _heartbeat_event(stage_id: str) -> dict[str, Any]: - return { - "timestamp": utc_now(), - "event_type": EVENT_TYPE_HEARTBEAT, - "stage_id": stage_id, - "status": STATUS_PROGRESS, - "message": None, - "details": {}, - } - - -def _unexpected_exit_event(stage_id: str) -> dict[str, Any]: - return { - "timestamp": utc_now(), - "event_type": EVENT_TYPE_RUN, - "stage_id": STAGE_FAILED, - "status": STATUS_FAILED, - "message": UNEXPECTED_PROCESS_EXIT_MESSAGE, - "details": { - "failure_class": FAILURE_CLASS_UNEXPECTED_PROCESS_EXIT, - "failed_during": stage_id, - }, - } - - -class EmitterService: - """Private local socket server with a concurrent delivery worker.""" - - def __init__( - self, - *, - socket_path: Path, - registration: Mapping[str, Any] | None, - spool: EventSpool, - delivery: CollectorDelivery, - sampler: ProcessTreeSampler, - heartbeat_seconds: float, - drain_seconds: float = DEFAULT_DRAIN_SECONDS, - graph_delivery: GraphPublicationDelivery | None = None, - ) -> None: - self.socket_path = socket_path - self.registration = dict(registration) if registration is not None else None - self.graph_delivery = graph_delivery - self.spool = spool - self.delivery = delivery - self.sampler = sampler - self.heartbeat_seconds = max( - MINIMUM_HEARTBEAT_SECONDS, - heartbeat_seconds, - ) - self.drain_seconds = max(0.0, drain_seconds) - self._stop = threading.Event() - self._last_stage = STAGE_CREATED - - def run(self) -> None: - """Serve local messages until the client closes or exits.""" - - if self.registration is not None: - self.spool.register(self.registration) - self.socket_path.parent.mkdir(parents=True, exist_ok=True) - try: - self.socket_path.unlink(missing_ok=True) - with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as server: - server.bind(str(self.socket_path)) - os.chmod(self.socket_path, 0o600) - server.listen(SOCKET_LISTEN_BACKLOG) - server.settimeout(SOCKET_ACCEPT_TIMEOUT_SECONDS) - workers = [] - if self.registration is not None: - workers.append( - threading.Thread( - target=self._worker, - args=(self.registration,), - daemon=True, - ) - ) - if self.graph_delivery is not None: - workers.append( - threading.Thread( - target=self._graph_worker, - args=(self.graph_delivery,), - daemon=True, - ) - ) - for worker in workers: - worker.start() - self._serve(server) - deadline = ( - time.monotonic() + self.drain_seconds + WORKER_INTERVAL_SECONDS - ) - for worker in workers: - worker.join(timeout=max(0, deadline - time.monotonic())) - finally: - self._stop.set() - self.socket_path.unlink(missing_ok=True) - try: - self.socket_path.parent.rmdir() - except OSError: - pass - - def _serve(self, server: socket.socket) -> None: - while not self._stop.is_set(): - if not self.sampler.parent_alive(): - if self.registration is not None: - self.spool.append( - self.registration, - _unexpected_exit_event(self._last_stage), - resources=self.sampler.sample(), - ) - self._stop.set() - break - try: - connection, _ = server.accept() - except TimeoutError: - continue - with connection: - connection.settimeout(SOCKET_CONNECTION_TIMEOUT_SECONDS) - response = self._serve_connection(connection) - try: - connection.sendall(response) - except OSError: - pass - - def _serve_connection(self, connection: socket.socket) -> bytes: - try: - message = self._read_message(connection) - self._handle(message) - except Exception: - return LOCAL_ACKNOWLEDGEMENT_ERROR - return LOCAL_ACKNOWLEDGEMENT_OK - - @staticmethod - def _read_message(connection: socket.socket) -> dict[str, Any]: - chunks = bytearray() - while len(chunks) <= MAX_LOCAL_MESSAGE_BYTES: - data = connection.recv(LOCAL_SOCKET_READ_BYTES) - if not data: - break - chunks.extend(data) - if LOCAL_MESSAGE_DELIMITER in data: - break - if len(chunks) > MAX_LOCAL_MESSAGE_BYTES: - raise ValueError(LOCAL_MESSAGE_TOO_LARGE_ERROR) - return json.loads(bytes(chunks).split(LOCAL_MESSAGE_DELIMITER, 1)[0]) - - def _handle(self, message: Mapping[str, Any]) -> None: - action = message.get("action") - if action == ACTION_EVENT and self.registration is not None: - event = message.get("event") - if not isinstance(event, Mapping): - raise ValueError(EVENT_OBJECT_ERROR) - stage_id = event.get("stage_id") - if isinstance(stage_id, str) and stage_id not in { - STAGE_BLOCKED, - STAGE_COMPLETE, - STAGE_FAILED, - }: - self._last_stage = stage_id - self.spool.append( - self.registration, - event, - resources=self.sampler.sample(), - ) - elif action == ACTION_CLOSE: - self._stop.set() - elif action == ACTION_PING: - return - else: - raise ValueError(UNSUPPORTED_ACTION_ERROR) - - def _worker(self, registration: Mapping[str, Any]) -> None: - next_heartbeat = time.monotonic() + self.heartbeat_seconds - while not self._stop.wait(WORKER_INTERVAL_SECONDS): - now = time.monotonic() - # Sample every worker iteration so short-lived build children are - # much less likely to disappear between stage and heartbeat events. - self.sampler.sample() - if now >= next_heartbeat: - self.spool.append( - registration, - _heartbeat_event(self._last_stage), - resources=self.sampler.sample(), - ) - next_heartbeat = now + self.heartbeat_seconds - self.delivery.flush_once() - self._drain() - - def _graph_worker(self, delivery: GraphPublicationDelivery) -> None: - # Uploads may take much longer than a heartbeat. They never run on the - # telemetry worker or the local socket thread. Pending jobs survive exit. - warned = False - while not self._stop.wait(WORKER_INTERVAL_SECONDS): - try: - delivery.flush_once() - except Exception: - # A database or filesystem error leaves the job leased for a - # later retry; it must not end delivery of every other job. - if not warned: - print(GRAPH_WORKER_ERROR_MESSAGE, file=sys.stderr, flush=True) - warned = True - - def _drain(self) -> None: - deadline = time.monotonic() + self.drain_seconds - while self.spool.has_deliverable() and time.monotonic() < deadline: - if not self.delivery.flush_once(): - remaining = deadline - time.monotonic() - if remaining > 0: - time.sleep(min(DRAIN_RETRY_SECONDS, remaining)) diff --git a/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/spool.py b/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/spool.py deleted file mode 100644 index 1ef43c5269..0000000000 --- a/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/spool.py +++ /dev/null @@ -1,258 +0,0 @@ -"""Durable local queue for telemetry events.""" - -from __future__ import annotations - -import threading -import time -import uuid -from collections.abc import Mapping -from datetime import UTC, datetime, timedelta -from pathlib import Path -from typing import Any - -from sqlalchemy import func, select -from sqlalchemy.orm import Session - -from microcosm.build.telemetry_emitter_service.constants import ( - BATCH_SIZE, - MAX_QUEUED_BYTES, - PRUNE_INTERVAL_SECONDS, - RETENTION_DAYS, - UPLOAD_STATE_LOCAL_ONLY, - UPLOAD_STATE_PENDING, -) -from microcosm.build.telemetry_emitter_service.database import ( - create_spool_engine, - create_spool_session_factory, -) -from microcosm.build.telemetry_emitter_service.migrations import ( - upgrade_spool_database, -) -from microcosm.build.telemetry_emitter_service.models import ( - TelemetryEventRecord, - TelemetryRunRecord, - serialized_json_length, -) -from microcosm.build.telemetry_emitter_service.timestamps import utc_now -from microcosm.build.telemetry_protocol import TELEMETRY_SCHEMA_VERSION - - -class EventSpool: - """Small SQLite queue shared by successive emitter service processes.""" - - def __init__(self, path: Path | str) -> None: - self.path = Path(path) - self.path.parent.mkdir(parents=True, exist_ok=True) - try: - self.path.parent.chmod(0o700) - except OSError: - pass - self._engine = create_spool_engine(self.path) - upgrade_spool_database(self._engine) - self._session_factory = create_spool_session_factory(self._engine) - self._lock = threading.RLock() - self._last_prune_at = 0.0 - self.prune() - - def register(self, registration: Mapping[str, Any]) -> None: - """Create or refresh a producer registration.""" - - run_id = str(registration["run_id"]) - producer_id = str(registration["producer_id"]) - with self._lock, self._session_factory.begin() as session: - existing = session.get(TelemetryRunRecord, (run_id, producer_id)) - if existing is not None: - if existing.registration.get("producer_id") != registration.get( - "producer_id" - ): - raise ValueError(f"run_id {run_id!r} already has another producer") - existing.registration = dict(registration) - existing.updated_at = utc_now() - return - session.add( - TelemetryRunRecord( - run_id=run_id, - producer_id=producer_id, - registration=dict(registration), - next_sequence=1, - upload_state=UPLOAD_STATE_PENDING, - local_only_reason=None, - updated_at=utc_now(), - ) - ) - - def append( - self, - registration: Mapping[str, Any], - event: Mapping[str, Any], - *, - resources: Mapping[str, Any] | None = None, - ) -> dict[str, Any]: - """Append an event and assign its stable producer sequence.""" - - run_id = str(registration["run_id"]) - producer_id = str(registration["producer_id"]) - with self._lock, self._session_factory.begin() as session: - run = session.get(TelemetryRunRecord, (run_id, producer_id)) - if run is None: - raise KeyError(run_id) - sequence = run.next_sequence - event_id = uuid.uuid4().hex - payload = { - "schema_version": TELEMETRY_SCHEMA_VERSION, - "event_id": event_id, - "run_id": run_id, - "producer_id": producer_id, - "sequence": sequence, - "timestamp": event.get("timestamp") or utc_now(), - "event_type": event["event_type"], - "stage_id": event.get("stage_id"), - "status": event["status"], - "message": event.get("message"), - "details": event.get("details") or {}, - "resources": resources, - } - session.add( - TelemetryEventRecord( - event_id=event_id, - run_id=run_id, - producer_id=producer_id, - sequence=sequence, - payload=payload, - created_at=utc_now(), - ) - ) - run.next_sequence = sequence + 1 - run.updated_at = utc_now() - if time.monotonic() - self._last_prune_at >= PRUNE_INTERVAL_SECONDS: - self.prune() - return payload - - def pending_runs(self) -> list[dict[str, Any]]: - """Return registrations that have events eligible for delivery.""" - - statement = ( - select(TelemetryRunRecord) - .join(TelemetryRunRecord.events) - .where(TelemetryRunRecord.upload_state == UPLOAD_STATE_PENDING) - .order_by(TelemetryRunRecord.updated_at) - .distinct() - ) - with self._lock, self._session_factory() as session: - runs = session.scalars(statement).all() - return [dict(run.registration) for run in runs] - - def make_local_only( - self, - run_id: str, - producer_id: str, - reason: str, - ) -> None: - """Permanently exclude one producer's queued events from upload.""" - - with self._lock, self._session_factory.begin() as session: - run = session.get(TelemetryRunRecord, (run_id, producer_id)) - if run is None: - return - run.upload_state = UPLOAD_STATE_LOCAL_ONLY - run.local_only_reason = reason - run.updated_at = utc_now() - - def has_deliverable(self) -> bool: - """Return whether any queued event remains eligible for delivery.""" - - statement = ( - select(TelemetryEventRecord) - .join(TelemetryEventRecord.run) - .where(TelemetryRunRecord.upload_state == UPLOAD_STATE_PENDING) - .limit(1) - ) - with self._lock, self._session_factory() as session: - return session.scalar(statement) is not None - - def batch( - self, - run_id: str, - producer_id: str, - limit: int = BATCH_SIZE, - ) -> list[dict[str, Any]]: - """Return the next ordered batch for one producer.""" - - statement = ( - select(TelemetryEventRecord) - .where( - TelemetryEventRecord.run_id == run_id, - TelemetryEventRecord.producer_id == producer_id, - ) - .order_by(TelemetryEventRecord.sequence) - .limit(limit) - ) - with self._lock, self._session_factory() as session: - events = session.scalars(statement).all() - return [dict(event.payload) for event in events] - - def acknowledge(self, event_ids: list[str]) -> None: - """Remove events acknowledged by the collector.""" - - if not event_ids: - return - statement = select(TelemetryEventRecord).where( - TelemetryEventRecord.event_id.in_(event_ids) - ) - with self._lock, self._session_factory.begin() as session: - for event in session.scalars(statement): - session.delete(event) - - def has_pending(self) -> bool: - """Return whether any event remains in local storage.""" - - statement = select(TelemetryEventRecord).limit(1) - with self._lock, self._session_factory() as session: - return session.scalar(statement) is not None - - def prune(self) -> None: - """Enforce the age and total-size retention limits.""" - - cutoff = (datetime.now(UTC) - timedelta(days=RETENTION_DAYS)).isoformat() - with self._lock, self._session_factory.begin() as session: - expired_events = session.scalars( - select(TelemetryEventRecord).where( - TelemetryEventRecord.created_at < cutoff - ) - ) - for event in expired_events: - session.delete(event) - session.flush() - stored_characters = session.scalar( - select( - func.coalesce( - func.sum(func.length(TelemetryEventRecord.payload)), - 0, - ) - ) - ) - excess = int(stored_characters or 0) - MAX_QUEUED_BYTES - if excess > 0: - self._remove_oldest_bytes(session, excess) - expired_runs = session.scalars( - select(TelemetryRunRecord).where( - TelemetryRunRecord.updated_at < cutoff, - ~TelemetryRunRecord.events.any(), - ) - ) - for run in expired_runs: - session.delete(run) - self._last_prune_at = time.monotonic() - - @staticmethod - def _remove_oldest_bytes(session: Session, excess: int) -> None: - statement = select(TelemetryEventRecord).order_by( - TelemetryEventRecord.created_at, - TelemetryEventRecord.sequence, - ) - removed = 0 - for event in session.scalars(statement): - session.delete(event) - removed += serialized_json_length(event.payload) - if removed >= excess: - break diff --git a/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/timestamps.py b/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/timestamps.py deleted file mode 100644 index 3f1f39a911..0000000000 --- a/packages/microcosm-build/src/microcosm/build/telemetry_emitter_service/timestamps.py +++ /dev/null @@ -1,11 +0,0 @@ -"""Timestamp helpers shared by the telemetry emitter service.""" - -from __future__ import annotations - -from datetime import UTC, datetime - - -def utc_now() -> str: - """Return the current UTC timestamp in ISO 8601 format.""" - - return datetime.now(UTC).isoformat() diff --git a/packages/microcosm-build/src/microcosm/build/telemetry_protocol.py b/packages/microcosm-build/src/microcosm/build/telemetry_protocol.py deleted file mode 100644 index cb27eb4e5c..0000000000 --- a/packages/microcosm-build/src/microcosm/build/telemetry_protocol.py +++ /dev/null @@ -1,70 +0,0 @@ -"""Shared constants for local and hosted Microcosm telemetry messages.""" - -from __future__ import annotations - -from types import MappingProxyType -from typing import Final, Literal - -type TelemetryAction = Literal["event", "close", "ping"] -type TelemetryEventType = Literal[ - "run", "stage", "progress", "calibration", "heartbeat" -] -type TelemetryStatus = Literal["started", "progress", "completed", "failed", "blocked"] - -TELEMETRY_SCHEMA_VERSION: Final = 1 - -ACTION_EVENT: Final = "event" -ACTION_CLOSE: Final = "close" -ACTION_PING: Final = "ping" - -EVENT_TYPE_RUN: Final = "run" -EVENT_TYPE_STAGE: Final = "stage" -EVENT_TYPE_PROGRESS: Final = "progress" -EVENT_TYPE_CALIBRATION: Final = "calibration" -EVENT_TYPE_HEARTBEAT: Final = "heartbeat" - -STATUS_STARTED: Final = "started" -STATUS_PROGRESS: Final = "progress" -STATUS_COMPLETED: Final = "completed" -STATUS_FAILED: Final = "failed" -#: The run reached a gate decision and the gates refused its candidate. -STATUS_BLOCKED: Final = "blocked" - -STAGE_CREATED: Final = "created" -STAGE_CALIBRATING: Final = "calibrating" -STAGE_COMPLETE: Final = "complete" -STAGE_FAILED: Final = "failed" -STAGE_BLOCKED: Final = "blocked" - -CALIBRATION_EVENT_KIND: Final = "calibration_epoch" -BUILD_STARTED_MESSAGE: Final = "Microcosm build started." -BUILD_COMPLETED_MESSAGE: Final = "Microcosm build completed." -BUILD_BLOCKED_MESSAGE: Final = "The gates refused the candidate at {phase}." -UNEXPECTED_PROCESS_EXIT_MESSAGE: Final = ( - "The build process exited without reporting completion." -) - -LOCAL_ACKNOWLEDGEMENT_OK: Final = b"ok\n" -LOCAL_ACKNOWLEDGEMENT_ERROR: Final = b"error\n" -LOCAL_MESSAGE_DELIMITER: Final = b"\n" -LOCAL_PING_MESSAGE: Final = b'{"action":"ping"}\n' - -MAX_TELEMETRY_TEXT_CHARS: Final = 2_000 -MAX_TELEMETRY_MESSAGE_CHARS: Final = 500 -MAX_TELEMETRY_DETAILS_DEPTH: Final = 6 -MAX_TELEMETRY_COLLECTION_ITEMS: Final = 200 -MAX_TELEMETRY_DETAILS_BYTES: Final = 8_192 -MAX_LOCAL_MESSAGE_BYTES: Final = 1_048_576 -LOCAL_SOCKET_READ_BYTES: Final = 65_536 -LOCAL_ACKNOWLEDGEMENT_READ_BYTES: Final = 16 - -SEQUENTIAL_STATUS_MAP: Final = MappingProxyType( - { - "failed": STATUS_FAILED, - "completed": STATUS_COMPLETED, - "passed": STATUS_COMPLETED, - "progress": STATUS_PROGRESS, - "started": STATUS_STARTED, - "running": STATUS_STARTED, - } -) diff --git a/packages/microcosm-build/src/microcosm/build/telemetry_sanitization.py b/packages/microcosm-build/src/microcosm/build/telemetry_sanitization.py deleted file mode 100644 index cabd12f338..0000000000 --- a/packages/microcosm-build/src/microcosm/build/telemetry_sanitization.py +++ /dev/null @@ -1,94 +0,0 @@ -"""Size bounds and credential redaction for telemetry payloads.""" - -from __future__ import annotations - -import json -import re -from collections.abc import Mapping -from pathlib import Path -from typing import Any, Final - -from microcosm.build.telemetry_protocol import ( - MAX_TELEMETRY_COLLECTION_ITEMS, - MAX_TELEMETRY_DETAILS_BYTES, - MAX_TELEMETRY_DETAILS_DEPTH, - MAX_TELEMETRY_TEXT_CHARS, -) - -_SENSITIVE_KEY_PARTS: Final = ( - "authorization", - "credential", - "password", - "secret", - "token", - "traceback", -) -_SECRET_TEXT: Final = re.compile( - r"(?i)(?:bearer\s+[^\s]+|(?:token|secret|password|credential)\s*[:=]\s*[^\s]+|hf_[A-Za-z0-9_-]{8,})" -) -_REDACTED_VALUE: Final = "[redacted]" -_MAXIMUM_DEPTH_VALUE: Final = "[maximum depth]" -_DETAILS_TRUNCATED_KEY: Final = "telemetry_details_truncated" - - -def sanitize_text(value: str, *, limit: int = MAX_TELEMETRY_TEXT_CHARS) -> str: - """Redact credential-like text and apply a character limit.""" - - return _SECRET_TEXT.sub(_REDACTED_VALUE, value)[:limit] - - -def sanitize_json(value: Any, *, depth: int = 0) -> Any: - """Return bounded JSON-compatible telemetry without model objects.""" - - if depth >= MAX_TELEMETRY_DETAILS_DEPTH: - return _MAXIMUM_DEPTH_VALUE - if isinstance(value, Mapping): - result = {} - for key, item in list(value.items())[:MAX_TELEMETRY_COLLECTION_ITEMS]: - name = str(key) - if any(part in name.lower() for part in _SENSITIVE_KEY_PARTS): - result[name] = _REDACTED_VALUE - else: - result[name] = sanitize_json(item, depth=depth + 1) - return result - if isinstance(value, (list, tuple)): - return [ - sanitize_json(item, depth=depth + 1) - for item in value[:MAX_TELEMETRY_COLLECTION_ITEMS] - ] - if isinstance(value, Path): - return str(value) - if isinstance(value, float): - return value if value == value and abs(value) != float("inf") else None - if isinstance(value, str): - return sanitize_text(value) - if isinstance(value, (int, bool)) or value is None: - return value - item = getattr(value, "item", None) - if callable(item): - return sanitize_json(item()) - return sanitize_text(str(value)) - - -def sanitize_details(value: Mapping[str, Any]) -> dict[str, Any]: - """Return a redacted details object within the wire-size limit.""" - - sanitized = sanitize_json(value) - assert isinstance(sanitized, dict) - if ( - len(json.dumps(sanitized, separators=(",", ":")).encode()) - <= MAX_TELEMETRY_DETAILS_BYTES - ): - return sanitized - compact: dict[str, Any] = {_DETAILS_TRUNCATED_KEY: True} - for key, item in sanitized.items(): - if not (isinstance(item, (str, int, float, bool)) or item is None): - continue - candidate = {**compact, key: item} - if ( - len(json.dumps(candidate, separators=(",", ":")).encode()) - > MAX_TELEMETRY_DETAILS_BYTES - ): - break - compact[key] = item - return compact diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/full_build_cli.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/full_build_cli.py index 69cdfef272..1ed2d9ec12 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/full_build_cli.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/full_build_cli.py @@ -1293,14 +1293,12 @@ def _publish_failure_evidence( temporary JSON are never published. """ from microcosm.build.artifact_files import publish_staged_bundle - from microcosm.build.telemetry_emitter_service.graph_publication import ( - GRAPH_PUBLICATION_WAIT_SECONDS, - ) from .orrery_publication import ( EVIDENCE_MANIFEST_NAME, FAILURE_BUNDLE_DIRECTORY, FAILURE_MARKER_NAME, + GRAPH_PUBLICATION_WAIT_SECONDS, finalize_graph, stage_graph_evidence, ) @@ -2914,6 +2912,7 @@ def _run_with_telemetry( args, build_id=build_id, run_kind="dry_run" if args.dry_run else "calibration", + close_service_on_finish=False, ) emitter.transition_stage( "preflight", diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/orrery_publication.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/orrery_publication.py index 81422e9dc7..3cd1be5c4b 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/orrery_publication.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/orrery_publication.py @@ -15,18 +15,14 @@ from pathlib import Path from typing import Any, TypedDict -from microcosm.build.artifact_files import file_artifact, materialize_bytes -from microcosm.build.graph_publication_types import ( +from microcosm_emitter.orrery.client import publication_inventory, publish_graph +from microcosm_emitter.orrery.contracts import ( PublicationFileRole, PublicationReceipt, ) + +from microcosm.build.artifact_files import file_artifact, materialize_bytes from microcosm.build.staging_dataset import StagedDatasetDelivery -from microcosm.build.telemetry_emitter import default_spool_path -from microcosm.build.telemetry_emitter_service.graph_publication import ( - GRAPH_PUBLICATION_WAIT_SECONDS, - GraphPublicationQueue, - publication_inventory, -) from microcosm.graph import ContentStore from microcosm.graph.canonical import canonical_json from microcosm.graph.evidence import collect_execution_evidence, load_run_evidence @@ -37,6 +33,8 @@ EVIDENCE_MANIFEST_NAME = "orrery.evidence-manifest.json" FAILURE_MARKER_NAME = "orrery.failure.json" FAILURE_BUNDLE_DIRECTORY = "orrery-failure" +# How long a build waits for a first upload result after the durable enqueue. +GRAPH_PUBLICATION_WAIT_SECONDS = 30.0 class GraphEvidenceOutput(TypedDict): @@ -155,7 +153,20 @@ def finalize_graph( preserved = Path(args.attempt_evidence) / "orrery-publication" for name in roles: materialize_bytes((output / name).read_bytes(), preserved / name) - inventory = publication_inventory(preserved, roles) + emitter = rowwise_staging.active_telemetry_emitter() + inventory = ( + publication_inventory(preserved, roles) + if emitter is None + else emitter.publication_inventory(preserved, roles) + ) + if inventory is None: + receipt.update( + status="failed", + error_code="local_service_unavailable", + preserved_directory=str(preserved), + recorded_execution=execution is not None, + ) + raise RuntimeError("Local graph inventory service is unavailable.") materialize_bytes(canonical_json(inventory), preserved / INVENTORY_NAME) materialize_bytes(canonical_json(inventory), output / INVENTORY_NAME) names = [*roles, INVENTORY_NAME] @@ -169,17 +180,10 @@ def finalize_graph( "recorded_execution": execution is not None, } if publication_enabled(args): - emitter = rowwise_staging.active_telemetry_emitter() if emitter is None: # Durable enqueue is still possible without a running # emitter; the retry command requires no telemetry run. - queue = GraphPublicationQueue(default_spool_path()) - queue.enqueue(preserved, inventory) - queued_receipt = queue.receipt(inventory["publication_id"]) - assert queued_receipt is not None, ( - "A queued graph publication must have a receipt." - ) - receipt.update(queued_receipt) + receipt.update(publish_graph(preserved, inventory, wait_seconds=0)) else: receipt.update( emitter.publish_graph( @@ -189,8 +193,9 @@ def finalize_graph( except Exception: # No credential, exception text, paths to raw input data, or signed URL # is copied from an exception into public evidence or telemetry. + if receipt["status"] != "failed": + receipt["error_code"] = "graph_export_or_enqueue_failed" receipt["status"] = "failed" - receipt["error_code"] = "graph_export_or_enqueue_failed" materialize_bytes(canonical_json(receipt), output / RECEIPT_NAME) names.append(RECEIPT_NAME) if manifest is not None: diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/rowwise_staging.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/rowwise_staging.py index e9a3be5de8..55c80a4565 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/rowwise_staging.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/rowwise_staging.py @@ -233,6 +233,7 @@ def start_telemetry_emitter( *, build_id: str, run_kind: str = "calibration", + close_service_on_finish: bool = True, ) -> LocalTelemetryEmitter: """Start or reuse the process-local emitter for one UK build attempt.""" @@ -253,6 +254,7 @@ def start_telemetry_emitter( pipeline=posture.pipeline, candidate_id=args.staging_candidate_id or build_id, run_kind=run_kind, + close_service_on_finish=close_service_on_finish, ) return _ACTIVE_EMITTER diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/worker_identity.py b/packages/microcosm-build/src/microcosm/build/us_runtime/worker_identity.py index 2d5aff5b0b..e890126dd1 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/worker_identity.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/worker_identity.py @@ -28,7 +28,7 @@ PRIMARY_QRF_WORKER_MODULE = "microcosm.build.us_runtime.puf_qrf_worker" PRIMARY_QRF_INTERPRETER_PLACEHOLDER = "{python_interpreter}" APPROVED_UV_LOCK_SHA256 = ( - "d73966a4020231e1b6d52bb5d69d852bbfc95827519dd4c98fc5c71ac8b20e19" + "75ffb31c9b711a288e7de3e8f0442047b5aafcdd146c7fd95219f4cc1ac769ae" ) LEGACY_CAMPAIGN_UV_LOCK_SHA256 = ( "27f47e385cfa35e2644a37410d1804b361ad9aee123577551c8421547bda65ee" diff --git a/packages/microcosm-build/tests/engine_free/shared/test_emitter_adapter.py b/packages/microcosm-build/tests/engine_free/shared/test_emitter_adapter.py new file mode 100644 index 0000000000..159ffb743c --- /dev/null +++ b/packages/microcosm-build/tests/engine_free/shared/test_emitter_adapter.py @@ -0,0 +1,394 @@ +"""Test Microcosm's adapter with emitter APIs replaced, not hosted services.""" + +from types import SimpleNamespace + +import pytest + +from microcosm.build import telemetry_emitter as adapter + +START = adapter.LocalTelemetryEmitter.start.__func__ + + +def test_selects_both_workers_once_with_shared_path_and_build_identity( + monkeypatch, tmp_path +): + calls, messages, queries, closed = [], [], [], [] + transport = SimpleNamespace(send=messages.append) + inventory = { + "version": 1, + "publication_id": "independent", + "files": [], + } + + def request(message, **options): + queries.append((message, options)) + if message["command"] == "inventory": + return {"inventory": inventory} + return { + "receipt": { + "version": 1, + "publication_id": "independent", + "status": "pending", + "error_code": None, + "attempts": 1, + } + } + + graph_transport = SimpleNamespace(request=request) + + def for_module(name): + return {"telemetry": transport, "orrery": graph_transport}[name] + + handle = SimpleNamespace( + client=SimpleNamespace(for_module=for_module), close=lambda: closed.append(True) + ) + + def start(modules, **options): + calls.append((modules, options)) + return handle + + monkeypatch.setattr(adapter, "start_services", start) + monkeypatch.setattr(adapter, "runtime_identity", lambda: {"commit": "synthetic"}) + path = tmp_path / "jobs.sqlite3" + emitter = START( + adapter.LocalTelemetryEmitter, + run_id="run-one", + country_code="uk", + pipeline="full", + spool_path=path, + heartbeat_seconds=123, + ) + assert len(calls) == 1 + modules, options = calls[0] + assert set(modules) == {"telemetry", "orrery"} + assert all(m["spool_path"] == str(path) for m in modules.values()) + assert "registration" not in modules["orrery"] + assert modules["telemetry"]["registration"]["run_id"] == "run-one" + assert modules["telemetry"]["heartbeat_seconds"] == 123 + assert modules["telemetry"]["identity"] == {"commit": "synthetic"} + assert options["startup_timeout"] == 3 + # The service emits one run-start event from its startup configuration. + # The build adapter must not send a second run-start request. + assert messages == [] + emitter.stage("stage-that-is-not-in-a-fixed-list") + assert messages[0]["stage_id"] == "stage-that-is-not-in-a-fixed-list" + assert emitter.publication_inventory(tmp_path, {}) == inventory + receipt = emitter.publish_graph(tmp_path, inventory, wait_seconds=0) + assert receipt["status"] == "pending" + assert [query[0]["command"] for query in queries] == ["inventory", "enqueue"] + assert not path.exists() + emitter.complete() + emitter.close() + assert closed == [True] + assert messages[-1]["command"] == "complete" + + +def test_disabled_startup_cannot_claim_persistence_or_start_a_replacement_service( + monkeypatch, tmp_path +): + monkeypatch.setattr( + adapter.GraphPublicationClient, + "start", + lambda **options: pytest.fail("started a replacement graph service"), + ) + emitter = adapter.LocalTelemetryEmitter( + run=adapter.TelemetryRun("run-one", "uk", "full"), + transport=None, + ) + value = {"version": 1, "publication_id": "independent", "files": []} + assert emitter.publication_inventory(tmp_path, {}) is None + receipt = emitter.publish_graph(tmp_path, value, wait_seconds=0) + assert receipt == { + "version": 1, + "publication_id": "independent", + "status": "failed", + "error_code": "local_service_unavailable", + } + assert list(tmp_path.iterdir()) == [] + + +def test_unavailable_graph_socket_returns_failure_without_writing_a_queue(tmp_path): + from microcosm_emitter.host.errors import LocalServiceError + + def unavailable(*args, **options): + raise LocalServiceError("socket unavailable") + + emitter = adapter.LocalTelemetryEmitter( + run=adapter.TelemetryRun("run-one", "uk", "full"), + transport=SimpleNamespace(send=lambda message: None), + graph_client=adapter.GraphPublicationClient( + SimpleNamespace(request=unavailable) + ), + ) + assert emitter.publication_inventory(tmp_path, {}) is None + receipt = emitter.publish_graph( + tmp_path, + {"version": 1, "publication_id": "independent", "files": []}, + wait_seconds=0, + ) + assert receipt["status"] == "failed" + assert receipt["error_code"] == "local_service_unavailable" + assert list(tmp_path.iterdir()) == [] + + +def test_start_failure_is_nonfatal_and_does_not_log_exception_payload( + monkeypatch, capsys, tmp_path +): + def fail(*args, **kwargs): + raise RuntimeError("private credential should not be logged") + + monkeypatch.setattr(adapter, "start_services", fail) + emitter = START( + adapter.LocalTelemetryEmitter, + run_id="run-one", + country_code="uk", + pipeline="full", + spool_path=tmp_path / "jobs.sqlite3", + ) + assert not emitter.available + warning = capsys.readouterr().err + assert "RuntimeError" in warning + assert "private credential" not in warning + + +@pytest.mark.parametrize("operation", ["complete", "close"]) +def test_default_test_emitter_uses_emitter_lifecycle_in_memory( + fake_telemetry_emitters, operation +): + emitter = adapter.LocalTelemetryEmitter.start( + run_id="run-one", country_code="uk", pipeline="full" + ) + emitter.stage("one") + getattr(emitter, operation)() + assert not emitter.available + assert emitter.events[0]["stage_id"] == "one" + + +def test_fixture_intercepts_preimported_build_entrypoint( + monkeypatch, fake_telemetry_emitters +): + from microcosm.build.uk_runtime import rowwise_staging + from test_support.microcosm_build.telemetry import FakeTelemetryEmitter + + def fail(*args, **kwargs): + pytest.fail("test started a real emitter") + + monkeypatch.setattr(adapter, "start_services", fail) + for start in ( + rowwise_staging.start_local_telemetry_emitter_service, + adapter.LocalTelemetryEmitter.start, + ): + emitter = start(run_id="fake", country_code="GB", pipeline="test") + assert isinstance(emitter, FakeTelemetryEmitter) + assert emitter in fake_telemetry_emitters + + +def test_fixture_isolates_cached_huggingface_credentials(tmp_path): + from pathlib import Path + + from huggingface_hub import constants, get_token + + assert Path(constants.HF_TOKEN_PATH).is_relative_to(tmp_path) + assert get_token() is None + + +@pytest.mark.parametrize("failure", ["default_spool_path", "runtime_identity"]) +def test_metadata_or_cache_failure_never_starts_a_service(monkeypatch, failure): + def fail(): + raise OSError("private setup detail") + + monkeypatch.setattr(adapter, failure, fail) + monkeypatch.setattr( + adapter, "start_services", lambda *a, **kw: pytest.fail("started after failure") + ) + emitter = START( + adapter.LocalTelemetryEmitter, + run_id="failed", + country_code="uk", + pipeline="full", + ) + assert not emitter.available + + +def test_graph_receipt_poll_uses_the_existing_module_client(monkeypatch, tmp_path): + from microcosm_emitter.orrery import client + + calls = [] + + def request(message, **options): + calls.append(message["command"]) + return { + "receipt": { + "version": 1, + "publication_id": "independent", + "status": ( + "pending" if message["command"] == "enqueue" else "published" + ), + "error_code": None, + "attempts": 0 if message["command"] == "enqueue" else 1, + } + } + + monkeypatch.setattr(client.time, "sleep", lambda duration: None) + emitter = adapter.LocalTelemetryEmitter( + run=adapter.TelemetryRun("run-one", "uk", "full"), + transport=SimpleNamespace(send=lambda message: None), + graph_client=adapter.GraphPublicationClient(SimpleNamespace(request=request)), + ) + receipt = emitter.publish_graph( + tmp_path, + {"version": 1, "publication_id": "independent", "files": []}, + wait_seconds=1, + ) + assert receipt["status"] == "published" + assert calls == ["enqueue", "receipt"] + + +def test_adapter_assigns_host_cleanup_to_the_build_finally(monkeypatch): + configured = [] + + def initialize(self, **options): + configured.append(options) + + monkeypatch.setattr(adapter.EmitterTelemetryClient, "__init__", initialize) + adapter.LocalTelemetryEmitter( + run=adapter.TelemetryRun("synthetic", "uk", "full"), + transport=None, + close_service_on_finish=False, + ) + assert configured[0]["close_service_on_finish"] is False + + +@pytest.mark.parametrize("outcome", ["blocked", "failed", "completed"]) +def test_graph_publication_precedes_host_cleanup_after_recording_an_outcome( + monkeypatch, tmp_path, outcome +): + from test_support.microcosm_build.telemetry import FakeTelemetryEmitter + + operations, queries = [], [] + telemetry = FakeTelemetryEmitter(adapter.TelemetryRun("synthetic", "uk", "full")) + inventory = {"version": 1, "publication_id": "synthetic", "files": []} + + def request(message, **options): + operations.append(message["command"]) + queries.append(message) + if message["command"] == "inventory": + return {"inventory": inventory} + return { + "receipt": { + "version": 1, + "publication_id": "synthetic", + "status": "pending", + "attempts": 1, + "error_code": None, + } + } + + def for_module(name): + return { + "telemetry": telemetry, + "orrery": SimpleNamespace(request=request), + }[name] + + handle = SimpleNamespace( + client=SimpleNamespace(for_module=for_module), + close=lambda: operations.append("close"), + ) + monkeypatch.setattr(adapter, "start_services", lambda *a, **k: handle) + monkeypatch.setattr(adapter, "runtime_identity", lambda: {"commit": "synthetic"}) + emitter = START( + adapter.LocalTelemetryEmitter, + run_id="synthetic", + country_code="uk", + pipeline="full", + spool_path=tmp_path / "queue.sqlite3", + close_service_on_finish=False, + ) + emitter.transition_stage("solve") + if outcome == "blocked": + emitter.block( + phase="post_calibration", + blocking_gate_ids=["uk.local.fit"], + blocking_failure_count=3, + gate_statuses={"uk.local.fit": "FAIL"}, + ) + elif outcome == "failed": + emitter.fail( + ValueError("synthetic validation refusal"), + error_code="BUILD_REFUSED", + failure_class="refused", + failed_during="solve", + ) + else: + emitter.complete() + assert operations == [] + assert emitter.available + final_events = [event for event in telemetry.events if event["event_type"] == "run"] + assert [event["status"] for event in final_events] == [outcome] + if outcome == "blocked": + assert final_events[0]["details"] == { + "phase": "post_calibration", + "blocking_gate_ids": ["uk.local.fit"], + "blocking_failure_count": 3, + "gate_statuses": {"uk.local.fit": "FAIL"}, + } + assert telemetry.events[-2]["event_type"] == "stage" + assert telemetry.events[-2]["status"] == "completed" + elif outcome == "failed": + assert final_events[0]["details"]["error_code"] == "BUILD_REFUSED" + assert final_events[0]["details"]["failure_class"] == "refused" + assert final_events[0]["details"]["failed_during"] == "solve" + # A completed telemetry lifecycle must not prevent requests on the other + # module; the existing build finally owns physical service shutdown. + try: + assert emitter.publication_inventory(tmp_path, {}) == inventory + assert emitter.publish_graph(tmp_path, inventory, wait_seconds=0)["status"] == ( + "pending" + ) + finally: + emitter.close() + emitter.close() + assert [query["command"] for query in queries] == ["inventory", "enqueue"] + assert operations == ["inventory", "enqueue", "close"] + assert not emitter.available + assert ( + len([event for event in telemetry.events if event["event_type"] == "run"]) == 1 + ) + + +@pytest.mark.parametrize("outcome", ["complete", "fail"]) +def test_default_finish_releases_host_without_an_outer_build_cleanup(outcome): + closed = [] + emitter = adapter.LocalTelemetryEmitter( + run=adapter.TelemetryRun("synthetic", "us", "release"), + transport=SimpleNamespace(send=lambda message: None), + handle=SimpleNamespace(close=lambda: closed.append(True)), + ) + if outcome == "complete": + emitter.complete() + else: + emitter.fail(ValueError("synthetic failure"), error_code="BUILD_FAILED") + assert closed == [True] + assert not emitter.available + emitter.close() + assert closed == [True] + + +@pytest.mark.parametrize("outcome", ["block", "fail"]) +def test_outcome_on_disabled_transport_is_safe_and_cannot_restart_service( + monkeypatch, outcome +): + monkeypatch.setattr( + adapter, + "start_services", + lambda *a, **k: pytest.fail("outcome started a replacement service"), + ) + emitter = adapter.LocalTelemetryEmitter( + run=adapter.TelemetryRun("synthetic", "uk", "full"), transport=None + ) + if outcome == "block": + emitter.block(phase="preflight", blocking_gate_ids=[], blocking_failure_count=1) + else: + emitter.fail(ValueError("refused"), error_code="BUILD_REFUSED") + assert not emitter.available + emitter.close() diff --git a/packages/microcosm-build/tests/engine_free/shared/test_graph_publication.py b/packages/microcosm-build/tests/engine_free/shared/test_graph_publication.py deleted file mode 100644 index f3997d9451..0000000000 --- a/packages/microcosm-build/tests/engine_free/shared/test_graph_publication.py +++ /dev/null @@ -1,358 +0,0 @@ -"""Durable graph jobs use fake HTTP, independently of the visualization app.""" - -import json -from dataclasses import FrozenInstanceError - -import httpx -import pytest - -from microcosm.build.graph_publication_types import ClaimedGraphPublication -from microcosm.build.telemetry_emitter_service.graph_publication import ( - GraphPublicationDelivery, - GraphPublicationQueue, - inventory_digest, - publication_inventory, -) - - -def source(tmp_path): - directory = tmp_path / "source" - directory.mkdir() - (directory / "graph.orrery.json").write_text('{"fixture":true}') - return directory, publication_inventory(directory, {"graph.orrery.json": "graph"}) - - -def test_jobs_survive_restart_without_run_registration_or_event_retention(tmp_path): - directory, inventory = source(tmp_path) - queue = GraphPublicationQueue(tmp_path / "jobs.sqlite3") - queue.enqueue(directory, inventory) - restarted = GraphPublicationQueue(queue.path) - job = restarted.claim() - assert isinstance(job, ClaimedGraphPublication) - assert job.inventory == inventory - assert "run_id" not in inventory - assert job.publication_id == inventory["publication_id"] - assert job.directory == directory.resolve() - assert restarted.claim() is None - - -def test_claim_is_a_frozen_record_and_expired_leases_can_be_reclaimed( - tmp_path, monkeypatch -): - from microcosm.build.telemetry_emitter_service.graph_publication import time - - directory, inventory = source(tmp_path) - queue = GraphPublicationQueue(tmp_path / "jobs.sqlite3") - queue.enqueue(directory, inventory) - clock = [1000.0] - monkeypatch.setattr(time, "time", lambda: clock[0]) - claimed = queue.claim(lease_seconds=5) - assert isinstance(claimed, ClaimedGraphPublication) - assert claimed.lease_until == 1005.0 - with pytest.raises(FrozenInstanceError): - claimed.lease_until = 0 - assert GraphPublicationQueue(queue.path).claim() is None - clock[0] = 1005.0 - replacement = GraphPublicationQueue(queue.path).claim() - assert replacement.publication_id == claimed.publication_id - assert replacement.lease_until > claimed.lease_until - - -def test_claim_respects_retry_delay_and_published_jobs_are_not_reclaimed( - tmp_path, monkeypatch -): - from microcosm.build.telemetry_emitter_service.graph_publication import time - - directory, inventory = source(tmp_path) - queue = GraphPublicationQueue(tmp_path / "jobs.sqlite3") - queue.enqueue(directory, inventory) - clock = [1000.0] - monkeypatch.setattr(time, "time", lambda: clock[0]) - claimed = queue.claim() - queue.finish( - claimed.publication_id, - error_code="publication_network_error", - expected_lease=claimed.lease_until, - ) - assert queue.claim() is None - clock[0] = 1002.0 - retried = queue.claim() - assert retried.publication_id == claimed.publication_id - queue.finish( - retried.publication_id, - url="https://runs.example/runs/published/", - expected_lease=retried.lease_until, - ) - assert queue.claim() is None - - -def test_queue_rejects_changed_bytes_and_conflicting_inventory(tmp_path): - directory, inventory = source(tmp_path) - queue = GraphPublicationQueue(tmp_path / "jobs.sqlite3") - (directory / "graph.orrery.json").write_text("changed") - with pytest.raises(ValueError, match="digest"): - queue.enqueue(directory, inventory) - with pytest.raises(ValueError): - publication_inventory(directory, {"private.h5": "graph"}) - - -def test_delivery_retries_missing_credentials_and_network_failure(tmp_path): - directory, inventory = source(tmp_path) - queue = GraphPublicationQueue(tmp_path / "jobs.sqlite3") - queue.enqueue(directory, inventory) - delivery = GraphPublicationDelivery( - queue, credential=lambda: ("missing", None), origin="https://runs.example" - ) - delivery.flush_once() - assert queue.receipt(inventory["publication_id"])["status"] == "pending" - queue.retry(inventory["publication_id"]) - client = httpx.Client( - transport=httpx.MockTransport( - lambda request: (_ for _ in ()).throw( - httpx.ConnectError("secret", request=request) - ) - ) - ) - GraphPublicationDelivery( - queue, - credential=lambda: ("ok", "session"), - origin="https://runs.example", - client=client, - ).flush_once() - receipt = queue.receipt(inventory["publication_id"]) - assert receipt["status"] == "pending" - assert "secret" not in json.dumps(receipt) - - -@pytest.mark.parametrize( - "upload_url", - [ - "https://store.blob.vercel-storage.com/upload?signed=credential", - "https://vercel.com/api/blob/?pathname=object&signed=credential", - ], -) -def test_delivery_uses_exact_bytes_and_does_not_forward_session_to_signed_upload( - tmp_path, - upload_url, -): - directory, inventory = source(tmp_path) - queue = GraphPublicationQueue(tmp_path / "jobs.sqlite3") - queue.enqueue(directory, inventory) - requests = [] - - def respond(request): - requests.append(request) - if request.url.path == "/api/uploads/": - return httpx.Response( - 200, - json={ - "uploads": [ - { - "name": "graph.orrery.json", - "url": upload_url, - "already_uploaded": False, - } - ] - }, - ) - if request.method == "PUT": - assert "authorization" not in request.headers - assert request.content == (directory / "graph.orrery.json").read_bytes() - return httpx.Response(200, json={}) - return httpx.Response( - 200, - json={ - "publication_id": inventory["publication_id"], - "inventory_sha256": inventory_digest(inventory), - }, - ) - - delivery = GraphPublicationDelivery( - queue, - credential=lambda: ("ok", "session"), - origin="https://runs.example", - client=httpx.Client(transport=httpx.MockTransport(respond)), - ) - delivery.flush_once() - receipt = queue.receipt(inventory["publication_id"]) - assert receipt["status"] == "published" - assert len(requests) == 3 - assert "signed=" not in json.dumps(receipt) - assert "session" not in json.dumps(receipt) - - -def test_queue_resumes_after_worker_exits_and_keeps_pending_jobs(tmp_path): - directory, inventory = source(tmp_path) - queue = GraphPublicationQueue(tmp_path / "jobs.sqlite3") - queue.enqueue(directory, inventory) - queue.claim(lease_seconds=0) - assert GraphPublicationQueue(queue.path).claim() is not None - - -def test_late_failed_attempt_cannot_replace_a_published_receipt(tmp_path): - directory, inventory = source(tmp_path) - queue = GraphPublicationQueue(tmp_path / "jobs.sqlite3") - queue.enqueue(directory, inventory) - queue.claim(lease_seconds=0) - replacement = GraphPublicationQueue(queue.path) - replacement.claim() - id_ = inventory["publication_id"] - replacement.finish(id_, url=f"https://runs.example/runs/{id_}/") - published = queue.receipt(id_) - queue.finish(id_, error_code="publication_network_error") - assert queue.receipt(id_) == published - assert json.loads((directory / "publication.status.json").read_bytes()) == published - - -def test_expired_worker_cannot_release_a_replacement_workers_lease(tmp_path): - directory, inventory = source(tmp_path) - queue = GraphPublicationQueue(tmp_path / "jobs.sqlite3") - queue.enqueue(directory, inventory) - expired = queue.claim(lease_seconds=0) - replacement = GraphPublicationQueue(queue.path) - current = replacement.claim() - id_ = inventory["publication_id"] - initial = queue.receipt(id_) - queue.finish( - id_, - error_code="publication_network_error", - expected_lease=expired.lease_until, - ) - assert queue.receipt(id_) == initial - assert queue.claim() is None - replacement.finish( - id_, - url=f"https://runs.example/runs/{id_}/", - expected_lease=current.lease_until, - ) - assert queue.receipt(id_)["status"] == "published" - - -def test_telemetry_pruning_cannot_remove_graph_jobs(tmp_path): - from microcosm.build.telemetry_emitter_service.spool import EventSpool - - directory, inventory = source(tmp_path) - queue = GraphPublicationQueue(tmp_path / "jobs.sqlite3") - queue.enqueue(directory, inventory) - EventSpool(queue.path).prune() - assert ( - GraphPublicationQueue(queue.path).receipt(inventory["publication_id"])["status"] - == "pending" - ) - - -def test_rejected_graph_session_is_refreshed_without_discarding_the_job(tmp_path): - directory, inventory = source(tmp_path) - queue = GraphPublicationQueue(tmp_path / "jobs.sqlite3") - queue.enqueue(directory, inventory) - invalidations = [] - delivery = GraphPublicationDelivery( - queue, - credential=lambda: ("ok", "expired"), - invalidate_credential=lambda: invalidations.append(True), - origin="https://runs.example", - client=httpx.Client( - transport=httpx.MockTransport(lambda request: httpx.Response(401)) - ), - ) - assert not delivery.flush_once() - assert invalidations == [True] - assert queue.receipt(inventory["publication_id"])["status"] == "pending" - - -def test_retry_cli_reuses_identity_and_leaves_hf_snapshot_unchanged( - tmp_path, monkeypatch -): - from microcosm.build import graph_publication_cli as cli - - directory, inventory = source(tmp_path) - (directory / "orrery.upload.json").write_text(json.dumps(inventory)) - snapshot = directory / "orrery.publication.json" - snapshot.write_text('{"status":"pending"}') - calls = [] - - class Delivery: - def __init__(self, queue, **kwargs): - self.queue = queue - - def flush_once(self, *, publication_id): - calls.append(publication_id) - self.queue.finish( - publication_id, url="https://runs.example/runs/published/" - ) - - monkeypatch.setattr(cli, "GraphPublicationDelivery", Delivery) - assert ( - cli.main( - [ - "--spool", - str(tmp_path / "jobs.sqlite3"), - "--publication-id", - inventory["publication_id"], - "--directory", - str(directory), - ] - ) - == 0 - ) - assert calls == [inventory["publication_id"]] - assert snapshot.read_text() == '{"status":"pending"}' - - -def test_deleted_preserved_directory_is_retained_without_stopping_delivery( - tmp_path, monkeypatch -): - import shutil - - from microcosm.build.telemetry_emitter_service.graph_publication import time - - clock = [1000.0] - monkeypatch.setattr(time, "time", lambda: clock[0]) - queue = GraphPublicationQueue(tmp_path / "jobs.sqlite3") - (tmp_path / "removed").mkdir() - (tmp_path / "kept").mkdir() - removed, removed_inventory = source(tmp_path / "removed") - queue.enqueue(removed, removed_inventory) - clock[0] = 1001.0 - kept, kept_inventory = source(tmp_path / "kept") - queue.enqueue(kept, kept_inventory) - shutil.rmtree(removed) - delivery = GraphPublicationDelivery( - queue, credential=lambda: ("missing", None), origin="https://runs.example" - ) - - assert delivery.flush_once() is False - receipt = queue.receipt(removed_inventory["publication_id"]) - assert receipt["status"] == "pending" - assert receipt["error_code"] == "preserved_files_missing" - assert receipt["attempts"] == 1 - # The failed job backs off, so the next claim reaches the other job. - assert queue.claim().publication_id == kept_inventory["publication_id"] - - -def test_status_file_failure_does_not_roll_back_the_queue_result(tmp_path): - directory, inventory = source(tmp_path) - queue = GraphPublicationQueue(tmp_path / "jobs.sqlite3") - queue.enqueue(directory, inventory) - claimed = queue.claim() - (directory / "publication.status.json").mkdir() - queue.finish( - claimed.publication_id, - url="https://runs.example/runs/published/", - expected_lease=claimed.lease_until, - ) - receipt = queue.receipt(claimed.publication_id) - assert receipt["status"] == "published" - assert list(directory.glob("*.tmp")) == [] - - -@pytest.mark.parametrize( - "origin", - ["http://runs.example", "https://runs.example/path", "https://user@runs.example"], -) -def test_delivery_rejects_non_origin_destinations(tmp_path, origin): - queue = GraphPublicationQueue(tmp_path / "jobs.sqlite3") - with pytest.raises(ValueError, match="service URL"): - GraphPublicationDelivery( - queue, credential=lambda: ("missing", None), origin=origin - ) diff --git a/packages/microcosm-build/tests/engine_free/shared/test_publication_workers.py b/packages/microcosm-build/tests/engine_free/shared/test_publication_workers.py deleted file mode 100644 index 71bfe1566b..0000000000 --- a/packages/microcosm-build/tests/engine_free/shared/test_publication_workers.py +++ /dev/null @@ -1,142 +0,0 @@ -"""Independent publication delivery in the existing local service.""" - -import socket -import tempfile -import threading -import time -from pathlib import Path - -from microcosm.build.telemetry_emitter_service.auth import CollectorSession -from microcosm.build.telemetry_emitter_service.main import build_parser -from microcosm.build.telemetry_emitter_service.runtime import EmitterService -from microcosm.build.telemetry_emitter_service.spool import EventSpool - - -def test_slow_graph_upload_does_not_block_telemetry_or_socket( - tmp_path, real_local_telemetry -): - entered, release, telemetry_worked = (threading.Event() for _ in range(3)) - - class GraphDelivery: - def flush_once(self): - entered.set() - release.wait(5) - - class TelemetryDelivery: - def flush_once(self): - telemetry_worked.set() - return False - - class Sampler: - def parent_alive(self): - return True - - def sample(self): - return {} - - path = Path(tempfile.mkdtemp(prefix="mc-pub-test-", dir="/tmp")) / "e.sock" - service = EmitterService( - socket_path=path, - registration={ - "run_id": "run", - "producer_id": "producer", - "country_code": "GB", - "pipeline": "test", - }, - spool=EventSpool(tmp_path / "events.sqlite3"), - delivery=TelemetryDelivery(), - graph_delivery=GraphDelivery(), - sampler=Sampler(), - heartbeat_seconds=60, - drain_seconds=0, - ) - thread = threading.Thread(target=service.run) - thread.start() - try: - assert entered.wait(3) - assert telemetry_worked.wait(3) - with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as peer: - peer.settimeout(0.5) - peer.connect(str(path)) - peer.sendall(b'{"action":"ping"}\n') - assert peer.recv(16) == b"ok\n" - finally: - service._stop.set() - release.set() - thread.join(3) - assert not thread.is_alive() - - -def test_graph_only_service_requires_no_run_registration(tmp_path): - args = build_parser().parse_args( - [ - "--socket", - str(tmp_path / "e.sock"), - "--spool", - str(tmp_path / "jobs.sqlite3"), - "--parent-pid", - "1", - ] - ) - assert args.registration_json is None - - -def test_both_workers_share_one_collector_exchange(): - calls = [] - - def exchange(*args): - calls.append(args) - time.sleep(0.01) - return 200, {"access_token": "synthetic-session", "expires_in": 3600} - - session = CollectorSession( - "https://collector.example", post=exchange, credential=lambda: "hf-synthetic" - ) - results = [] - threads = [ - threading.Thread(target=lambda: results.append(session.credential())) - for _ in range(2) - ] - for thread in threads: - thread.start() - for thread in threads: - thread.join(2) - assert results == [("ok", "synthetic-session")] * 2 - assert len(calls) == 1 - - -def test_graph_worker_survives_a_local_queue_error(tmp_path, capsys): - from sqlalchemy.exc import OperationalError - - attempts = [] - recovered = threading.Event() - - class GraphDelivery: - def flush_once(self): - attempts.append(None) - if len(attempts) == 1: - raise OperationalError("UPDATE", {}, Exception("database is locked")) - recovered.set() - return False - - service = EmitterService( - socket_path=tmp_path / "unused.sock", - registration=None, - spool=EventSpool(tmp_path / "events.sqlite3"), - delivery=None, - graph_delivery=GraphDelivery(), - sampler=None, - heartbeat_seconds=60, - drain_seconds=0, - ) - worker = threading.Thread( - target=service._graph_worker, args=(service.graph_delivery,) - ) - worker.start() - try: - assert recovered.wait(10) - finally: - service._stop.set() - worker.join(5) - assert not worker.is_alive() - assert "graph publication hit a local queue error" in capsys.readouterr().err diff --git a/packages/microcosm-build/tests/engine_free/shared/test_telemetry_emitter.py b/packages/microcosm-build/tests/engine_free/shared/test_telemetry_emitter.py deleted file mode 100644 index a2cb501161..0000000000 --- a/packages/microcosm-build/tests/engine_free/shared/test_telemetry_emitter.py +++ /dev/null @@ -1,970 +0,0 @@ -import json -import os -import socket -import tempfile -import threading -import time -from datetime import UTC, datetime, timedelta -from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer -from pathlib import Path -from types import SimpleNamespace - -import pytest -from alembic import command -from sqlalchemy import Column, Integer, MetaData, String, Table, create_engine -from sqlalchemy.orm import Session - -from microcosm.build.telemetry_emitter import ( - LocalTelemetryEmitter, - TelemetryRun, -) -from microcosm.build.telemetry_emitter_service import ( - CollectorDelivery, - EmitterService, - EventSpool, -) -from microcosm.build.telemetry_emitter_service import collector as collector_module -from microcosm.build.telemetry_emitter_service import resources as resources_module -from microcosm.build.telemetry_emitter_service import runtime as runtime_module -from microcosm.build.telemetry_emitter_service import spool as spool_module -from microcosm.build.telemetry_emitter_service.constants import ( - DRAIN_RETRY_SECONDS, - PRODUCTION_COLLECTOR_URL, -) -from microcosm.build.telemetry_emitter_service.database import ( - create_spool_engine, - sqlite_database_url, -) -from microcosm.build.telemetry_emitter_service.migrations import ( - alembic_config, - current_database_revision, - migration_head_revision, -) -from microcosm.build.telemetry_emitter_service.models import ( - SpoolModel, - TelemetryEventRecord, - TelemetryRunRecord, - serialized_json_length, -) -from microcosm.build.telemetry_protocol import ( - BUILD_COMPLETED_MESSAGE, - BUILD_STARTED_MESSAGE, - MAX_TELEMETRY_DETAILS_BYTES, -) -from microcosm.build.telemetry_sanitization import ( - sanitize_details, - sanitize_json, - sanitize_text, -) - - -def _recent_timestamp() -> str: - """A stored-row timestamp inside the spool's retention window. - - The spool prunes rows older than ``RETENTION_DAYS`` when it opens, so a - fixed calendar date in a migrated-spool fixture expires a week later. - """ - - return (datetime.now(UTC) - timedelta(minutes=5)).isoformat() - - -def _registration(run_id: str = "run-a") -> dict[str, object]: - return TelemetryRun( - run_id=run_id, - country_code="US", - pipeline="us_fiscal_refresh", - candidate_id="candidate-a", - producer_id="producer-a", - ).as_registration() - - -def _event(stage_id: str = "compile_targets") -> dict[str, object]: - return { - "timestamp": "2026-10-02T10:00:00+00:00", - "event_type": "stage", - "stage_id": stage_id, - "status": "started", - "message": "Compiling targets.", - "details": {"batches": 12}, - } - - -def test_development_collector_must_be_on_loopback() -> None: - assert collector_module._development_collector_url("http://127.0.0.1:8080") == ( - "http://127.0.0.1:8080" - ) - for value in ("https://collector.example", "http://192.0.2.1:8080"): - try: - collector_module._development_collector_url(value) - except ValueError as error: - assert "loopback" in str(error) - else: - raise AssertionError("non-loopback development collector was accepted") - - -def test_production_collector_cannot_be_replaced_by_environment( - tmp_path, monkeypatch -) -> None: - monkeypatch.setenv( - "MICROCOSM_TELEMETRY_COLLECTOR_URL", - "https://untrusted.example", - ) - - delivery = CollectorDelivery(EventSpool(tmp_path / "events.sqlite3")) - - assert delivery.collector_url == PRODUCTION_COLLECTOR_URL - - -def test_token_bearing_http_post_does_not_follow_redirects() -> None: - paths: list[str] = [] - - class Handler(BaseHTTPRequestHandler): - def do_POST(self): - paths.append(self.path) - self.send_response(307) - self.send_header("Location", "/credential-leak") - self.send_header("Content-Length", "0") - self.end_headers() - - def log_message(self, format, *args): - return - - server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) - server_thread = threading.Thread(target=server.serve_forever) - server_thread.start() - try: - status, _ = collector_module._http_post( - f"http://127.0.0.1:{server.server_port}/exchange", - {"run_id": "run-a"}, - "hf-private-token", - ) - finally: - server.shutdown() - server_thread.join(timeout=2) - server.server_close() - - assert status == 307 - assert paths == ["/exchange"] - - -def test_outbound_payload_redacts_credentials_and_tracebacks() -> None: - assert sanitize_text("Bearer hf_abcdefghijk") == "[redacted]" - assert sanitize_json( - { - "HF_TOKEN": "hf_abcdefghijk", - "traceback": "private stack", - "message": "credential=top-secret", - } - ) == { - "HF_TOKEN": "[redacted]", - "traceback": "[redacted]", - "message": "[redacted]", - } - bounded = sanitize_details( - {"done": 2, **{f"large_{index}": "x" * 2_000 for index in range(10)}} - ) - assert bounded["done"] == 2 - assert bounded["telemetry_details_truncated"] is True - assert len(json.dumps(bounded).encode()) <= MAX_TELEMETRY_DETAILS_BYTES - - -def test_event_spool_assigns_stable_sequences_and_acknowledges(tmp_path) -> None: - spool_path = tmp_path / "events.sqlite3" - spool = EventSpool(spool_path) - registration = _registration() - spool.register(registration) - - first = spool.append(registration, _event()) - second = spool.append(registration, _event("calibrate")) - - assert first["sequence"] == 1 - assert second["sequence"] == 2 - assert first["producer_id"] == "producer-a" - assert [event["sequence"] for event in spool.batch("run-a", "producer-a")] == [ - 1, - 2, - ] - - spool.acknowledge([first["event_id"]]) - assert [event["sequence"] for event in spool.batch("run-a", "producer-a")] == [2] - assert current_database_revision(spool_path) == migration_head_revision() - - -def test_alembic_schema_matches_sqlalchemy_models(tmp_path) -> None: - spool_path = tmp_path / "events.sqlite3" - EventSpool(spool_path) - engine = create_spool_engine(spool_path) - try: - with engine.begin() as connection: - with alembic_config(connection=connection) as config: - command.check(config) - finally: - engine.dispose() - - -def test_sequential_stage_updates_close_the_previous_stage() -> None: - emitter = LocalTelemetryEmitter( - run=TelemetryRun( - run_id="run-a", - country_code="US", - pipeline="us_fiscal_refresh", - ), - process=None, - socket_path=Path("/unused"), - runtime_dir=None, - ) - messages: list[dict[str, object]] = [] - emitter._send = messages.append - - emitter.transition_stage("load", message="Loading.") - emitter.transition_stage("compile", message="Compiling.") - emitter.transition_stage("compile", status="completed", batches=4) - - events = [message["event"] for message in messages] - assert [(event["stage_id"], event["status"]) for event in events] == [ - ("load", "started"), - ("load", "completed"), - ("compile", "started"), - ("compile", "completed"), - ] - - -def test_event_spool_keeps_repeated_run_producers_separate(tmp_path) -> None: - spool = EventSpool(tmp_path / "events.sqlite3") - first = _registration() - second = {**first, "producer_id": "producer-b"} - spool.register(first) - spool.register(second) - - spool.append(first, _event()) - spool.append(second, _event()) - - assert spool.batch("run-a", "producer-a")[0]["sequence"] == 1 - assert spool.batch("run-a", "producer-b")[0]["sequence"] == 1 - assert len(spool.pending_runs()) == 2 - - -def _recent_timestamp() -> str: - """A stored-row timestamp inside the spool's retention window on any date.""" - - return (datetime.now(UTC) - timedelta(hours=1)).isoformat() - - -def test_pre_eligibility_spool_is_not_uploaded_after_upgrade(tmp_path) -> None: - path = tmp_path / "events.sqlite3" - registration = _registration() - metadata = MetaData() - runs = Table( - "telemetry_runs", - metadata, - Column("run_id", String, primary_key=True), - Column("producer_id", String, primary_key=True), - Column("registration_json", String, nullable=False), - Column("next_sequence", Integer, nullable=False, default=1), - Column("updated_at", String, nullable=False), - ) - events = Table( - "telemetry_events", - metadata, - Column("event_id", String, primary_key=True), - Column("run_id", String, nullable=False), - Column("producer_id", String, nullable=False), - Column("sequence", Integer, nullable=False), - Column("payload_json", String, nullable=False), - Column("created_at", String, nullable=False), - ) - engine = create_engine(sqlite_database_url(path)) - metadata.create_all(engine) - with engine.begin() as connection: - connection.execute( - runs.insert().values( - run_id="run-a", - producer_id="producer-a", - registration_json=json.dumps(registration), - next_sequence=2, - updated_at=_recent_timestamp(), - ) - ) - connection.execute( - events.insert().values( - event_id="old-event", - run_id="run-a", - producer_id="producer-a", - sequence=1, - payload_json=json.dumps({"event_id": "old-event"}), - created_at=_recent_timestamp(), - ) - ) - engine.dispose() - - spool = EventSpool(path) - - assert spool.has_pending() - assert spool.pending_runs() == [] - assert current_database_revision(path) == migration_head_revision() - - -def test_current_pre_alembic_spool_is_adopted_without_losing_events(tmp_path) -> None: - path = tmp_path / "events.sqlite3" - registration = _registration() - engine = create_engine(sqlite_database_url(path)) - SpoolModel.metadata.create_all( - engine, - tables=[TelemetryRunRecord.__table__, TelemetryEventRecord.__table__], - ) - with Session(engine) as session, session.begin(): - session.add( - TelemetryRunRecord( - run_id="run-a", - producer_id="producer-a", - registration=registration, - next_sequence=2, - upload_state="pending", - local_only_reason=None, - updated_at=_recent_timestamp(), - ) - ) - session.add( - TelemetryEventRecord( - event_id="existing-event", - run_id="run-a", - producer_id="producer-a", - sequence=1, - payload={"event_id": "existing-event", "sequence": 1}, - created_at=_recent_timestamp(), - ) - ) - engine.dispose() - - spool = EventSpool(path) - - assert spool.pending_runs() == [registration] - assert spool.batch("run-a", "producer-a") == [ - {"event_id": "existing-event", "sequence": 1} - ] - assert current_database_revision(path) == migration_head_revision() - - -def test_event_spool_prunes_oldest_events_to_size_limit( - tmp_path, - monkeypatch, -) -> None: - spool = EventSpool(tmp_path / "events.sqlite3") - registration = _registration() - spool.register(registration) - first = spool.append(registration, _event("first")) - second = spool.append(registration, _event("second")) - monkeypatch.setattr( - spool_module, - "MAX_QUEUED_BYTES", - serialized_json_length(second), - ) - - spool.prune() - - assert first["event_id"] != second["event_id"] - assert spool.batch("run-a", "producer-a") == [second] - - -def test_collector_delivery_exchanges_hf_token_then_flushes( - tmp_path, monkeypatch -) -> None: - spool = EventSpool(tmp_path / "events.sqlite3") - registration = _registration() - spool.register(registration) - queued = spool.append(registration, _event()) - requests: list[tuple[str, dict[str, object], str]] = [] - - monkeypatch.setattr(collector_module, "_huggingface_token", lambda: "hf-secret") - - def fake_post(url, payload, token, *, timeout=5.0): - requests.append((url, payload, token)) - if url.endswith("/v1/auth/huggingface/exchange"): - return 200, {"access_token": "collector-token", "expires_in": 3600} - if url.endswith("/v1/runs"): - return 201, {"registered": True} - return 202, {"accepted": 1, "duplicates": 0} - - monkeypatch.setattr(collector_module, "_http_post", fake_post) - - delivery = CollectorDelivery( - spool, - development_collector_url="http://127.0.0.1:8080", - ) - assert delivery.flush_once() - assert requests[0][2] == "hf-secret" - assert requests[0][1] == {} - assert requests[1] == ( - "http://127.0.0.1:8080/v1/runs", - registration, - "collector-token", - ) - assert requests[2][2] == "collector-token" - assert requests[2][1] == {"events": [queued]} - assert not spool.has_pending() - - -def test_non_org_credential_keeps_event_local(tmp_path, monkeypatch, capsys) -> None: - spool = EventSpool(tmp_path / "events.sqlite3") - registration = _registration() - spool.register(registration) - spool.append(registration, _event()) - requests: list[dict[str, object]] = [] - monkeypatch.setattr(collector_module, "_huggingface_token", lambda: "hf-outsider") - - def reject(url, payload, token, *, timeout=5.0): - requests.append(payload) - return 403, {"detail": "not a member"} - - monkeypatch.setattr(collector_module, "_http_post", reject) - delivery = CollectorDelivery( - spool, - development_collector_url="http://127.0.0.1:8080", - ) - - assert not delivery.flush_once() - assert not delivery.flush_once() - assert spool.has_pending() - assert spool.pending_runs() == [] - assert requests == [{}] - assert capsys.readouterr().err.count("local-only for this run") == 1 - - -@pytest.mark.parametrize( - ("status", "retried"), [(422, False), (404, False), (429, True), (408, True)] -) -def test_a_settled_collector_rejection_goes_local_only_instead_of_retrying( - tmp_path, monkeypatch, capsys, status, retried -) -> None: - """A 4xx the collector will repeat (an event shape it does not accept, say) - must not wedge the queue: the run goes local-only with a reason and one - warning. A timeout or a rate limit still retries.""" - spool = EventSpool(tmp_path / "events.sqlite3") - registration = _registration() - spool.register(registration) - spool.append(registration, _event()) - monkeypatch.setattr(collector_module, "_huggingface_token", lambda: "hf-member") - - def fake_post(url, payload, token, *, timeout=5.0): - if url.endswith("/v1/auth/huggingface/exchange"): - return 200, {"access_token": "collector-token", "expires_in": 3600} - if url.endswith("/v1/runs"): - return 201, {"registered": True} - return status, {"detail": "rejected"} - - monkeypatch.setattr(collector_module, "_http_post", fake_post) - delivery = CollectorDelivery( - spool, development_collector_url="http://127.0.0.1:8080" - ) - assert not delivery.flush_once() - assert spool.has_pending() - if retried: - assert spool.pending_runs() == [registration] - assert "local-only" not in capsys.readouterr().err - return - assert spool.pending_runs() == [] - assert not delivery.flush_once() - err = capsys.readouterr().err - assert err.count("local-only for this run") == 1 - assert f"HTTP {status}" in err - - -def test_identity_provider_outage_keeps_events_eligible_for_retry( - tmp_path, monkeypatch -) -> None: - spool = EventSpool(tmp_path / "events.sqlite3") - registration = _registration() - spool.register(registration) - spool.append(registration, _event()) - monkeypatch.setattr(collector_module, "_huggingface_token", lambda: "hf-member") - monkeypatch.setattr( - collector_module, - "_http_post", - lambda *args, **kwargs: (503, {"detail": "temporarily unavailable"}), - ) - - assert not CollectorDelivery(spool).flush_once() - assert spool.pending_runs() == [registration] - - -def test_missing_credential_never_contacts_collector_and_stays_local_only( - tmp_path, monkeypatch -) -> None: - spool_path = tmp_path / "events.sqlite3" - spool = EventSpool(spool_path) - registration = _registration() - spool.register(registration) - spool.append(registration, _event()) - monkeypatch.setattr(collector_module, "_huggingface_token", lambda: None) - monkeypatch.setattr( - collector_module, - "_http_post", - lambda *args, **kwargs: (_ for _ in ()).throw( - AssertionError("network request") - ), - ) - - delivery = CollectorDelivery(spool) - assert not delivery.flush_once() - assert spool.pending_runs() == [] - - reopened = EventSpool(spool_path) - monkeypatch.setattr(collector_module, "_huggingface_token", lambda: "hf-later") - assert reopened.pending_runs() == [] - - -def test_resource_sampler_has_a_base_install_fallback(monkeypatch) -> None: - monkeypatch.setattr(resources_module, "psutil", None) - sampler = resources_module.ProcessTreeSampler(os.getpid()) - - assert sampler.parent_alive() - sample = sampler.sample() - assert sample.keys() == { - "cpu_user_seconds", - "cpu_system_seconds", - "rss_bytes", - "peak_rss_bytes", - } - assert all(value >= 0 for value in sample.values()) - - -def test_resource_sampler_keeps_reaped_child_cpu_monotonic(monkeypatch) -> None: - state = {"child_alive": True} - - class FakeProcess: - def __init__(self, pid): - self.pid = pid - - def create_time(self): - return float(self.pid) - - def children(self, recursive=False): - assert recursive is True - if self.pid == 10 and state["child_alive"]: - return [FakeProcess(11)] - return [] - - def cpu_times(self): - if self.pid == 10: - return SimpleNamespace( - user=10.0, - system=2.0, - children_user=100.0 if not state["child_alive"] else 0.0, - children_system=20.0 if not state["child_alive"] else 0.0, - ) - return SimpleNamespace( - user=80.0, - system=15.0, - children_user=0.0, - children_system=0.0, - ) - - def memory_info(self): - return SimpleNamespace(rss=100) - - monkeypatch.setattr( - resources_module, - "psutil", - SimpleNamespace(Process=FakeProcess, Error=OSError, STATUS_ZOMBIE="zombie"), - ) - sampler = resources_module.ProcessTreeSampler(10) - - before = sampler.sample() - state["child_alive"] = False - after = sampler.sample() - - assert before["cpu_user_seconds"] == 90.0 - assert before["cpu_system_seconds"] == 17.0 - assert after["cpu_user_seconds"] == 110.0 - assert after["cpu_system_seconds"] == 22.0 - - -def test_emitter_startup_timeout_terminates_and_reaps_service( - tmp_path, monkeypatch, real_local_telemetry -) -> None: - class FakeProcess: - def __init__(self): - self.terminated = False - self.waited = False - - def poll(self): - return None - - def terminate(self): - self.terminated = True - - def wait(self, timeout=None): - self.waited = True - return 0 - - process = FakeProcess() - monkeypatch.setattr( - "microcosm.build.telemetry_emitter.subprocess.Popen", - lambda *args, **kwargs: process, - ) - - emitter = LocalTelemetryEmitter.start( - run_id="startup-timeout", - country_code="US", - pipeline="test-pipeline", - development_collector_url="http://127.0.0.1:8080", - spool_path=tmp_path / "events.sqlite3", - startup_timeout_seconds=0, - ) - - assert not emitter.available - assert process.terminated - assert process.waited - - -class _FakeSampler: - def sample(self): - return { - "cpu_user_seconds": 1.0, - "cpu_system_seconds": 0.5, - "rss_bytes": 100, - "peak_rss_bytes": 120, - } - - def parent_alive(self): - return True - - -class _FakeDelivery: - def flush_once(self): - return False - - -def test_local_socket_acknowledges_after_durable_queue( - tmp_path, real_local_telemetry -) -> None: - socket_path = ( - Path(tempfile.mkdtemp(prefix="microcosm-test-", dir="/tmp")) / "e.sock" - ) - spool = EventSpool(tmp_path / "events.sqlite3") - registration = _registration() - emitter = EmitterService( - socket_path=socket_path, - registration=registration, - spool=spool, - delivery=_FakeDelivery(), - sampler=_FakeSampler(), - heartbeat_seconds=60, - drain_seconds=0, - ) - thread = threading.Thread(target=emitter.run) - thread.start() - deadline = time.monotonic() + 2 - while not socket_path.exists() and time.monotonic() < deadline: - time.sleep(0.01) - - with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as client: - client.connect(str(socket_path)) - client.sendall( - json.dumps({"action": "event", "event": _event()}).encode() + b"\n" - ) - assert client.recv(16) == b"ok\n" - - assert spool.batch("run-a", "producer-a")[0]["resources"]["rss_bytes"] == 100 - - with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as client: - client.connect(str(socket_path)) - client.sendall(b'{"action":"close"}\n') - assert client.recv(16) == b"ok\n" - thread.join(timeout=2) - assert not thread.is_alive() - - -def test_subprocess_exchanges_ambient_token_and_delivers_events( - tmp_path, monkeypatch, real_local_telemetry -) -> None: - requests: list[tuple[str, str, dict[str, object]]] = [] - - class Handler(BaseHTTPRequestHandler): - def do_POST(self): - length = int(self.headers.get("Content-Length", "0")) - payload = json.loads(self.rfile.read(length)) - requests.append((self.path, self.headers["Authorization"], payload)) - if self.path.endswith("/v1/auth/huggingface/exchange"): - response = {"access_token": "collector-token", "expires_in": 3600} - status = 200 - elif self.path == "/v1/runs": - response = {"registered": True} - status = 201 - else: - response = { - "accepted": len(payload["events"]), - "duplicates": 0, - } - status = 202 - body = json.dumps(response).encode() - self.send_response(status) - self.send_header("Content-Type", "application/json") - self.send_header("Content-Length", str(len(body))) - self.end_headers() - self.wfile.write(body) - - def log_message(self, format, *args): - return - - server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) - server_thread = threading.Thread(target=server.serve_forever) - server_thread.start() - monkeypatch.setenv("HF_TOKEN", "hf-ambient-test-token") - emitter = LocalTelemetryEmitter.start( - run_id="subprocess-run", - country_code="US", - pipeline="test-pipeline", - development_collector_url=f"http://127.0.0.1:{server.server_port}", - spool_path=tmp_path / "subprocess.sqlite3", - heartbeat_seconds=60, - ) - assert emitter.available - emitter.stage("compile", message="Started.") - emitter.complete() - assert emitter._process is not None - emitter._process.wait(timeout=10) - server.shutdown() - server_thread.join(timeout=2) - server.server_close() - - assert requests[0][0] == "/v1/auth/huggingface/exchange" - assert requests[0][1] == "Bearer hf-ambient-test-token" - assert requests[0][2] == {} - assert requests[1][0] == "/v1/runs" - assert requests[1][1] == "Bearer collector-token" - ingestion = [request for request in requests if request[0].endswith("/events")] - assert ingestion - events = [ - event - for _, authorization, payload in ingestion - for event in payload["events"] - if authorization == "Bearer collector-token" - ] - assert [event["sequence"] for event in events] == list(range(1, len(events) + 1)) - identity = events[0]["details"]["identity"] - assert identity["host"]["cpu_count"] is not None - assert "runtime" in identity - assert events[0]["message"] == BUILD_STARTED_MESSAGE - assert events[-1]["message"] == BUILD_COMPLETED_MESSAGE - assert events[-1]["status"] == "completed" - - -def test_failure_marks_active_stage_failed_before_run_failure(): - from test_support.microcosm_build.telemetry import FakeTelemetryEmitter - - emitter = FakeTelemetryEmitter(TelemetryRun("failed-build", "GB", "test")) - emitter.transition_stage("calibration") - emitter.fail(ValueError("invalid calibration")) - - assert [(event["event_type"], event["status"]) for event in emitter.events] == [ - ("stage", "started"), - ("stage", "failed"), - ("run", "failed"), - ] - assert emitter.events[1]["message"] == "invalid calibration" - assert emitter.events[1]["details"]["error_type"] == "ValueError" - assert emitter.events[2]["details"]["failed_during"] == "calibration" - assert not emitter.available - - -@pytest.mark.parametrize("failure_point", ["mkdtemp", "chmod", "cache_dir"]) -def test_early_startup_failure_returns_disabled_handle( - tmp_path, monkeypatch, capsys, real_local_telemetry, failure_point -): - from microcosm.build import telemetry_emitter as module - - runtime_dir = tmp_path / "runtime" - - def make_runtime(**kwargs): - runtime_dir.mkdir() - return str(runtime_dir) - - def fail(*args, **kwargs): - raise OSError("telemetry setup refused") - - monkeypatch.setattr(module.tempfile, "mkdtemp", make_runtime) - monkeypatch.setattr( - module.subprocess, - "Popen", - lambda *a, **kw: pytest.fail("spawned after failed setup"), - ) - if failure_point == "mkdtemp": - monkeypatch.setattr(module.tempfile, "mkdtemp", fail) - elif failure_point == "chmod": - monkeypatch.setattr(Path, "chmod", fail) - else: - monkeypatch.setattr(module, "_cache_dir", fail) - - emitter = LocalTelemetryEmitter.start( - run_id="failed-start", - country_code="GB", - pipeline="test", - development_collector_url="http://127.0.0.1:1", - ) - - assert not emitter.available - assert not runtime_dir.exists() - assert "warning" in capsys.readouterr().err.lower() - emitter.close() - - -def test_default_fixture_intercepts_preimported_entrypoint_and_class( - monkeypatch, fake_telemetry_emitters -): - from microcosm.build import telemetry_emitter as module - from microcosm.build.uk_runtime import rowwise_staging - from test_support.microcosm_build.telemetry import FakeTelemetryEmitter - - monkeypatch.setenv("HF_TOKEN", "hf-ambient-must-not-be-used") - monkeypatch.setattr( - module.subprocess, - "Popen", - lambda *a, **kw: pytest.fail("test spawned real emitter"), - ) - for start in ( - rowwise_staging.start_local_telemetry_emitter_service, - LocalTelemetryEmitter.start, - ): - emitter = start(run_id="fake", country_code="GB", pipeline="test") - assert isinstance(emitter, FakeTelemetryEmitter) - assert emitter in fake_telemetry_emitters - - -def test_default_fixture_isolates_cached_huggingface_credentials(tmp_path): - from huggingface_hub import constants, get_token - - assert Path(constants.HF_TOKEN_PATH).is_relative_to(tmp_path) - assert get_token() is None - - -@pytest.mark.parametrize("collector_url", [None, "https://collector.example"]) -def test_real_service_opt_in_requires_explicit_loopback( - real_local_telemetry, collector_url -): - with pytest.raises(ValueError, match="loopback"): - LocalTelemetryEmitter.start( - run_id="isolated", - country_code="GB", - pipeline="test", - development_collector_url=collector_url, - ) - - -def test_close_does_not_report_success(): - from test_support.microcosm_build.telemetry import FakeTelemetryEmitter - - emitter = FakeTelemetryEmitter(TelemetryRun("cleanup", "GB", "test")) - emitter.transition_stage("calibration") - emitter.close() - assert [(event["event_type"], event["status"]) for event in emitter.events] == [ - ("stage", "started") - ] - - -def test_failure_without_active_stage_only_reports_run_failure(): - from test_support.microcosm_build.telemetry import FakeTelemetryEmitter - - emitter = FakeTelemetryEmitter(TelemetryRun("failed", "GB", "test")) - emitter.fail(ValueError("before first stage"), failed_during="preflight") - assert [(event["event_type"], event["status"]) for event in emitter.events] == [ - ("run", "failed") - ] - assert emitter.events[0]["details"]["failed_during"] == "preflight" - - -@pytest.mark.parametrize("drain_seconds", [0.1, 1.0]) -def test_shutdown_retries_wait_after_stop_without_exceeding_deadline( - tmp_path, monkeypatch, drain_seconds -) -> None: - clock = SimpleNamespace(now=0.0) - attempts: list[float] = [] - - def flush_once() -> bool: - attempts.append(clock.now) - # Simulate a small amount of delivery work, so a broken busy loop - # still reaches its deadline without relying on real elapsed time. - clock.now += 0.01 - return False - - def sleep(seconds: float) -> None: - clock.now += seconds - - monkeypatch.setattr( - runtime_module, - "time", - SimpleNamespace(monotonic=lambda: clock.now, sleep=sleep), - ) - service = EmitterService( - socket_path=tmp_path / "unused.sock", - registration=_registration(), - spool=SimpleNamespace(has_deliverable=lambda: True), - delivery=SimpleNamespace(flush_once=flush_once), - sampler=_FakeSampler(), - heartbeat_seconds=60, - drain_seconds=drain_seconds, - ) - service._stop.set() - - service._drain() - - assert attempts - assert all( - later - earlier >= DRAIN_RETRY_SECONDS - for earlier, later in zip(attempts, attempts[1:], strict=False) - ) - assert clock.now == pytest.approx(drain_seconds) - - -def test_fallback_sampler_retains_reaped_child_cpu(tmp_path, monkeypatch) -> None: - proc_root = tmp_path / "proc" - - def write_stat( - pid: int, - parent_pid: int, - user_ticks: int, - system_ticks: int, - child_user_ticks: int = 0, - child_system_ticks: int = 0, - ) -> Path: - # Fields start at the state after the parenthesized process name. - fields = ["0"] * 22 - fields[0] = "S" - fields[1] = str(parent_pid) - fields[11:15] = map( - str, (user_ticks, system_ticks, child_user_ticks, child_system_ticks) - ) - fields[19] = "1000" - fields[21] = "4" - path = proc_root / str(pid) / "stat" - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(f"{pid} (build worker) {' '.join(fields)}") - return path - - write_stat(100, 1, 100, 200) - child_path = write_stat(101, 100, 500, 300) - monkeypatch.setattr(resources_module, "psutil", None) - monkeypatch.setattr( - resources_module, - "Path", - lambda path: proc_root / Path(path).relative_to("/proc"), - ) - monkeypatch.setattr( - resources_module, - "os", - SimpleNamespace( - sysconf=lambda name: {"SC_CLK_TCK": 100, "SC_PAGE_SIZE": 4096}[name] - ), - ) - sampler = resources_module.ProcessTreeSampler(100) - - before = sampler.sample() - child_path.unlink() - write_stat(100, 1, 100, 200, child_user_ticks=500, child_system_ticks=300) - after = sampler.sample() - - assert before["cpu_user_seconds"] == after["cpu_user_seconds"] == 6.0 - assert before["cpu_system_seconds"] == after["cpu_system_seconds"] == 5.0 - assert before["rss_bytes"] == 8 * 4096 - assert after["rss_bytes"] == 4 * 4096 - assert after["peak_rss_bytes"] == before["rss_bytes"] diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_full_build_cli.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_full_build_cli.py index 9fc790b549..46547b9eb7 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_full_build_cli.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_full_build_cli.py @@ -689,7 +689,7 @@ def close(self): monkeypatch.setattr( cli, "start_telemetry_emitter", - lambda requested, *, build_id, run_kind: ( + lambda requested, *, build_id, run_kind, close_service_on_finish: ( events.append(("start", build_id, run_kind)) or FakeEmitter() ), ) @@ -733,7 +733,7 @@ def close(self): monkeypatch.setattr( cli, "start_telemetry_emitter", - lambda requested, *, build_id, run_kind: ( + lambda requested, *, build_id, run_kind, close_service_on_finish: ( events.append(("start", build_id, run_kind)) or FakeEmitter() ), ) diff --git a/packages/microcosm-build/tests/engine_free/uk/test_uk_orrery_publication.py b/packages/microcosm-build/tests/engine_free/uk/test_uk_orrery_publication.py index 1e6b3c8da1..d35c21ea5e 100644 --- a/packages/microcosm-build/tests/engine_free/uk/test_uk_orrery_publication.py +++ b/packages/microcosm-build/tests/engine_free/uk/test_uk_orrery_publication.py @@ -12,6 +12,7 @@ ) from microcosm.build.uk_runtime.orrery_contract import save_uk_graph_schema from microcosm.graph import Graph, Node, Owned, SourceRef, StructuralDelta +from test_support.microcosm_build.telemetry import fake_publication_inventory def fixture(tmp_path, **options): @@ -78,6 +79,8 @@ def test_publication_precedes_manifest_registration_and_receipt_is_a_snapshot( events = [] class Emitter: + publication_inventory = staticmethod(fake_publication_inventory) + def publish_graph(self, directory, inventory, *, wait_seconds): assert not (args.out / "orrery.publication.json").exists() events.append("publication_attempt") @@ -166,6 +169,8 @@ def test_interrupt_preserves_local_evidence_without_waiting_or_staging( waits = [] class Emitter: + publication_inventory = staticmethod(fake_publication_inventory) + def publish_graph(self, directory, inventory, *, wait_seconds): waits.append(wait_seconds) return { @@ -211,6 +216,24 @@ def broken(*args, **kwargs): ) +def test_no_emitter_delegates_publication_without_a_run_id(tmp_path, monkeypatch): + args = fixture(tmp_path) + args.publish_orrery = True + monkeypatch.setattr(rowwise_staging, "_ACTIVE_EMITTER", None) + calls = [] + + def publish(directory, inventory, **options): + assert "run_id" not in inventory + assert options == {"wait_seconds": 0} + calls.append(directory) + return {"status": "pending", "error_code": "credential_missing"} + + monkeypatch.setattr(orrery_publication, "publish_graph", publish) + receipt = orrery_publication.finalize_graph(args, manifest=None).receipt + assert receipt["status"] == "pending" + assert calls == [args.attempt_evidence / "orrery-publication"] + + @pytest.mark.parametrize( "explicit,local,disabled,expected", [ @@ -228,3 +251,75 @@ def test_publication_has_explicit_local_only_and_opt_out_controls( publish_orrery=explicit, staging_local_only=local, no_staging=disabled ) assert orrery_publication.publication_enabled(args) is expected + + +@pytest.mark.parametrize("active", [False, True]) +def test_unavailable_inventory_preserves_graph_without_claiming_a_queued_job( + tmp_path, monkeypatch, active +): + args = fixture(tmp_path) + args.publish_orrery = True + calls = [] + + def unavailable(directory, roles): + calls.append(directory) + return None + + def must_not_publish(*args, **options): + pytest.fail("publication attempted without a measured inventory") + + if active: + from types import SimpleNamespace + + emitter = SimpleNamespace( + publication_inventory=unavailable, publish_graph=must_not_publish + ) + monkeypatch.setattr(rowwise_staging, "_ACTIVE_EMITTER", emitter) + monkeypatch.setattr( + orrery_publication, + "publication_inventory", + lambda *a, **k: pytest.fail("started another inventory service"), + ) + else: + monkeypatch.setattr(rowwise_staging, "_ACTIVE_EMITTER", None) + monkeypatch.setattr(orrery_publication, "publication_inventory", unavailable) + monkeypatch.setattr(orrery_publication, "publish_graph", must_not_publish) + manifest = {"outputs": {}} + result = orrery_publication.finalize_graph(args, manifest=manifest) + assert result.receipt["status"] == "failed" + assert result.receipt["publication_id"] is None + assert result.receipt["error_code"] == "local_service_unavailable" + preserved = args.attempt_evidence / "orrery-publication" + assert calls == [preserved] + for name in ("graph.schema.json", "graph.orrery.json"): + assert (preserved / name).read_bytes() == (args.out / name).read_bytes() + assert f"orrery/{name}" in manifest["outputs"] + assert "orrery.upload.json" not in result.files + assert "orrery/orrery.publication.json" in manifest["outputs"] + + +def test_enqueue_failure_is_recorded_without_replacing_graph_exports( + tmp_path, monkeypatch +): + args = fixture(tmp_path) + args.publish_orrery = True + + class Emitter: + publication_inventory = staticmethod(fake_publication_inventory) + + def publish_graph(self, directory, inventory, *, wait_seconds): + return { + "version": 1, + "publication_id": inventory["publication_id"], + "status": "failed", + "error_code": "local_service_unavailable", + } + + monkeypatch.setattr(rowwise_staging, "_ACTIVE_EMITTER", Emitter()) + result = orrery_publication.finalize_graph(args, manifest=None) + assert result.receipt["status"] == "failed" + assert result.receipt["error_code"] == "local_service_unavailable" + assert result.receipt["publication_id"] + assert (args.out / "graph.orrery.json").is_file() + assert (args.out / "orrery.upload.json").is_file() + assert (args.out / "orrery.evidence-manifest.json").is_file() diff --git a/test_support/microcosm_build/telemetry.py b/test_support/microcosm_build/telemetry.py index 7ff55a2475..8cd80791be 100644 --- a/test_support/microcosm_build/telemetry.py +++ b/test_support/microcosm_build/telemetry.py @@ -1,32 +1,93 @@ -"""In-memory telemetry and isolated opt-in service startup for tests.""" +"""In-memory telemetry and synthetic graph requests for build tests.""" from __future__ import annotations +import hashlib +import uuid from collections.abc import Mapping from pathlib import Path from typing import Any +from microcosm_emitter.orrery.contracts import ( + PublicationFileRole, + PublicationInventory, + PublicationReceipt, +) +from microcosm_emitter.telemetry.service.lifecycle import ( + LifecycleState, + process_request, +) + from microcosm.build.telemetry_emitter import LocalTelemetryEmitter, TelemetryRun +def fake_publication_inventory( + directory: Path, roles: Mapping[str, PublicationFileRole] +) -> PublicationInventory: + """Measure only synthetic test files without starting an emitter service.""" + return { + "version": 1, + "publication_id": uuid.uuid4().hex, + "files": [ + { + "name": name, + "role": role, + "bytes": (directory / name).stat().st_size, + "sha256": hashlib.sha256((directory / name).read_bytes()).hexdigest(), + } + for name, role in sorted(roles.items()) + ], + } + + +def fake_publish_graph( + directory: Path, + inventory: PublicationInventory, + *, + wait_seconds: float = 30.0, +) -> PublicationReceipt: + """Return an in-memory receipt without persistence, credentials or HTTP.""" + return { + "version": 1, + "publication_id": inventory["publication_id"], + "status": "pending", + "error_code": None, + } + + class FakeTelemetryEmitter(LocalTelemetryEmitter): """Exercise real lifecycle methods without starting a service or socket.""" - def __init__(self, run: TelemetryRun) -> None: + def __init__( + self, run: TelemetryRun, *, close_service_on_finish: bool = True + ) -> None: + self.messages: list[Mapping[str, Any]] = [] + self._lifecycle = LifecycleState() super().__init__( run=run, - process=None, - socket_path=Path("/unused-test-telemetry-socket"), - runtime_dir=None, + transport=self, + close_service_on_finish=close_service_on_finish, ) - self.messages: list[Mapping[str, Any]] = [] - def _send(self, payload: Mapping[str, Any]) -> None: - self.messages.append(payload) + def send(self, payload: Mapping[str, Any]) -> None: + self._lifecycle, events = process_request(self._lifecycle, payload) + self.messages.extend({"action": "event", "event": event} for event in events) + + def close(self) -> None: + self._closed = True + + def publication_inventory( + self, directory: Path, roles: Mapping[str, PublicationFileRole] + ) -> PublicationInventory: + return fake_publication_inventory(directory, roles) def publish_graph( - self, directory: Path, inventory: dict, *, wait_seconds: float = 30.0 - ) -> dict: + self, + directory: Path, + inventory: PublicationInventory, + *, + wait_seconds: float = 30.0, + ) -> PublicationReceipt: self.messages.append( { "action": "graph_publication", @@ -34,12 +95,7 @@ def publish_graph( "inventory": inventory, } ) - return { - "version": 1, - "publication_id": inventory["publication_id"], - "status": "pending", - "error_code": None, - } + return fake_publish_graph(directory, inventory, wait_seconds=wait_seconds) @property def events(self) -> list[Mapping[str, Any]]: diff --git a/uv.lock b/uv.lock index 8da37c6d36..fa69800b45 100644 --- a/uv.lock +++ b/uv.lock @@ -800,12 +800,11 @@ name = "microcosm-build" version = "0.1.0" source = { editable = "packages/microcosm-build" } dependencies = [ - { name = "alembic" }, - { name = "httpx" }, { name = "huggingface-hub" }, { name = "jsonschema" }, { name = "microcosm-calibrate" }, { name = "microcosm-diagnostics" }, + { name = "microcosm-emitter" }, { name = "microcosm-fit" }, { name = "microcosm-frame" }, { name = "microcosm-graph" }, @@ -815,7 +814,6 @@ dependencies = [ { name = "pyyaml" }, { name = "referencing" }, { name = "scipy" }, - { name = "sqlalchemy" }, ] [package.optional-dependencies] @@ -842,16 +840,15 @@ dev = [ [package.metadata] requires-dist = [ - { name = "alembic", specifier = ">=1.13.3,<2" }, { name = "h5py", marker = "extra == 'uk'", specifier = ">=3" }, { name = "h5py", marker = "extra == 'us'", specifier = ">=3" }, - { name = "httpx", specifier = ">=0.27,<1" }, { name = "huggingface-hub", specifier = ">=0.20" }, { name = "jsonschema", specifier = ">=4.23,<5" }, { name = "microcosm-calibrate", editable = "packages/microcosm-calibrate" }, { name = "microcosm-data", marker = "extra == 'uk'", editable = "packages/microcosm-data" }, { name = "microcosm-data", marker = "extra == 'us'", editable = "packages/microcosm-data" }, { name = "microcosm-diagnostics", editable = "packages/microcosm-diagnostics" }, + { name = "microcosm-emitter", git = "https://github.com/PolicyEngine/microcosm-emitter?rev=12624a118db255405a0ce7397d53b3181deaf49a" }, { name = "microcosm-fit", editable = "packages/microcosm-fit" }, { name = "microcosm-frame", editable = "packages/microcosm-frame" }, { name = "microcosm-graph", editable = "packages/microcosm-graph" }, @@ -866,7 +863,6 @@ requires-dist = [ { name = "pyyaml", specifier = ">=6" }, { name = "referencing", specifier = ">=0.35,<1" }, { name = "scipy", specifier = ">=1.13" }, - { name = "sqlalchemy", specifier = ">=2,<3" }, { name = "tables", marker = "extra == 'uk'", specifier = ">=3" }, { name = "tables", marker = "extra == 'us'", specifier = ">=3" }, ] @@ -967,6 +963,18 @@ requires-dist = [{ name = "pydantic", specifier = ">=2.10,<3" }] [package.metadata.requires-dev] dev = [{ name = "pytest", specifier = ">=8" }] +[[package]] +name = "microcosm-emitter" +version = "0.1.0" +source = { git = "https://github.com/PolicyEngine/microcosm-emitter?rev=12624a118db255405a0ce7397d53b3181deaf49a#12624a118db255405a0ce7397d53b3181deaf49a" } +dependencies = [ + { name = "alembic" }, + { name = "httpx" }, + { name = "huggingface-hub" }, + { name = "psutil" }, + { name = "sqlalchemy" }, +] + [[package]] name = "microcosm-fit" version = "0.1.0"