diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 920345e5..b32f5980 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -50,6 +50,18 @@ jobs: name: distributions path: dist/* if-no-files-found: error + # push.yaml publishes each distribution from its own artifact, the + # interface before the emitter that requires it. + - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: interface-distribution + path: dist/microcosm_emitter_interface-* + if-no-files-found: error + - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: emitter-distribution + path: dist/microcosm_emitter-* + if-no-files-found: error collector: needs: build-packages diff --git a/.github/workflows/push.yaml b/.github/workflows/push.yaml index e41d87cc..be83b0eb 100644 --- a/.github/workflows/push.yaml +++ b/.github/workflows/push.yaml @@ -14,7 +14,8 @@ permissions: # remaining jobs verify, tag, then publish, each only after the one before. # # PyPI trusts this file by name, in the pypi environment, which only main can -# use. Renaming it stops publication. +# use, for both distributions (microcosm-emitter and microcosm-emitter-interface). +# Renaming it stops publication. jobs: versioning: name: Update versioning @@ -68,11 +69,25 @@ jobs: environment: pypi permissions: id-token: write + # The emitter requires the interface of the same version, so the interface + # is uploaded first: PyPI never offers an emitter whose interface is + # missing. Files already on PyPI are skipped, so rerunning this job after + # a failed or partial upload uploads only the missing files instead of + # stopping on "File already exists". steps: - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 with: - name: distributions - path: dist + name: interface-distribution + path: dist/interface - uses: pypa/gh-action-pypi-publish@dc37677b2e1c63e2034f94d8a5b11f265b73ba33 # release/v1 with: - packages-dir: dist/ + packages-dir: dist/interface/ + skip-existing: true + - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 + with: + name: emitter-distribution + path: dist/emitter + - uses: pypa/gh-action-pypi-publish@dc37677b2e1c63e2034f94d8a5b11f265b73ba33 # release/v1 + with: + packages-dir: dist/emitter/ + skip-existing: true diff --git a/AGENTS.md b/AGENTS.md index b9c49e47..65dd9070 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -3,7 +3,7 @@ - Use uv, Python 3.13+, pytest, and Ruff. Keep workflows as single-command calls to Bash scripts when shell logic has multiple lines. - The generic runtime must not import telemetry or Microcosm. The telemetry client must not import service implementation, database, or authentication modules. - Graph adapters must not import service, database, authentication or HTTP code. Inventory hashing, durable enqueue, receipt lookup and retries run in the separate local emitter service; no direct-database fallback. -- Publish only the `microcosm-emitter` distribution. Keep the generic host and required telemetry library internally separate under `microcosm_emitter`; do not introduce standalone telemetry packages or optional telemetry installation. +- Publish exactly two distributions, always together and at the same version: `microcosm-emitter` (the separately installed emitter, with its generic host and required telemetry) and `microcosm-emitter-interface` (the dependency-free package Microcosm builds import to start and call the emitter). `microcosm-emitter==X` requires `microcosm-emitter-interface==X`. Keep the generic host and required telemetry library internally separate under `microcosm_emitter`; do not introduce standalone telemetry packages, optional telemetry installation, or further distributions. - Python databases use SQLAlchemy declarative ORM and Alembic exclusively. No handwritten SQL, direct-driver persistence, create_all(), schema repair, or pre-Alembic adoption. TypeScript database work, if introduced, must use Drizzle ORM and Drizzle Kit migrations. - Tests use temporary databases created through real migrations, synthetic credentials, and loopback network services. Never use ambient credentials or contact the production collector in tests. - Preserve the existing collector protocol; do not add a general hosted events API. diff --git a/README.md b/README.md index 284e3d47..cf201662 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,11 @@ # Microcosm emitter -`microcosm-emitter` is one Python distribution containing the process host, required telemetry library and completed-graph publication library: +This repository publishes two Python distributions with one shared version: + +- `microcosm-emitter-interface` (`packages/microcosm-emitter-interface/`) is the small, dependency-free package a Microcosm build imports. It starts the separately installed emitter as the build's child process and offers one typed call per kind of hand-over. See its README for the compatibility rules and protocol fixtures. +- `microcosm-emitter` is the emitter itself: the process host, required telemetry library and completed-graph publication library, installed on its own with `uv tool install microcosm-emitter`. It depends on the interface release of the same version. + +The `microcosm-emitter` distribution contains: - `microcosm_emitter.host` owns subprocess startup, a private Unix socket, explicit module loading, parent-process monitoring, and shutdown. Its code does not import telemetry, database, authentication, or Microcosm modules. - `microcosm_emitter.telemetry.client` is the thin build adapter. It starts the local telemetry emitter service and sends build requests, without importing the service implementation, database, or authentication libraries. @@ -67,13 +72,17 @@ The collector compatibility job tests the installed wheel against a pinned dashb ## Publication -Every merge to `main` releases the Python distribution `microcosm-emitter` to PyPI. Each pull request adds a changelog fragment named `changelog.d/..md`, where the type is `breaking`, `added`, `changed`, `fixed` or `removed`; `tools/check-changelog.sh` refuses a pull request without one. +Every merge to `main` releases both Python distributions, `microcosm-emitter` and `microcosm-emitter-interface`, to PyPI with one shared version. Each pull request adds a changelog fragment named `changelog.d/..md`, where the type is `breaking`, `added`, `changed`, `fixed` or `removed`; `tools/check-changelog.sh` refuses a pull request without one. `push.yaml` runs twice for each merge: -1. On the merge commit, it bumps the version in `pyproject.toml` and `uv.lock`: a major bump for a breaking fragment, a minor bump for an added or removed fragment, and a patch bump otherwise. It collects the fragments into `CHANGELOG.md` and commits the result as "Update microcosm-emitter" using the PolicyEngine GitHub App, which may bypass the `main` ruleset. -2. On that commit, it runs every CI check, tags the commit with the bare version (for example `0.2.0`), and publishes the tested wheel and source archive to PyPI. Each step starts only after the one before succeeds. +1. On the merge commit, it bumps the version in both `pyproject.toml` files, the emitter's exact `microcosm-emitter-interface` pin and `uv.lock`: a major bump for a breaking fragment, a minor bump for an added or removed fragment, and a patch bump otherwise. It collects the fragments into `CHANGELOG.md` and commits the result as "Update microcosm-emitter" using the PolicyEngine GitHub App, which may bypass the `main` ruleset. +2. On that commit, it runs every CI check, tags the commit with the bare version (for example `0.2.0`), and publishes the tested wheels and source archives of both distributions to PyPI. Each step starts only after the one before succeeds. + +The publish job uploads `microcosm-emitter-interface` before `microcosm-emitter`, because the emitter requires the interface of the same version: if the interface upload fails, the emitter is not uploaded. Both uploads skip files PyPI already holds, so rerunning a failed publish job uploads only the missing files. + +PyPI accepts uploads only through trusted publishing from `push.yaml` in the `pypi` environment, which only `main` can use; each of the two PyPI projects needs its own trusted-publisher entry for that workflow and environment. CI holds no long-lived registry token. Renaming the workflow or the environment stops publication until PyPI's trusted-publisher settings change to match. -PyPI accepts uploads only through trusted publishing from `push.yaml` in the `pypi` environment, which only `main` can use. CI holds no long-lived registry token. Renaming the workflow or the environment stops publication until PyPI's trusted-publisher settings change to match. +The artifact check (`tools/check-artifacts.sh`) builds both distributions, installs the emitter with its interface into one clean environment, as `uv tool install` would, and the interface alone into another, as a build's environment would. From the second, it starts the first's `microcosm-emitter` command and sends telemetry and an Orrery hand-over. -No hosted-service deployment or consumer activation follows from publishing this package. +No hosted-service deployment or consumer activation follows from publishing these packages. diff --git a/changelog.d/emitter-interface-package.added.md b/changelog.d/emitter-interface-package.added.md new file mode 100644 index 00000000..ee1cfa23 --- /dev/null +++ b/changelog.d/emitter-interface-package.added.md @@ -0,0 +1 @@ +Add the `microcosm-emitter-interface` distribution: a dependency-free package that a Microcosm build depends on with a version range. `EmitterClient.start` runs the installed `microcosm-emitter` command as the build's child process, which inherits the build's environment and finds the same Hugging Face credential, and refuses an emitter that does not accept its socket protocol version. It offers typed calls for telemetry events and Orrery input hand-overs, and never raises into the build: not because the emitter is missing or refusing a request, and not because an argument cannot be converted to JSON. A refused call reports why in `EmitterClient.last_failure`. The emitter's in-process telemetry and Orrery clients now build their messages with the interface's code, and the in-process Orrery client reports an argument it cannot send (`invalid_arguments`) instead of raising. diff --git a/packages/microcosm-emitter-interface/LICENSE b/packages/microcosm-emitter-interface/LICENSE new file mode 100644 index 00000000..bdb2af69 --- /dev/null +++ b/packages/microcosm-emitter-interface/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 PolicyEngine + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/packages/microcosm-emitter-interface/README.md b/packages/microcosm-emitter-interface/README.md new file mode 100644 index 00000000..210901fb --- /dev/null +++ b/packages/microcosm-emitter-interface/README.md @@ -0,0 +1,35 @@ +# Microcosm emitter interface + +`microcosm-emitter-interface` is the small, dependency-free package a Microcosm build imports to hand work to the separately installed `microcosm-emitter`. Microcosm decides what to hand over; the emitter decides where it goes. + +Install the emitter on its own, outside the build's environment: + +```bash +uv tool install microcosm-emitter +``` + +The build then depends on this package with a version range and starts the emitter through it: + +```python +from microcosm_emitter_interface import EmitterClient, TelemetryRun, TelemetrySettings + +client = EmitterClient.start( + telemetry=TelemetrySettings(run=TelemetryRun("run-id", "UK", "uk-build")), +) +client.telemetry.transition_stage("calibrating") +client.telemetry.complete() +client.close() +``` + +`EmitterClient.start` runs the `microcosm-emitter` command as the build's own child process. The command comes from the `command` argument, then the `MICROCOSM_EMITTER_COMMAND` environment variable, then `PATH`. The child inherits the build's environment variables and user, so it finds the same Hugging Face credential: `HF_TOKEN`, then `HUGGINGFACE_TOKEN`, then the token saved by `hf auth login`. + +No call raises into the build: not because the emitter is missing, slow or refusing a request, and not because an argument cannot be converted to JSON (for example a multi-element NumPy array or a self-referencing mapping). Each call converts its arguments inside its error handling and reports whether the emitter accepted it; after a refusal, `client.last_failure` says why (`emitter_unavailable`, `invalid_arguments` or `not_accepted`). Startup settings that cannot be converted start an unavailable client (`invalid_configuration`). The build continues in every case. + +## Compatibility + +Both packages are released together with one version number, and `microcosm-emitter==X` requires `microcosm-emitter-interface==X`. A build, though, may run with a different emitter from the interface it depends on: it keeps the emitter it started with, and the next build uses whichever emitter is installed then. So the socket protocol carries its own version: + +- `PROTOCOL_VERSION` names the shape of every message this package sends. The emitter lists the versions it accepts in its `hello` answer, and the client refuses an emitter that does not list its version (`unsupported_protocol_version`); the build continues without it. +- **Every emitter release must accept every protocol version that interface releases still in use send.** Adding a module or a message kind keeps the version; changing the shape of an existing message needs a new version, which emitters then accept alongside the old one. +- `fixtures/protocol-v/` holds one JSON file per message of protocol version `N`, including how the command is started (`host-launch.json`: its arguments and the configuration object on standard input) and each socket request's whole envelope (`host-ping.json`, `host-message-envelope.json`, `host-query-envelope.json`, `host-close.json`, and `host-hello.json`). This package's tests check that each typed call, the launcher and the socket transport send exactly their fixture, and the emitter's contract tests check that it accepts every fixture of every version it supports, so a change to the command's arguments or the envelope cannot pass by changing both sides at once. An emitter's `hello` answer must contain the recorded keys and may add more. A released fixture is never edited or removed: the emitter's tests pin each one's SHA-256. +- An emitter that lacks a module this package asks for starts without it (`--skip-unavailable-modules`); calls for that module report that they were not accepted. diff --git a/packages/microcosm-emitter-interface/pyproject.toml b/packages/microcosm-emitter-interface/pyproject.toml new file mode 100644 index 00000000..0853b936 --- /dev/null +++ b/packages/microcosm-emitter-interface/pyproject.toml @@ -0,0 +1,18 @@ +[project] +name = "microcosm-emitter-interface" +version = "0.4.0" +description = "Typed hand-over calls from Microcosm builds to the separately installed microcosm-emitter" +readme = "README.md" +requires-python = ">=3.13" +license = "MIT" +dependencies = [] + +[project.urls] +Source = "https://github.com/PolicyEngine/microcosm-emitter" + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[tool.hatch.build.targets.wheel] +packages = ["src/microcosm_emitter_interface"] diff --git a/packages/microcosm-emitter-interface/src/microcosm_emitter_interface/__init__.py b/packages/microcosm-emitter-interface/src/microcosm_emitter_interface/__init__.py new file mode 100644 index 00000000..c02750d6 --- /dev/null +++ b/packages/microcosm-emitter-interface/src/microcosm_emitter_interface/__init__.py @@ -0,0 +1,28 @@ +"""Typed hand-overs from Microcosm builds to the separately installed emitter. + +Importing this package starts no process and opens no socket or file. +""" + +from microcosm_emitter_interface.client import EmitterClient, RequestFailure +from microcosm_emitter_interface.launcher import ( + COMMAND_ENVIRONMENT_VARIABLE, + DEFAULT_COMMAND, + EmitterStartupError, +) +from microcosm_emitter_interface.orrery import OrrerySettings +from microcosm_emitter_interface.protocol import PROTOCOL_VERSION +from microcosm_emitter_interface.spool import default_spool_path +from microcosm_emitter_interface.telemetry import TelemetryRun, TelemetrySettings + +__all__ = [ + "COMMAND_ENVIRONMENT_VARIABLE", + "DEFAULT_COMMAND", + "PROTOCOL_VERSION", + "EmitterClient", + "EmitterStartupError", + "OrrerySettings", + "RequestFailure", + "TelemetryRun", + "TelemetrySettings", + "default_spool_path", +] diff --git a/packages/microcosm-emitter-interface/src/microcosm_emitter_interface/client.py b/packages/microcosm-emitter-interface/src/microcosm_emitter_interface/client.py new file mode 100644 index 00000000..1ba3f3e4 --- /dev/null +++ b/packages/microcosm-emitter-interface/src/microcosm_emitter_interface/client.py @@ -0,0 +1,355 @@ +"""One typed call per kind of hand-over from a build to its local emitter. + +Starting the client starts the installed ``microcosm-emitter`` command as this +process's child. No call raises into the build: not when the emitter is missing, +slow or refusing, and not when an argument cannot be converted to JSON (for +example a multi-element NumPy array or a self-referencing mapping). Each call +converts its arguments inside its error handling, reports that it was not +accepted, records why in ``EmitterClient.last_failure``, and the build continues. +""" + +from __future__ import annotations + +import os +import sys +import threading +import time +from collections.abc import Callable, Mapping +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Literal + +from microcosm_emitter_interface.launcher import ( + EmitterProcess, + EmitterStartupError, + start_emitter, +) +from microcosm_emitter_interface.orrery import ( + OrrerySettings, + PublicationFileRole, + PublicationInventory, + PublicationReceipt, + answered_inventory, + answered_receipt, + enqueue_message, + inventory_message, + publication_id_of, + receipt_message, + retry_message, + unavailable_receipt, +) +from microcosm_emitter_interface.protocol import ( + ACKNOWLEDGEMENT_TIMEOUT_SECONDS, + FILE_REQUEST_TIMEOUT_SECONDS, + ORRERY_MODULE, + STARTUP_TIMEOUT_SECONDS, + TELEMETRY_MODULE, +) +from microcosm_emitter_interface.spool import default_spool_path +from microcosm_emitter_interface.telemetry import ( + TelemetryCommand, + TelemetryRecorder, + TelemetrySettings, + telemetry_message, +) + +STARTUP_WARNING = ( + "Local Microcosm emitter failed to start ({reason}); the build will continue." +) +REQUEST_WARNING = ( + "The local Microcosm emitter did not accept a {kind} request " + "({reason}, {error_type}); the build will continue." +) + +type RequestFailureReason = Literal[ + # No emitter is running, or it did not load the call's module. + "emitter_unavailable", + # The call's arguments could not be converted to a protocol message. + "invalid_arguments", + # The emitter refused the request, or did not answer in time. + "not_accepted", +] + +type MessageBuilder = Callable[[], Mapping[str, Any]] + + +@dataclass(frozen=True) +class RequestFailure: + """Why a call reported that the emitter did not accept it. + + ``error_type`` is only the exception's type name, never its message. + """ + + kind: str + reason: RequestFailureReason + error_type: str | None = None + + +class EmitterClient: + """Typed hand-overs to one emitter process owned by this build.""" + + def __init__( + self, + process: EmitterProcess | None = None, + *, + startup_error: EmitterStartupError | None = None, + ) -> None: + self._process = process + self.startup_error = startup_error + self._closed = False + self._warned: set[str] = set() + self._warning_lock = threading.Lock() + #: The most recent call that reported failure, and why. + self.last_failure: RequestFailure | None = None + self.telemetry = TelemetryRecorder(self) + + @classmethod + def start( + cls, + *, + telemetry: TelemetrySettings | None = None, + orrery: OrrerySettings | None = None, + spool_path: Path | str | None = None, + command: str | os.PathLike[str] | None = None, + startup_timeout: float = STARTUP_TIMEOUT_SECONDS, + ) -> EmitterClient: + """Start the selected modules; return an unavailable client on failure.""" + try: + path = Path(spool_path) if spool_path is not None else default_spool_path() + modules: dict[str, dict[str, Any]] = {} + if telemetry is not None: + modules[TELEMETRY_MODULE] = telemetry.configuration(path) + if orrery is not None: + modules[ORRERY_MODULE] = orrery.configuration(path) + except Exception: + failure = EmitterStartupError("invalid_configuration") + print(STARTUP_WARNING.format(reason=failure.reason), file=sys.stderr) + return cls(startup_error=failure) + try: + process = start_emitter( + modules, command=command, startup_timeout=startup_timeout + ) + except EmitterStartupError as error: + print(STARTUP_WARNING.format(reason=error.reason), file=sys.stderr) + return cls(startup_error=error) + except Exception: + failure = EmitterStartupError("startup_failure") + print(STARTUP_WARNING.format(reason=failure.reason), file=sys.stderr) + return cls(startup_error=failure) + return cls(process) + + @property + def available(self) -> bool: + return self._process is not None and not self._closed + + @property + def modules(self) -> frozenset[str]: + """The modules the running emitter loaded.""" + if not self.available: + return frozenset() + assert self._process is not None + return frozenset(self._process.description.get("modules", ())) + + @property + def emitter_version(self) -> str | None: + if self._process is None: + return None + version = self._process.description.get("emitter_version") + return version if isinstance(version, str) else None + + def _fail( + self, + kind: str, + reason: RequestFailureReason, + error: BaseException | None = None, + ) -> None: + """Record why a call failed; warn once per kind of call. + + A missing emitter was already reported when it failed to start, so + only refused requests and unconvertible arguments print a warning. + """ + error_type = None if error is None else type(error).__name__ + self.last_failure = RequestFailure(kind, reason, error_type) + if reason == "emitter_unavailable": + return + with self._warning_lock: + if kind in self._warned: + return + self._warned.add(kind) + print( + REQUEST_WARNING.format(kind=kind, reason=reason, error_type=error_type), + file=sys.stderr, + ) + + def _message( + self, build_message: MessageBuilder, *, kind: str + ) -> dict[str, Any] | None: + """The converted message, or None after recording why it failed.""" + try: + return dict(build_message()) + except Exception as error: + self._fail(kind, "invalid_arguments", error) + return None + + def _send(self, module: str, build_message: MessageBuilder, *, kind: str) -> bool: + if module not in self.modules: + self._fail(kind, "emitter_unavailable") + return False + message = self._message(build_message, kind=kind) + if message is None: + return False + assert self._process is not None + try: + self._process.transport.send( + module, message, timeout=ACKNOWLEDGEMENT_TIMEOUT_SECONDS + ) + except Exception as error: + self._fail(kind, "not_accepted", error) + return False + return True + + def _request( + self, + module: str, + build_message: MessageBuilder, + *, + kind: str, + timeout: float = FILE_REQUEST_TIMEOUT_SECONDS, + ) -> tuple[dict[str, Any] | None, RequestFailureReason | None]: + """The module's answer, or None and why this call failed. + + The reason belongs to this call only. ``last_failure`` is shared by + every call and thread, so a receipt's error code never reads it. + """ + if module not in self.modules: + self._fail(kind, "emitter_unavailable") + return None, "emitter_unavailable" + message = self._message(build_message, kind=kind) + if message is None: + return None, "invalid_arguments" + assert self._process is not None + try: + return self._process.transport.query(module, message, timeout=timeout), None + except Exception as error: + self._fail(kind, "not_accepted", error) + return None, "not_accepted" + + def _query( + self, + module: str, + build_message: MessageBuilder, + *, + kind: str, + timeout: float = FILE_REQUEST_TIMEOUT_SECONDS, + ) -> dict[str, Any] | None: + answer, _ = self._request(module, build_message, kind=kind, timeout=timeout) + return answer + + def emit_telemetry(self, command: TelemetryCommand) -> bool: + """Hand over one telemetry event; True once the emitter committed it.""" + return self._send( + TELEMETRY_MODULE, lambda: telemetry_message(command), kind="telemetry" + ) + + def orrery_inventory( + self, + directory: Path, + roles: Mapping[str, PublicationFileRole], + *, + publication_id: str | None = None, + ) -> PublicationInventory | None: + """Ask the emitter to measure the selected preserved Orrery input files.""" + response = self._query( + ORRERY_MODULE, + lambda: inventory_message(directory, roles, publication_id), + kind="Orrery", + ) + return answered_inventory(response) + + def hand_over_orrery_inputs( + self, + directory: Path, + inventory: PublicationInventory, + *, + wait_seconds: float = 30.0, + ) -> PublicationReceipt: + """Queue the inventoried files; wait only after the durable enqueue.""" + publication_id = publication_id_of(inventory) + if publication_id is None: + self._fail("Orrery", "invalid_arguments") + return unavailable_receipt(None, error_code="invalid_arguments") + receipt = self._orrery_receipt_request( + lambda: enqueue_message(directory, inventory), publication_id + ) + deadline = time.monotonic() + _wait_seconds(wait_seconds) + while ( + self.available + and receipt["status"] == "pending" + and not receipt.get("attempts", 0) + and time.monotonic() < deadline + ): + time.sleep(0.1) + receipt = self.orrery_receipt(publication_id) + return receipt + + def orrery_receipt(self, publication_id: str) -> PublicationReceipt: + return self._orrery_receipt_request( + lambda: receipt_message(publication_id), publication_id + ) + + def retry_orrery_inputs( + self, publication_id: str, *, wait_seconds: float = 0 + ) -> PublicationReceipt: + receipt = self._orrery_receipt_request( + lambda: retry_message(publication_id), publication_id + ) + previous_attempts = receipt.get("attempts", 0) + deadline = time.monotonic() + _wait_seconds(wait_seconds) + while ( + self.available + and receipt["status"] == "pending" + and receipt.get("attempts", 0) == previous_attempts + and time.monotonic() < deadline + ): + time.sleep(0.1) + receipt = self.orrery_receipt(publication_id) + return receipt + + def _orrery_receipt_request( + self, build_message: MessageBuilder, publication_id: str | None + ) -> PublicationReceipt: + answer, failure = self._request(ORRERY_MODULE, build_message, kind="Orrery") + return answered_receipt( + answer, publication_id, error_code=_receipt_error_code(failure) + ) + + def close(self) -> None: + """Ask the emitter to drain and stop; repeated calls do nothing.""" + if self._closed: + return + self._closed = True + if self._process is not None: + try: + self._process.close() + except Exception as error: + self._fail("close", "not_accepted", error) + + +def _receipt_error_code(failure: RequestFailureReason | None) -> str: + """The error code of an unavailable receipt, from its own call's failure. + + An answer without a receipt, or no answer, is ``local_service_unavailable``, + as in the emitter's in-process client. + """ + if failure == "invalid_arguments": + return "invalid_arguments" + return "local_service_unavailable" + + +def _wait_seconds(value: Any) -> float: + """A usable waiting time; an unusable one means not waiting.""" + try: + seconds = float(value) + except Exception: + return 0.0 + return seconds if seconds > 0 else 0.0 diff --git a/packages/microcosm-emitter-interface/src/microcosm_emitter_interface/fixtures/protocol-v1/host-close.json b/packages/microcosm-emitter-interface/src/microcosm_emitter_interface/fixtures/protocol-v1/host-close.json new file mode 100644 index 00000000..2bd16620 --- /dev/null +++ b/packages/microcosm-emitter-interface/src/microcosm_emitter_interface/fixtures/protocol-v1/host-close.json @@ -0,0 +1,10 @@ +{ + "protocol_version": 1, + "module": "host", + "kind": "envelope", + "sequence": 1, + "body": { + "action": "close" + }, + "response": "acknowledgement" +} diff --git a/packages/microcosm-emitter-interface/src/microcosm_emitter_interface/fixtures/protocol-v1/host-hello.json b/packages/microcosm-emitter-interface/src/microcosm_emitter_interface/fixtures/protocol-v1/host-hello.json new file mode 100644 index 00000000..86aaf941 --- /dev/null +++ b/packages/microcosm-emitter-interface/src/microcosm_emitter_interface/fixtures/protocol-v1/host-hello.json @@ -0,0 +1,14 @@ +{ + "protocol_version": 1, + "module": "host", + "kind": "hello", + "body": { + "action": "hello" + }, + "response_keys": [ + "emitter_version", + "modules", + "protocol_versions", + "skipped_modules" + ] +} diff --git a/packages/microcosm-emitter-interface/src/microcosm_emitter_interface/fixtures/protocol-v1/host-launch.json b/packages/microcosm-emitter-interface/src/microcosm_emitter_interface/fixtures/protocol-v1/host-launch.json new file mode 100644 index 00000000..6848fcf8 --- /dev/null +++ b/packages/microcosm-emitter-interface/src/microcosm_emitter_interface/fixtures/protocol-v1/host-launch.json @@ -0,0 +1,22 @@ +{ + "protocol_version": 1, + "module": "host", + "kind": "launch", + "body": { + "arguments": [ + "host", + "--socket", + "{socket_path}", + "--parent-pid", + "{parent_pid}", + "--skip-unavailable-modules" + ], + "standard_input": { + "orrery": { + "spool_path": "{spool_path}", + "development_collector_url": "http://127.0.0.1:1", + "publication_origin": "http://127.0.0.1:1" + } + } + } +} diff --git a/packages/microcosm-emitter-interface/src/microcosm_emitter_interface/fixtures/protocol-v1/host-message-envelope.json b/packages/microcosm-emitter-interface/src/microcosm_emitter_interface/fixtures/protocol-v1/host-message-envelope.json new file mode 100644 index 00000000..fb13d7e1 --- /dev/null +++ b/packages/microcosm-emitter-interface/src/microcosm_emitter_interface/fixtures/protocol-v1/host-message-envelope.json @@ -0,0 +1,13 @@ +{ + "protocol_version": 1, + "module": "host", + "kind": "envelope", + "body": { + "action": "message", + "module": "telemetry", + "message": { + "command": "complete" + } + }, + "response": "acknowledgement" +} diff --git a/packages/microcosm-emitter-interface/src/microcosm_emitter_interface/fixtures/protocol-v1/host-ping.json b/packages/microcosm-emitter-interface/src/microcosm_emitter_interface/fixtures/protocol-v1/host-ping.json new file mode 100644 index 00000000..8ba4aeba --- /dev/null +++ b/packages/microcosm-emitter-interface/src/microcosm_emitter_interface/fixtures/protocol-v1/host-ping.json @@ -0,0 +1,9 @@ +{ + "protocol_version": 1, + "module": "host", + "kind": "envelope", + "body": { + "action": "ping" + }, + "response": "acknowledgement" +} diff --git a/packages/microcosm-emitter-interface/src/microcosm_emitter_interface/fixtures/protocol-v1/host-query-envelope.json b/packages/microcosm-emitter-interface/src/microcosm_emitter_interface/fixtures/protocol-v1/host-query-envelope.json new file mode 100644 index 00000000..c95a4b61 --- /dev/null +++ b/packages/microcosm-emitter-interface/src/microcosm_emitter_interface/fixtures/protocol-v1/host-query-envelope.json @@ -0,0 +1,14 @@ +{ + "protocol_version": 1, + "module": "host", + "kind": "envelope", + "body": { + "action": "query", + "module": "orrery", + "message": { + "command": "receipt", + "publication_id": "fixture-publication" + } + }, + "response": "answer" +} diff --git a/packages/microcosm-emitter-interface/src/microcosm_emitter_interface/fixtures/protocol-v1/orrery-configuration.json b/packages/microcosm-emitter-interface/src/microcosm_emitter_interface/fixtures/protocol-v1/orrery-configuration.json new file mode 100644 index 00000000..b169c907 --- /dev/null +++ b/packages/microcosm-emitter-interface/src/microcosm_emitter_interface/fixtures/protocol-v1/orrery-configuration.json @@ -0,0 +1,10 @@ +{ + "protocol_version": 1, + "module": "orrery", + "kind": "configuration", + "body": { + "spool_path": "{spool_path}", + "development_collector_url": "http://127.0.0.1:1", + "publication_origin": "http://127.0.0.1:1" + } +} diff --git a/packages/microcosm-emitter-interface/src/microcosm_emitter_interface/fixtures/protocol-v1/orrery-enqueue.json b/packages/microcosm-emitter-interface/src/microcosm_emitter_interface/fixtures/protocol-v1/orrery-enqueue.json new file mode 100644 index 00000000..b113c332 --- /dev/null +++ b/packages/microcosm-emitter-interface/src/microcosm_emitter_interface/fixtures/protocol-v1/orrery-enqueue.json @@ -0,0 +1,35 @@ +{ + "protocol_version": 1, + "module": "orrery", + "kind": "query", + "sequence": 2, + "files": { + "graph.orrery.json": "{\"nodes\":[]}", + "graph.schema.json": "{\"schema\":true}" + }, + "body": { + "command": "enqueue", + "directory": "{directory}", + "inventory": { + "version": 1, + "publication_id": "fixture-publication", + "files": [ + { + "name": "graph.orrery.json", + "role": "graph", + "bytes": 12, + "sha256": "acf2fa576acb702442f9d0101673354c398db67315c066ca48be8db8e0d2c75b" + }, + { + "name": "graph.schema.json", + "role": "schema", + "bytes": 15, + "sha256": "e296bc1725afc9c1e1585549e7f3c3b0cf58acba82fc60188d8eda40b9e17a13" + } + ] + } + }, + "response_keys": [ + "receipt" + ] +} diff --git a/packages/microcosm-emitter-interface/src/microcosm_emitter_interface/fixtures/protocol-v1/orrery-inventory.json b/packages/microcosm-emitter-interface/src/microcosm_emitter_interface/fixtures/protocol-v1/orrery-inventory.json new file mode 100644 index 00000000..8162258d --- /dev/null +++ b/packages/microcosm-emitter-interface/src/microcosm_emitter_interface/fixtures/protocol-v1/orrery-inventory.json @@ -0,0 +1,22 @@ +{ + "protocol_version": 1, + "module": "orrery", + "kind": "query", + "sequence": 1, + "files": { + "graph.orrery.json": "{\"nodes\":[]}", + "graph.schema.json": "{\"schema\":true}" + }, + "body": { + "command": "inventory", + "directory": "{directory}", + "roles": { + "graph.orrery.json": "graph", + "graph.schema.json": "schema" + }, + "publication_id": "fixture-publication" + }, + "response_keys": [ + "inventory" + ] +} diff --git a/packages/microcosm-emitter-interface/src/microcosm_emitter_interface/fixtures/protocol-v1/orrery-receipt.json b/packages/microcosm-emitter-interface/src/microcosm_emitter_interface/fixtures/protocol-v1/orrery-receipt.json new file mode 100644 index 00000000..156ddcfe --- /dev/null +++ b/packages/microcosm-emitter-interface/src/microcosm_emitter_interface/fixtures/protocol-v1/orrery-receipt.json @@ -0,0 +1,13 @@ +{ + "protocol_version": 1, + "module": "orrery", + "kind": "query", + "sequence": 3, + "body": { + "command": "receipt", + "publication_id": "fixture-publication" + }, + "response_keys": [ + "receipt" + ] +} diff --git a/packages/microcosm-emitter-interface/src/microcosm_emitter_interface/fixtures/protocol-v1/orrery-retry.json b/packages/microcosm-emitter-interface/src/microcosm_emitter_interface/fixtures/protocol-v1/orrery-retry.json new file mode 100644 index 00000000..d50f65ec --- /dev/null +++ b/packages/microcosm-emitter-interface/src/microcosm_emitter_interface/fixtures/protocol-v1/orrery-retry.json @@ -0,0 +1,13 @@ +{ + "protocol_version": 1, + "module": "orrery", + "kind": "query", + "sequence": 4, + "body": { + "command": "retry", + "publication_id": "fixture-publication" + }, + "response_keys": [ + "receipt" + ] +} diff --git a/packages/microcosm-emitter-interface/src/microcosm_emitter_interface/fixtures/protocol-v1/telemetry-calibration-progress.json b/packages/microcosm-emitter-interface/src/microcosm_emitter_interface/fixtures/protocol-v1/telemetry-calibration-progress.json new file mode 100644 index 00000000..20370f21 --- /dev/null +++ b/packages/microcosm-emitter-interface/src/microcosm_emitter_interface/fixtures/protocol-v1/telemetry-calibration-progress.json @@ -0,0 +1,15 @@ +{ + "protocol_version": 1, + "module": "telemetry", + "kind": "message", + "sequence": 0, + "body": { + "command": "calibration_progress", + "event": { + "kind": "calibration_epoch", + "epoch": 10, + "epochs": 100, + "loss": 0.25 + } + } +} diff --git a/packages/microcosm-emitter-interface/src/microcosm_emitter_interface/fixtures/protocol-v1/telemetry-complete.json b/packages/microcosm-emitter-interface/src/microcosm_emitter_interface/fixtures/protocol-v1/telemetry-complete.json new file mode 100644 index 00000000..8b54fd9d --- /dev/null +++ b/packages/microcosm-emitter-interface/src/microcosm_emitter_interface/fixtures/protocol-v1/telemetry-complete.json @@ -0,0 +1,9 @@ +{ + "protocol_version": 1, + "module": "telemetry", + "kind": "message", + "sequence": 0, + "body": { + "command": "complete" + } +} diff --git a/packages/microcosm-emitter-interface/src/microcosm_emitter_interface/fixtures/protocol-v1/telemetry-configuration.json b/packages/microcosm-emitter-interface/src/microcosm_emitter_interface/fixtures/protocol-v1/telemetry-configuration.json new file mode 100644 index 00000000..d8e77408 --- /dev/null +++ b/packages/microcosm-emitter-interface/src/microcosm_emitter_interface/fixtures/protocol-v1/telemetry-configuration.json @@ -0,0 +1,22 @@ +{ + "protocol_version": 1, + "module": "telemetry", + "kind": "configuration", + "body": { + "registration": { + "run_id": "fixture-run", + "producer_id": "fixture-producer", + "country_code": "UK", + "pipeline": "fixture-pipeline", + "candidate_id": null, + "release_id": null, + "run_kind": "build" + }, + "spool_path": "{spool_path}", + "identity": { + "host": "fixture" + }, + "development_collector_url": "http://127.0.0.1:1", + "heartbeat_seconds": 60.0 + } +} diff --git a/packages/microcosm-emitter-interface/src/microcosm_emitter_interface/fixtures/protocol-v1/telemetry-emit.json b/packages/microcosm-emitter-interface/src/microcosm_emitter_interface/fixtures/protocol-v1/telemetry-emit.json new file mode 100644 index 00000000..c170c0a4 --- /dev/null +++ b/packages/microcosm-emitter-interface/src/microcosm_emitter_interface/fixtures/protocol-v1/telemetry-emit.json @@ -0,0 +1,16 @@ +{ + "protocol_version": 1, + "module": "telemetry", + "kind": "message", + "sequence": 0, + "body": { + "command": "emit", + "event_type": "progress", + "stage_id": "compile", + "status": "progress", + "message": "Compiling.", + "details": { + "step": 1 + } + } +} diff --git a/packages/microcosm-emitter-interface/src/microcosm_emitter_interface/fixtures/protocol-v1/telemetry-fail-without-code.json b/packages/microcosm-emitter-interface/src/microcosm_emitter_interface/fixtures/protocol-v1/telemetry-fail-without-code.json new file mode 100644 index 00000000..f8239852 --- /dev/null +++ b/packages/microcosm-emitter-interface/src/microcosm_emitter_interface/fixtures/protocol-v1/telemetry-fail-without-code.json @@ -0,0 +1,13 @@ +{ + "protocol_version": 1, + "module": "telemetry", + "kind": "message", + "sequence": 0, + "body": { + "command": "fail", + "message": "Synthetic failure.", + "error_type": "MemoryError", + "failed_during": null, + "failure_class": "oom" + } +} diff --git a/packages/microcosm-emitter-interface/src/microcosm_emitter_interface/fixtures/protocol-v1/telemetry-fail.json b/packages/microcosm-emitter-interface/src/microcosm_emitter_interface/fixtures/protocol-v1/telemetry-fail.json new file mode 100644 index 00000000..9a448d35 --- /dev/null +++ b/packages/microcosm-emitter-interface/src/microcosm_emitter_interface/fixtures/protocol-v1/telemetry-fail.json @@ -0,0 +1,14 @@ +{ + "protocol_version": 1, + "module": "telemetry", + "kind": "message", + "sequence": 0, + "body": { + "command": "fail", + "message": "Synthetic failure.", + "error_type": "RuntimeError", + "failed_during": "calibrating", + "failure_class": "build_failure", + "error_code": "BUILD_FAILED" + } +} diff --git a/packages/microcosm-emitter-interface/src/microcosm_emitter_interface/fixtures/protocol-v1/telemetry-progress.json b/packages/microcosm-emitter-interface/src/microcosm_emitter_interface/fixtures/protocol-v1/telemetry-progress.json new file mode 100644 index 00000000..c8f28903 --- /dev/null +++ b/packages/microcosm-emitter-interface/src/microcosm_emitter_interface/fixtures/protocol-v1/telemetry-progress.json @@ -0,0 +1,16 @@ +{ + "protocol_version": 1, + "module": "telemetry", + "kind": "message", + "sequence": 0, + "body": { + "command": "progress", + "stage_id": "calibrating", + "done": 2, + "total": 10, + "unit": "epochs", + "details": { + "phase": "solve" + } + } +} diff --git a/packages/microcosm-emitter-interface/src/microcosm_emitter_interface/fixtures/protocol-v1/telemetry-refusal.json b/packages/microcosm-emitter-interface/src/microcosm_emitter_interface/fixtures/protocol-v1/telemetry-refusal.json new file mode 100644 index 00000000..59cf8298 --- /dev/null +++ b/packages/microcosm-emitter-interface/src/microcosm_emitter_interface/fixtures/protocol-v1/telemetry-refusal.json @@ -0,0 +1,17 @@ +{ + "protocol_version": 1, + "module": "telemetry", + "kind": "message", + "sequence": 0, + "body": { + "command": "block", + "phase": "preflight", + "blocking_gate_ids": [ + "fixture-check" + ], + "blocking_failure_count": 1, + "gate_statuses": { + "fixture-check": "failed" + } + } +} diff --git a/packages/microcosm-emitter-interface/src/microcosm_emitter_interface/fixtures/protocol-v1/telemetry-stage.json b/packages/microcosm-emitter-interface/src/microcosm_emitter_interface/fixtures/protocol-v1/telemetry-stage.json new file mode 100644 index 00000000..8864d60a --- /dev/null +++ b/packages/microcosm-emitter-interface/src/microcosm_emitter_interface/fixtures/protocol-v1/telemetry-stage.json @@ -0,0 +1,15 @@ +{ + "protocol_version": 1, + "module": "telemetry", + "kind": "message", + "sequence": 0, + "body": { + "command": "stage", + "stage_id": "compile", + "status": "started", + "message": null, + "details": { + "rows": 3 + } + } +} diff --git a/packages/microcosm-emitter-interface/src/microcosm_emitter_interface/fixtures/protocol-v1/telemetry-transition-calibration-progress.json b/packages/microcosm-emitter-interface/src/microcosm_emitter_interface/fixtures/protocol-v1/telemetry-transition-calibration-progress.json new file mode 100644 index 00000000..88cecdc9 --- /dev/null +++ b/packages/microcosm-emitter-interface/src/microcosm_emitter_interface/fixtures/protocol-v1/telemetry-transition-calibration-progress.json @@ -0,0 +1,15 @@ +{ + "protocol_version": 1, + "module": "telemetry", + "kind": "message", + "sequence": 0, + "body": { + "command": "transition_calibration_progress", + "event": { + "kind": "calibration_epoch", + "epoch": 20, + "epochs": 100, + "loss": 0.125 + } + } +} diff --git a/packages/microcosm-emitter-interface/src/microcosm_emitter_interface/fixtures/protocol-v1/telemetry-transition-stage.json b/packages/microcosm-emitter-interface/src/microcosm_emitter_interface/fixtures/protocol-v1/telemetry-transition-stage.json new file mode 100644 index 00000000..1ae3bb5c --- /dev/null +++ b/packages/microcosm-emitter-interface/src/microcosm_emitter_interface/fixtures/protocol-v1/telemetry-transition-stage.json @@ -0,0 +1,13 @@ +{ + "protocol_version": 1, + "module": "telemetry", + "kind": "message", + "sequence": 0, + "body": { + "command": "transition_stage", + "stage_id": "calibrating", + "status": "running", + "message": "Calibrating.", + "details": {} + } +} diff --git a/packages/microcosm-emitter-interface/src/microcosm_emitter_interface/launcher.py b/packages/microcosm-emitter-interface/src/microcosm_emitter_interface/launcher.py new file mode 100644 index 00000000..150e44ce --- /dev/null +++ b/packages/microcosm-emitter-interface/src/microcosm_emitter_interface/launcher.py @@ -0,0 +1,273 @@ +"""Start the installed ``microcosm-emitter`` command as the build's child process. + +The child is started exactly as the emitter's in-process launcher starts +``python -m microcosm_emitter.host``: no ``env`` argument, so it inherits every +environment variable of the build (including ``HF_TOKEN``, ``HUGGINGFACE_TOKEN`` +and ``HF_TOKEN_PATH``) and runs as the same user. Its module configuration +travels on standard input, never in process-list arguments. It is never a +separately started system service, which would not see the build's environment. +""" + +from __future__ import annotations + +import json +import math +import os +import shutil +import socket +import subprocess +import tempfile +import threading +import time +from collections.abc import Mapping +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Final, Literal + +from microcosm_emitter_interface.protocol import ( + DRAIN_SECONDS, + MAX_MESSAGE_BYTES, + PROCESS_EXIT_TIMEOUT_SECONDS, + PROTOCOL_VERSION, + RUNTIME_PREFIX, + SOCKET_FILENAME, + STARTUP_POLL_SECONDS, + STARTUP_TIMEOUT_SECONDS, +) +from microcosm_emitter_interface.transport import LocalEmitterError, SocketTransport + +#: Names the emitter command when it is not on ``PATH`` (for example a +#: ``uv tool`` directory outside the build's ``PATH``). +COMMAND_ENVIRONMENT_VARIABLE: Final = "MICROCOSM_EMITTER_COMMAND" +DEFAULT_COMMAND: Final = "microcosm-emitter" + +type StartupFailureReason = Literal[ + "command_not_found", + "invalid_configuration", + "configuration_timeout", + "child_exit", + "readiness_timeout", + "unsupported_protocol_version", + "startup_failure", +] + +_STARTUP_MESSAGES: Final = { + "command_not_found": "The microcosm-emitter command was not found", + "invalid_configuration": "Emitter configuration is invalid or too large", + "configuration_timeout": "Emitter configuration transfer timed out", + "child_exit": "The emitter exited before readiness", + "readiness_timeout": "Emitter readiness timed out", + "unsupported_protocol_version": ( + f"The installed emitter does not accept protocol version {PROTOCOL_VERSION}" + ), + "startup_failure": "Emitter startup failed", +} + + +class EmitterStartupError(LocalEmitterError): + """A fixed startup reason and optional exit code, never exception details.""" + + def __init__( + self, reason: StartupFailureReason, *, exit_code: int | None = None + ) -> None: + if reason not in _STARTUP_MESSAGES: + raise ValueError("Unknown startup failure reason.") + self.reason: StartupFailureReason = reason + self.exit_code = exit_code if type(exit_code) is int else None + details = [reason] + if self.exit_code is not None: + details.append(f"exit_code={self.exit_code}") + super().__init__(f"{_STARTUP_MESSAGES[reason]} ({', '.join(details)}).") + + +def resolve_command(command: str | os.PathLike[str] | None = None) -> str: + """The emitter command: the argument, then the environment, then ``PATH``.""" + candidate = ( + str(command) + if command is not None + else os.environ.get(COMMAND_ENVIRONMENT_VARIABLE, "").strip() or DEFAULT_COMMAND + ) + resolved = shutil.which(candidate) + if resolved is None: + raise EmitterStartupError("command_not_found") + return resolved + + +@dataclass +class EmitterProcess: + """A running emitter host owned by this build process.""" + + process: subprocess.Popen + transport: SocketTransport + runtime_directory: Path + description: dict[str, Any] = field(default_factory=dict) + + def close(self) -> None: + """Ask the host to drain and stop, then reap it without blocking.""" + try: + self.transport.close() + except LocalEmitterError: + pass + finally: + threading.Thread(target=self._reap, daemon=True).start() + + def _reap(self) -> None: + try: + self.process.wait(timeout=DRAIN_SECONDS + PROCESS_EXIT_TIMEOUT_SECONDS) + except subprocess.TimeoutExpired: + _terminate(self.process) + finally: + _remove_runtime_directory(self.runtime_directory) + + +def start_emitter( + modules: Mapping[str, Mapping[str, Any]], + *, + command: str | os.PathLike[str] | None = None, + startup_timeout: float = STARTUP_TIMEOUT_SECONDS, +) -> EmitterProcess: + """Return after readiness and a protocol check, or clean up and raise.""" + directory: Path | None = None + process: subprocess.Popen | None = None + phase: StartupFailureReason = "invalid_configuration" + try: + if ( + not isinstance(startup_timeout, (int, float)) + or not math.isfinite(startup_timeout) + or startup_timeout < 0 + ): + raise EmitterStartupError("invalid_configuration") + if ( + not isinstance(modules, Mapping) + or not modules + or any( + not isinstance(name, str) or not isinstance(settings, Mapping) + for name, settings in modules.items() + ) + ): + raise EmitterStartupError("invalid_configuration") + payload = json.dumps( + {name: dict(settings) for name, settings in modules.items()}, + allow_nan=False, + ).encode() + if len(payload) > MAX_MESSAGE_BYTES: + raise EmitterStartupError("invalid_configuration") + executable = resolve_command(command) + deadline = time.monotonic() + startup_timeout + phase = "startup_failure" + # Short paths are necessary for macOS's Unix-socket path limit. + directory = Path(tempfile.mkdtemp(prefix=RUNTIME_PREFIX, dir="/tmp")) + directory.chmod(0o700) + socket_path = directory / SOCKET_FILENAME + reader, writer = socket.socketpair() + with reader, writer: + process = subprocess.Popen( + [ + executable, + "host", + "--socket", + str(socket_path), + "--parent-pid", + str(os.getpid()), + "--skip-unavailable-modules", + ], + stdin=reader, + stdout=subprocess.DEVNULL, + start_new_session=True, + ) + phase = "configuration_timeout" + remaining = deadline - time.monotonic() + if remaining <= 0: + raise EmitterStartupError("configuration_timeout") + writer.settimeout(remaining) + writer.sendall(payload) + writer.shutdown(socket.SHUT_WR) + if time.monotonic() >= deadline: + raise EmitterStartupError("configuration_timeout") + phase = "startup_failure" + transport = SocketTransport(socket_path) + while True: + exit_code = process.poll() + if exit_code is not None: + raise EmitterStartupError("child_exit", exit_code=exit_code) + remaining = deadline - time.monotonic() + if remaining <= 0: + raise EmitterStartupError("readiness_timeout") + try: + transport.ping(timeout=min(transport.timeout, remaining)) + except LocalEmitterError: + time.sleep(min(STARTUP_POLL_SECONDS, max(0, remaining))) + continue + break + description = _accepted_description(transport, deadline) + return EmitterProcess(process, transport, directory, description) + except Exception as error: + # Only transfer timeouts are classified as such; an unrelated writer + # failure is a startup failure rather than evidence of deadline expiry. + if isinstance(error, EmitterStartupError): + failure = EmitterStartupError(error.reason, exit_code=error.exit_code) + elif phase == "configuration_timeout" and isinstance(error, TimeoutError): + failure = EmitterStartupError("configuration_timeout") + else: + failure = EmitterStartupError( + phase if phase != "configuration_timeout" else "startup_failure" + ) + try: + try: + if process is not None: + _terminate(process) + finally: + if directory is not None: + _remove_runtime_directory(directory) + except Exception: + failure = EmitterStartupError("startup_failure") + # Raise outside the handler so neither a cause nor an implicit context + # can carry private configuration or exception details. + raise failure from None + + +def _accepted_description( + transport: SocketTransport, deadline: float +) -> dict[str, Any]: + remaining = deadline - time.monotonic() + if remaining <= 0: + raise EmitterStartupError("readiness_timeout") + try: + description = transport.hello(timeout=remaining) + except LocalEmitterError: + # An emitter without a ``hello`` answer predates this protocol. + raise EmitterStartupError("unsupported_protocol_version") from None + versions = description.get("protocol_versions") + modules = description.get("modules") + # Every call reads the module names as a set of strings, so a later + # emitter's differently shaped answer is refused here, at startup. + if ( + not isinstance(versions, list) + or not all(type(version) is int for version in versions) + or PROTOCOL_VERSION not in versions + or not isinstance(modules, list) + or not all(isinstance(module, str) for module in modules) + ): + raise EmitterStartupError("unsupported_protocol_version") + return description + + +def _terminate(process: subprocess.Popen) -> None: + try: + if process.poll() is None: + process.terminate() + process.wait(timeout=PROCESS_EXIT_TIMEOUT_SECONDS) + except subprocess.TimeoutExpired: + process.kill() + process.wait(timeout=PROCESS_EXIT_TIMEOUT_SECONDS) + + +def _remove_runtime_directory(directory: Path) -> None: + (directory / SOCKET_FILENAME).unlink(missing_ok=True) + try: + directory.rmdir() + except FileNotFoundError: + # The child removes its own directory on a normal or failed exit. + pass + except OSError: + pass diff --git a/packages/microcosm-emitter-interface/src/microcosm_emitter_interface/orrery.py b/packages/microcosm-emitter-interface/src/microcosm_emitter_interface/orrery.py new file mode 100644 index 00000000..be80f26c --- /dev/null +++ b/packages/microcosm-emitter-interface/src/microcosm_emitter_interface/orrery.py @@ -0,0 +1,156 @@ +"""Typed Orrery input hand-over: the emitter's existing graph publication queue. + +The emitter hashes the selected preserved files, durably queues them and +delivers them; this package only names the files and their roles. The shapes +equal ``microcosm_emitter.orrery.contracts``. +""" + +from __future__ import annotations + +import os +from collections.abc import Mapping +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Literal, NotRequired, TypedDict, cast + +type PublicationFileRole = Literal[ + "graph", + "schema", + "index", + "declaration", + "manifest", + "binding", + "summaries", + "upstream", +] + + +class PublicationFile(TypedDict): + """One preserved file, identified by its exact bytes and allowed role.""" + + name: str + role: PublicationFileRole + bytes: int + sha256: str + + +class PublicationInventory(TypedDict): + """The versioned file inventory the emitter computed and will deliver.""" + + version: Literal[1] + publication_id: str + files: list[PublicationFile] + + +class PublicationReceipt(TypedDict): + """The emitter's latest delivery result for one publication.""" + + version: Literal[1] + publication_id: str | None + status: Literal["pending", "published", "skipped", "failed", "no_evidence"] + error_code: str | None + url: NotRequired[str | None] + attempts: NotRequired[int] + inventory_sha256: NotRequired[str] + preserved_directory: NotRequired[str] + recorded_execution: NotRequired[bool] + + +@dataclass(frozen=True) +class OrrerySettings: + """Configuration for the emitter's Orrery module; defaults are production.""" + + development_collector_url: str | None = None + publication_origin: str | None = None + + def configuration(self, spool_path: Path) -> dict[str, Any]: + configuration: dict[str, Any] = {"spool_path": str(spool_path)} + if self.development_collector_url is not None: + configuration["development_collector_url"] = self.development_collector_url + if self.publication_origin is not None: + configuration["publication_origin"] = self.publication_origin + return configuration + + +def unavailable_receipt( + publication_id: str | None, *, error_code: str = "local_service_unavailable" +) -> PublicationReceipt: + """No acknowledgement: no claim that the emitter queued anything. + + ``error_code`` is ``local_service_unavailable`` when no emitter answered, + or ``invalid_arguments`` when the call's arguments could not be sent. + """ + return { + "version": 1, + "publication_id": publication_id, + "status": "failed", + "error_code": error_code, + } + + +# The Orrery module's requests. Both this package's client and the emitter's +# in-process client build them here, inside their error handling, so the two +# send the same messages and neither raises on an unconvertible argument. + + +def inventory_message( + directory: str | os.PathLike[str], + roles: Mapping[str, PublicationFileRole], + publication_id: str | None, +) -> dict[str, Any]: + return { + "command": "inventory", + "directory": str(Path(directory).resolve()), + "roles": dict(roles), + "publication_id": publication_id, + } + + +def enqueue_message( + directory: str | os.PathLike[str], inventory: PublicationInventory +) -> dict[str, Any]: + return { + "command": "enqueue", + "directory": str(Path(directory).resolve()), + "inventory": inventory, + } + + +def receipt_message(publication_id: str) -> dict[str, Any]: + return {"command": "receipt", "publication_id": publication_id} + + +def retry_message(publication_id: str) -> dict[str, Any]: + return {"command": "retry", "publication_id": publication_id} + + +def publication_id_of(inventory: Any) -> str | None: + """The inventory's publication identifier, or None if it has none.""" + try: + value = inventory["publication_id"] + except Exception: + return None + return value if isinstance(value, str) else None + + +def answered_inventory(answer: Mapping[str, Any] | None) -> PublicationInventory | None: + """The inventory in the module's answer, if it holds one.""" + value = None if answer is None else answer.get("inventory") + return cast(PublicationInventory, value) if isinstance(value, dict) else None + + +def answered_receipt( + answer: Mapping[str, Any] | None, + publication_id: str | None, + *, + error_code: str = "local_service_unavailable", +) -> PublicationReceipt: + """The receipt in the module's answer, or an unavailable receipt. + + An answer whose receipt has no status is not a receipt: the build reads + ``status`` from every receipt it gets. + """ + value = None if answer is None else answer.get("receipt") + if isinstance(value, dict) and isinstance(value.get("status"), str): + return cast(PublicationReceipt, value) + return unavailable_receipt(publication_id, error_code=error_code) diff --git a/packages/microcosm-emitter-interface/src/microcosm_emitter_interface/protocol.py b/packages/microcosm-emitter-interface/src/microcosm_emitter_interface/protocol.py new file mode 100644 index 00000000..532cdc03 --- /dev/null +++ b/packages/microcosm-emitter-interface/src/microcosm_emitter_interface/protocol.py @@ -0,0 +1,32 @@ +"""The socket protocol between a build and its separately installed emitter. + +``PROTOCOL_VERSION`` names the shape of every message this package sends. An +emitter lists the versions it accepts in its ``hello`` response; this package +refuses an emitter that does not list its version. Adding a module or a message +kind keeps the version. Changing an existing message's shape needs a new version, +and every emitter release keeps accepting every version still in use. +""" + +from typing import Final, Literal + +PROTOCOL_VERSION: Final = 1 + +DELIMITER: Final = b"\n" +ACK_OK: Final = b"ok\n" +ACK_ERROR: Final = b"error\n" +MAX_MESSAGE_BYTES: Final = 1_048_576 + +TELEMETRY_MODULE: Final = "telemetry" +ORRERY_MODULE: Final = "orrery" +type ModuleName = Literal["telemetry", "orrery"] + +#: Telemetry requests only wait for the service to commit their events. +ACKNOWLEDGEMENT_TIMEOUT_SECONDS: Final = 0.2 +#: Orrery requests hash a bounded set of preserved files before answering. +FILE_REQUEST_TIMEOUT_SECONDS: Final = 30.0 +STARTUP_TIMEOUT_SECONDS: Final = 15.0 +STARTUP_POLL_SECONDS: Final = 0.02 +DRAIN_SECONDS: Final = 15.0 +PROCESS_EXIT_TIMEOUT_SECONDS: Final = 1.0 +RUNTIME_PREFIX: Final = "pe-service-" +SOCKET_FILENAME: Final = "service.sock" diff --git a/packages/microcosm-emitter-interface/src/microcosm_emitter_interface/protocol_fixtures.py b/packages/microcosm-emitter-interface/src/microcosm_emitter_interface/protocol_fixtures.py new file mode 100644 index 00000000..e56ac84f --- /dev/null +++ b/packages/microcosm-emitter-interface/src/microcosm_emitter_interface/protocol_fixtures.py @@ -0,0 +1,140 @@ +"""Recorded protocol messages, shipped so every emitter release can test them. + +``fixtures/protocol-v/`` holds one JSON file per message this package sends +under protocol version ``N``. The interface's tests check that each typed call +sends exactly its fixture; the emitter's contract tests check that it accepts +every fixture of every version it supports. A released fixture never changes: +a new message shape gets a new fixture, and an incompatible one a new version. + +Each file holds: + +- ``protocol_version`` and ``module`` (``telemetry``, ``orrery`` or ``host``); +- ``kind``: + - ``message`` (acknowledged) or ``query`` (answered): a module's message; + - ``configuration``: a module's startup settings; + - ``hello``: the request for the host's description; + - ``launch``: how the command is started, as ``arguments`` after the command + and the ``standard_input`` object (module name to its configuration); + - ``envelope``: one whole socket request, as the host receives it; +- ``sequence``: the order within a module, for requests that build on earlier + ones; +- ``body``: the message, with the placeholders ``{directory}``, + ``{spool_path}``, ``{socket_path}`` and ``{parent_pid}`` in string values; +- ``files``: text files to create in ``{directory}`` before sending; +- ``response_keys``: the keys an answer must contain (at least); +- ``response``: for an envelope, ``acknowledgement`` (``ok``) or ``answer`` + (a JSON object). +""" + +from __future__ import annotations + +import hashlib +import json +from dataclasses import dataclass, field +from importlib.resources import files +from importlib.resources.abc import Traversable +from pathlib import Path +from typing import Any, Literal + +FIXTURE_ROOT = "fixtures" +DIRECTORY_PLACEHOLDER = "{directory}" +SPOOL_PATH_PLACEHOLDER = "{spool_path}" +SOCKET_PATH_PLACEHOLDER = "{socket_path}" +PARENT_PID_PLACEHOLDER = "{parent_pid}" + +type FixtureKind = Literal[ + "message", "query", "configuration", "hello", "launch", "envelope" +] +type EnvelopeResponse = Literal["acknowledgement", "answer"] + + +@dataclass(frozen=True) +class ProtocolFixture: + name: str + path: str + protocol_version: int + module: str + kind: FixtureKind + body: dict[str, Any] + sequence: int = 0 + files: dict[str, str] = field(default_factory=dict) + response_keys: tuple[str, ...] = () + response: EnvelopeResponse | None = None + + +def _fixture_files() -> list[tuple[str, Traversable]]: + root = files("microcosm_emitter_interface").joinpath(FIXTURE_ROOT) + found = [] + for version_directory in root.iterdir(): + if not version_directory.is_dir() or not version_directory.name.startswith( + "protocol-v" + ): + continue + for entry in version_directory.iterdir(): + if entry.name.endswith(".json"): + found.append((f"{version_directory.name}/{entry.name}", entry)) + return sorted(found, key=lambda item: item[0]) + + +def protocol_fixtures(version: int | None = None) -> list[ProtocolFixture]: + """Every shipped fixture, or those of one protocol version.""" + fixtures = [] + for relative_path, entry in _fixture_files(): + document = json.loads(entry.read_text(encoding="utf-8")) + fixture = ProtocolFixture( + name=entry.name.removesuffix(".json"), + path=relative_path, + protocol_version=document["protocol_version"], + module=document["module"], + kind=document["kind"], + body=document["body"], + sequence=document.get("sequence", 0), + files=dict(document.get("files", {})), + response_keys=tuple(document.get("response_keys", ())), + response=document.get("response"), + ) + if relative_path.split("/", 1)[0] != f"protocol-v{fixture.protocol_version}": + raise ValueError(f"{relative_path} is filed under the wrong version.") + if version is None or fixture.protocol_version == version: + fixtures.append(fixture) + return fixtures + + +def fixture_digests() -> dict[str, str]: + """SHA-256 of every fixture file, keyed by its path below ``fixtures/``.""" + return { + relative_path: hashlib.sha256(entry.read_bytes()).hexdigest() + for relative_path, entry in _fixture_files() + } + + +def materialize( + fixture: ProtocolFixture, + *, + directory: Path, + spool_path: Path, + socket_path: Path | None = None, + parent_pid: int | None = None, +) -> dict[str, Any]: + """The fixture's body with its placeholders replaced.""" + replacements = { + DIRECTORY_PLACEHOLDER: str(directory), + SPOOL_PATH_PLACEHOLDER: str(spool_path), + } + if socket_path is not None: + replacements[SOCKET_PATH_PLACEHOLDER] = str(socket_path) + if parent_pid is not None: + replacements[PARENT_PID_PLACEHOLDER] = str(parent_pid) + + def replace(value: Any) -> Any: + if isinstance(value, str): + for placeholder, replacement in replacements.items(): + value = value.replace(placeholder, replacement) + return value + if isinstance(value, dict): + return {key: replace(item) for key, item in value.items()} + if isinstance(value, list): + return [replace(item) for item in value] + return value + + return replace(fixture.body) diff --git a/packages/microcosm-emitter-interface/src/microcosm_emitter_interface/py.typed b/packages/microcosm-emitter-interface/src/microcosm_emitter_interface/py.typed new file mode 100644 index 00000000..e69de29b diff --git a/packages/microcosm-emitter-interface/src/microcosm_emitter_interface/serialization.py b/packages/microcosm-emitter-interface/src/microcosm_emitter_interface/serialization.py new file mode 100644 index 00000000..a61c98af --- /dev/null +++ b/packages/microcosm-emitter-interface/src/microcosm_emitter_interface/serialization.py @@ -0,0 +1,28 @@ +"""Convert Python call arguments to JSON without interpreting them.""" + +import math +from collections.abc import Mapping +from pathlib import Path +from typing import Any + + +def wire_value(value: Any) -> Any: + """Preserve values where possible, including scientific-library scalars. + + The same conversion as ``microcosm_emitter.telemetry.serialization``, so + the emitter receives the messages its own in-process client sends. + """ + if isinstance(value, Mapping): + return {str(key): wire_value(item) for key, item in value.items()} + if isinstance(value, (list, tuple)): + return [wire_value(item) for item in value] + if isinstance(value, Path): + return str(value) + if isinstance(value, float): + return value if math.isfinite(value) else None + if isinstance(value, (str, int, bool)) or value is None: + return value + item = getattr(value, "item", None) + if callable(item): + return wire_value(item()) + return str(value) diff --git a/packages/microcosm-emitter-interface/src/microcosm_emitter_interface/spool.py b/packages/microcosm-emitter-interface/src/microcosm_emitter_interface/spool.py new file mode 100644 index 00000000..9d03d43e --- /dev/null +++ b/packages/microcosm-emitter-interface/src/microcosm_emitter_interface/spool.py @@ -0,0 +1,11 @@ +"""Where the emitter keeps its durable queue.""" + +import os +from pathlib import Path + + +def default_spool_path() -> Path: + """The emitter's existing queue location, so queued work survives upgrades.""" + configured = os.environ.get("XDG_CACHE_HOME", "").strip() + root = Path(configured).expanduser() if configured else Path.home() / ".cache" + return root / "microcosm" / "telemetry" / "events.sqlite3" diff --git a/packages/microcosm-emitter-interface/src/microcosm_emitter_interface/telemetry.py b/packages/microcosm-emitter-interface/src/microcosm_emitter_interface/telemetry.py new file mode 100644 index 00000000..a1717c23 --- /dev/null +++ b/packages/microcosm-emitter-interface/src/microcosm_emitter_interface/telemetry.py @@ -0,0 +1,351 @@ +"""Typed telemetry events, sent unchanged to the emitter's telemetry module. + +Each command below becomes exactly the message the emitter's own in-process +client (``microcosm_emitter.telemetry.client``) sends, so a build's telemetry +does not change when it moves to this interface. The emitter's service, not +this package, timestamps, sanitizes, persists and delivers the events. +""" + +from __future__ import annotations + +import uuid +from collections.abc import Mapping, Sequence +from dataclasses import dataclass, field +from pathlib import Path +from typing import TYPE_CHECKING, Any, Literal + +from microcosm_emitter_interface.serialization import wire_value + +if TYPE_CHECKING: + from microcosm_emitter_interface.client import EmitterClient + +type TelemetryEventType = Literal[ + "run", "stage", "progress", "calibration", "heartbeat" +] +type TelemetryStatus = Literal["started", "progress", "completed", "failed", "blocked"] + +DEFAULT_HEARTBEAT_SECONDS = 60.0 +DEFAULT_FAILURE_CLASS = "build_failure" + + +@dataclass(frozen=True) +class TelemetryRun: + """Identity registered with the hosted collector for one build.""" + + run_id: str + country_code: str + pipeline: str + candidate_id: str | None = None + release_id: str | None = None + run_kind: str = "build" + producer_id: str = field(default_factory=lambda: uuid.uuid4().hex) + + def as_registration(self) -> dict[str, Any]: + return { + "run_id": self.run_id, + "producer_id": self.producer_id, + "country_code": self.country_code, + "pipeline": self.pipeline, + "candidate_id": self.candidate_id, + "release_id": self.release_id, + "run_kind": self.run_kind, + } + + +@dataclass(frozen=True) +class TelemetrySettings: + """Configuration for the emitter's telemetry module for one build.""" + + run: TelemetryRun + identity: Mapping[str, Any] = field(default_factory=dict) + development_collector_url: str | None = None + heartbeat_seconds: float = DEFAULT_HEARTBEAT_SECONDS + + def configuration(self, spool_path: Path) -> dict[str, Any]: + return wire_value( + { + "registration": self.run.as_registration(), + "spool_path": str(spool_path), + "identity": dict(self.identity), + "development_collector_url": self.development_collector_url, + "heartbeat_seconds": self.heartbeat_seconds, + } + ) + + +@dataclass(frozen=True) +class EventRecord: + """Any event, with its type and status chosen by the caller.""" + + event_type: TelemetryEventType + status: TelemetryStatus + stage_id: str | None = None + message: str | None = None + details: Mapping[str, Any] = field(default_factory=dict) + + +@dataclass(frozen=True) +class StageEvent: + """One stage event; the service does not finish the previous stage.""" + + stage_id: str + status: TelemetryStatus = "started" + message: str | None = None + details: Mapping[str, Any] = field(default_factory=dict) + + +@dataclass(frozen=True) +class StageTransition: + """Finish the current stage, if any, and start ``stage_id``.""" + + stage_id: str + status: str = "running" + message: str | None = None + details: Mapping[str, Any] = field(default_factory=dict) + + +@dataclass(frozen=True) +class StageProgress: + stage_id: str + done: int + total: int + unit: str | None = None + details: Mapping[str, Any] = field(default_factory=dict) + + +@dataclass(frozen=True) +class CalibrationProgress: + """One calibration epoch; ``transition`` also moves to the calibrating stage.""" + + event: Mapping[str, Any] + transition: bool = False + + +@dataclass(frozen=True) +class BuildFailure: + error_type: str + message: str | None = None + failed_during: str | None = None + failure_class: str = DEFAULT_FAILURE_CLASS + error_code: str | None = None + + +@dataclass(frozen=True) +class BuildRefused: + """The build's validation checks refused its candidate at ``phase``. + + On the wire these are the telemetry protocol's existing fields + ``blocking_gate_ids``, ``blocking_failure_count`` and ``gate_statuses``: + the identifiers of the refusing checks, how many failed, and each check's + status. + """ + + phase: str + refusing_check_ids: Sequence[str] + refusing_check_count: int | None = None + check_statuses: Mapping[str, str] | None = None + + +@dataclass(frozen=True) +class BuildCompleted: + pass + + +type TelemetryCommand = ( + EventRecord + | StageEvent + | StageTransition + | StageProgress + | CalibrationProgress + | BuildFailure + | BuildRefused + | BuildCompleted +) + + +def telemetry_message(command: TelemetryCommand) -> dict[str, Any]: + """The telemetry module's request for one typed command.""" + match command: + case EventRecord(): + message: dict[str, Any] = { + "command": "emit", + "event_type": command.event_type, + "stage_id": command.stage_id, + "status": command.status, + "message": command.message, + "details": dict(command.details), + } + case StageEvent(): + message = { + "command": "stage", + "stage_id": command.stage_id, + "status": command.status, + "message": command.message, + "details": dict(command.details), + } + case StageTransition(): + message = { + "command": "transition_stage", + "stage_id": command.stage_id, + "status": command.status, + "message": command.message, + "details": dict(command.details), + } + case StageProgress(): + message = { + "command": "progress", + "stage_id": command.stage_id, + "done": command.done, + "total": command.total, + "unit": command.unit, + "details": dict(command.details), + } + case CalibrationProgress(): + message = { + "command": ( + "transition_calibration_progress" + if command.transition + else "calibration_progress" + ), + "event": dict(command.event), + } + case BuildFailure(): + message = { + "command": "fail", + "message": command.message, + "error_type": command.error_type, + "failed_during": command.failed_during, + "failure_class": command.failure_class, + } + if command.error_code is not None: + message["error_code"] = command.error_code + case BuildRefused(): + message = { + "command": "block", + "phase": command.phase, + "blocking_gate_ids": list(command.refusing_check_ids), + "blocking_failure_count": command.refusing_check_count, + "gate_statuses": ( + None + if command.check_statuses is None + else dict(command.check_statuses) + ), + } + case BuildCompleted(): + message = {"command": "complete"} + case _: + raise TypeError(f"Unsupported telemetry command {type(command).__name__}.") + return wire_value(message) + + +class TelemetryRecorder: + """The telemetry calls a build makes, each one ``emit_telemetry`` request. + + Finishing the run (``complete``, ``fail`` or ``refuse``) does not stop the + emitter: other hand-overs may follow. The build closes the client. + """ + + def __init__(self, client: EmitterClient) -> None: + self._client = client + + def emit( + self, + *, + event_type: TelemetryEventType, + status: TelemetryStatus, + stage_id: str | None = None, + message: str | None = None, + details: Mapping[str, Any] | None = None, + ) -> bool: + return self._client.emit_telemetry( + EventRecord( + event_type, + status, + stage_id, + message, + {} if details is None else details, + ) + ) + + def stage( + self, + stage_id: str, + *, + status: TelemetryStatus = "started", + message: str | None = None, + **details: Any, + ) -> bool: + return self._client.emit_telemetry( + StageEvent(stage_id, status, message, details) + ) + + def transition_stage( + self, + stage_id: str, + *, + status: str = "running", + message: str | None = None, + **details: Any, + ) -> bool: + return self._client.emit_telemetry( + StageTransition(stage_id, status, message, details) + ) + + def progress( + self, + stage_id: str, + *, + done: int, + total: int, + unit: str | None = None, + **details: Any, + ) -> bool: + return self._client.emit_telemetry( + StageProgress(stage_id, done, total, unit, details) + ) + + def calibration_progress(self, event: Mapping[str, Any]) -> bool: + return self._client.emit_telemetry(CalibrationProgress(event)) + + def transition_calibration_progress(self, event: Mapping[str, Any]) -> bool: + return self._client.emit_telemetry(CalibrationProgress(event, transition=True)) + + def fail( + self, + error: BaseException, + *, + failed_during: str | None = None, + failure_class: str = DEFAULT_FAILURE_CLASS, + error_code: str | None = None, + ) -> bool: + try: + text = str(error) + except Exception: + text = None + return self._client.emit_telemetry( + BuildFailure( + error_type=type(error).__name__, + message=text, + failed_during=failed_during, + failure_class=failure_class, + error_code=error_code, + ) + ) + + def refuse( + self, + *, + phase: str, + refusing_check_ids: Sequence[str], + refusing_check_count: int | None = None, + check_statuses: Mapping[str, str] | None = None, + ) -> bool: + """Record that the build's validation checks refused its candidate.""" + return self._client.emit_telemetry( + BuildRefused( + phase, refusing_check_ids, refusing_check_count, check_statuses + ) + ) + + def complete(self) -> bool: + return self._client.emit_telemetry(BuildCompleted()) diff --git a/packages/microcosm-emitter-interface/src/microcosm_emitter_interface/transport.py b/packages/microcosm-emitter-interface/src/microcosm_emitter_interface/transport.py new file mode 100644 index 00000000..08b643dd --- /dev/null +++ b/packages/microcosm-emitter-interface/src/microcosm_emitter_interface/transport.py @@ -0,0 +1,139 @@ +"""Bounded requests over the emitter's private Unix socket.""" + +from __future__ import annotations + +import json +import socket +import time +from collections.abc import Mapping +from pathlib import Path +from typing import Any + +from microcosm_emitter_interface.protocol import ( + ACK_OK, + ACKNOWLEDGEMENT_TIMEOUT_SECONDS, + DELIMITER, + MAX_MESSAGE_BYTES, +) + + +class LocalEmitterError(RuntimeError): + """The local emitter could not start, accept a request, or answer it.""" + + +class SocketTransport: + """One connection per request; each step shares the request's time limit.""" + + def __init__( + self, + socket_path: Path, + *, + timeout: float = ACKNOWLEDGEMENT_TIMEOUT_SECONDS, + ) -> None: + self.socket_path = Path(socket_path) + self.timeout = timeout + self.closed = False + + def _exchange( + self, + payload: Mapping[str, Any], + *, + timeout: float | None = None, + deadline: float | None = None, + ) -> bytes: + try: + data = ( + json.dumps(payload, separators=(",", ":"), allow_nan=False).encode() + + DELIMITER + ) + if len(data) > MAX_MESSAGE_BYTES: + raise ValueError("local message exceeds size limit") + with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as peer: + + def bound_next_step() -> None: + remaining = self.timeout if timeout is None else timeout + if deadline is not None: + remaining = min(remaining, deadline - time.monotonic()) + if remaining <= 0: + raise TimeoutError() + peer.settimeout(remaining) + + bound_next_step() + peer.connect(str(self.socket_path)) + bound_next_step() + peer.sendall(data) + response = bytearray() + while DELIMITER not in response: + bound_next_step() + part = peer.recv(min(65_536, MAX_MESSAGE_BYTES + 1 - len(response))) + if not part: + break + response.extend(part) + if len(response) > MAX_MESSAGE_BYTES: + raise ValueError("local response exceeds size limit") + if not response.endswith(DELIMITER): + raise ValueError("incomplete response") + return bytes(response) + except (OSError, ValueError, TypeError) as error: + raise LocalEmitterError(type(error).__name__) from error + + def _acknowledged(self, payload: Mapping[str, Any], **limits: Any) -> None: + if self._exchange(payload, **limits) != ACK_OK: + raise LocalEmitterError("emitter refused the request") + + def _answered(self, payload: Mapping[str, Any], **limits: Any) -> dict[str, Any]: + response = self._exchange(payload, **limits) + try: + result = json.loads(response.removesuffix(DELIMITER)) + except ValueError as error: + raise LocalEmitterError("emitter refused the request") from error + if not isinstance(result, dict): + raise LocalEmitterError("emitter refused the request") + return result + + def ping(self, *, timeout: float | None = None) -> None: + """Bound a readiness probe across connection, send and all reads.""" + self._acknowledged( + {"action": "ping"}, + timeout=timeout, + deadline=None if timeout is None else time.monotonic() + timeout, + ) + + def hello(self, *, timeout: float | None = None) -> dict[str, Any]: + """The emitter version, accepted protocol versions and loaded modules.""" + return self._answered( + {"action": "hello"}, + timeout=timeout, + deadline=None if timeout is None else time.monotonic() + timeout, + ) + + def send( + self, module: str, message: Mapping[str, Any], *, timeout: float | None = None + ) -> None: + """Return only after the module has handled and committed the message.""" + if self.closed: + raise LocalEmitterError("transport is closed") + self._acknowledged( + {"action": "message", "module": module, "message": dict(message)}, + timeout=timeout, + ) + + def query( + self, module: str, message: Mapping[str, Any], *, timeout: float | None = None + ) -> dict[str, Any]: + """Return the module's answer after its request has completed.""" + if self.closed: + raise LocalEmitterError("transport is closed") + return self._answered( + {"action": "query", "module": module, "message": dict(message)}, + timeout=timeout, + ) + + def close(self) -> None: + """Ask the host to drain and stop; repeated calls do nothing.""" + if self.closed: + return + try: + self._acknowledged({"action": "close"}) + finally: + self.closed = True diff --git a/pyproject.toml b/pyproject.toml index b157a95b..4d71eb1d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,6 +11,7 @@ dependencies = [ "huggingface-hub>=0.20,<2", "psutil>=6,<8", "httpx>=0.27,<1", + "microcosm-emitter-interface==0.4.0", ] [project.urls] @@ -31,6 +32,12 @@ build-backend = "hatchling.build" [tool.hatch.build.targets.wheel] packages = ["src/microcosm_emitter"] +[tool.uv.workspace] +members = ["packages/microcosm-emitter-interface"] + +[tool.uv.sources] +microcosm-emitter-interface = { workspace = true } + [dependency-groups] dev = ["pytest>=8,<9", "ruff>=0.11,<1", "build>=1,<2", "twine>=7,<8", "pyyaml>=6,<7"] @@ -47,6 +54,9 @@ target-version = "py313" select = ["E", "F", "I", "N", "W", "UP", "B"] ignore = ["E501"] +[tool.ruff.lint.isort] +known-first-party = ["microcosm_emitter", "microcosm_emitter_interface"] + [tool.towncrier] directory = "changelog.d" filename = "CHANGELOG.md" diff --git a/src/microcosm_emitter/orrery/client.py b/src/microcosm_emitter/orrery/client.py index f4fc5c3c..744f9d62 100644 --- a/src/microcosm_emitter/orrery/client.py +++ b/src/microcosm_emitter/orrery/client.py @@ -1,12 +1,18 @@ -"""Thin graph requests; hashing, persistence, auth and uploads stay in the service.""" +"""Thin graph requests; hashing, persistence, auth and uploads stay in the service. + +The messages, receipts and spool location come from +``microcosm_emitter_interface``, so this in-process client and the interface +package send the same requests. Each message is built inside the error +handling: no call raises, and an argument that cannot be sent gives an +unavailable receipt with the error code ``invalid_arguments``. +""" from __future__ import annotations -import os import time -from collections.abc import Mapping +from collections.abc import Callable, Mapping from pathlib import Path -from typing import Protocol, cast +from typing import Protocol from microcosm_emitter.host.constants import STARTUP_TIMEOUT from microcosm_emitter.host.contracts import JsonObject @@ -22,6 +28,22 @@ PublicationInventory, PublicationReceipt, ) +from microcosm_emitter_interface.orrery import ( + answered_inventory, + answered_receipt, + enqueue_message, + inventory_message, + publication_id_of, + receipt_message, + retry_message, + unavailable_receipt, +) +from microcosm_emitter_interface.spool import ( + default_spool_path as default_spool_path, +) + +INVALID_ARGUMENTS = "invalid_arguments" +UNAVAILABLE = "local_service_unavailable" class GraphTransport(Protocol): @@ -34,22 +56,6 @@ def request( def close(self) -> None: ... -def default_spool_path() -> Path: - """Keep the existing queue location so versioned jobs survive extraction.""" - configured = os.environ.get("XDG_CACHE_HOME", "").strip() - root = Path(configured).expanduser() if configured else Path.home() / ".cache" - return root / "microcosm" / "telemetry" / "events.sqlite3" - - -def _unavailable(publication_id: str | None) -> PublicationReceipt: - return { - "version": 1, - "publication_id": publication_id, - "status": "failed", - "error_code": "local_service_unavailable", - } - - class GraphPublicationClient: """Send metadata to a separately running graph publisher over its private socket.""" @@ -87,15 +93,29 @@ def start( client.startup_error = safe_startup_error(error) return client - def _request(self, message: JsonObject) -> JsonObject | None: + def _request( + self, build_message: Callable[[], JsonObject] + ) -> tuple[JsonObject | None, str]: + """The module's answer, or None and the receipt error code to report.""" if self.transport is None: - return None + return None, UNAVAILABLE try: - return self.transport.request( - message, timeout=GRAPH_REQUEST_TIMEOUT_SECONDS + message = dict(build_message()) + except Exception: + return None, INVALID_ARGUMENTS + try: + return ( + self.transport.request(message, timeout=GRAPH_REQUEST_TIMEOUT_SECONDS), + UNAVAILABLE, ) - except (LocalServiceError, OSError, ValueError, TypeError): - return None + except Exception: + return None, UNAVAILABLE + + def _receipt( + self, build_message: Callable[[], JsonObject], publication_id: str | None + ) -> PublicationReceipt: + answer, error_code = self._request(build_message) + return answered_receipt(answer, publication_id, error_code=error_code) def inventory( self, @@ -104,55 +124,28 @@ def inventory( *, publication_id: str | None = None, ) -> PublicationInventory | None: - response = self._request( - { - "command": "inventory", - "directory": str(directory.resolve()), - "roles": dict(roles), - "publication_id": publication_id, - } + answer, _ = self._request( + lambda: inventory_message(directory, roles, publication_id) ) - value = response.get("inventory") if response is not None else None - return cast(PublicationInventory, value) if isinstance(value, dict) else None + return answered_inventory(answer) def enqueue( self, directory: Path, inventory: PublicationInventory ) -> PublicationReceipt: - response = self._request( - { - "command": "enqueue", - "directory": str(directory.resolve()), - "inventory": inventory, - } - ) - value = response.get("receipt") if response is not None else None - return ( - cast(PublicationReceipt, value) - if isinstance(value, dict) - else _unavailable(inventory["publication_id"]) + publication_id = publication_id_of(inventory) + if publication_id is None: + return unavailable_receipt(None, error_code=INVALID_ARGUMENTS) + return self._receipt( + lambda: enqueue_message(directory, inventory), publication_id ) def receipt(self, publication_id: str) -> PublicationReceipt: - response = self._request( - {"command": "receipt", "publication_id": publication_id} - ) - value = response.get("receipt") if response is not None else None - return ( - cast(PublicationReceipt, value) - if isinstance(value, dict) - else _unavailable(publication_id) - ) + return self._receipt(lambda: receipt_message(publication_id), publication_id) def retry( self, publication_id: str, *, wait_seconds: float = 0 ) -> PublicationReceipt: - response = self._request({"command": "retry", "publication_id": publication_id}) - value = response.get("receipt") if response is not None else None - receipt = ( - cast(PublicationReceipt, value) - if isinstance(value, dict) - else _unavailable(publication_id) - ) + receipt = self._receipt(lambda: retry_message(publication_id), publication_id) previous_attempts = receipt.get("attempts", 0) deadline = time.monotonic() + max(0, wait_seconds) while ( diff --git a/src/microcosm_emitter/orrery/contracts.py b/src/microcosm_emitter/orrery/contracts.py index 94f4452f..32f06e48 100644 --- a/src/microcosm_emitter/orrery/contracts.py +++ b/src/microcosm_emitter/orrery/contracts.py @@ -1,50 +1,23 @@ -"""Explicit data contracts for immutable graph publication.""" +"""Explicit data contracts for immutable graph publication. + +The request and receipt shapes are the interface package's, so the emitter's +service, its in-process client and ``microcosm_emitter_interface`` share one +definition. +""" from dataclasses import dataclass from pathlib import Path -from typing import Literal, NotRequired, TypedDict - -type PublicationFileRole = Literal[ - "graph", - "schema", - "index", - "declaration", - "manifest", - "binding", - "summaries", - "upstream", -] - - -class PublicationFile(TypedDict): - """One public file, identified by its exact bytes and allowed role.""" - - name: str - role: PublicationFileRole - bytes: int - sha256: str - - -class PublicationInventory(TypedDict): - """The versioned file inventory submitted to the publication API.""" - - version: Literal[1] - publication_id: str - files: list[PublicationFile] - - -class PublicationReceipt(TypedDict): - """Local publication result, including retry and build-snapshot fields.""" - version: Literal[1] - publication_id: str | None - status: Literal["pending", "published", "skipped", "failed", "no_evidence"] - error_code: str | None - url: NotRequired[str | None] - attempts: NotRequired[int] - inventory_sha256: NotRequired[str] - preserved_directory: NotRequired[str] - recorded_execution: NotRequired[bool] +from microcosm_emitter_interface.orrery import PublicationFile as PublicationFile +from microcosm_emitter_interface.orrery import ( + PublicationFileRole as PublicationFileRole, +) +from microcosm_emitter_interface.orrery import ( + PublicationInventory as PublicationInventory, +) +from microcosm_emitter_interface.orrery import ( + PublicationReceipt as PublicationReceipt, +) @dataclass(frozen=True) diff --git a/src/microcosm_emitter/telemetry/client.py b/src/microcosm_emitter/telemetry/client.py index d60472c4..795e5e25 100644 --- a/src/microcosm_emitter/telemetry/client.py +++ b/src/microcosm_emitter/telemetry/client.py @@ -1,11 +1,16 @@ -"""Thin build adapter: start the service, send requests, and close the connection.""" +"""Thin build adapter: start the service, send requests, and close the connection. + +The run identity, the configuration and every message come from +``microcosm_emitter_interface.telemetry``, so this in-process client and the +interface package send identical messages. Each message is built inside the +error handling: an argument that cannot be converted to JSON prints one warning +and the build continues. +""" from __future__ import annotations import sys -import uuid from collections.abc import Mapping -from dataclasses import dataclass, field from pathlib import Path from typing import Any, Protocol @@ -23,19 +28,21 @@ TelemetryEventType, TelemetryStatus, ) -from microcosm_emitter.telemetry.requests import ( - COMMAND_BLOCK, - COMMAND_CALIBRATION_PROGRESS, - COMMAND_COMPLETE, - COMMAND_EMIT, - COMMAND_FAIL, - COMMAND_PROGRESS, - COMMAND_STAGE, - COMMAND_TRANSITION_CALIBRATION_PROGRESS, - COMMAND_TRANSITION_STAGE, - DEFAULT_FAILURE_CLASS, +from microcosm_emitter.telemetry.requests import DEFAULT_FAILURE_CLASS +from microcosm_emitter_interface.telemetry import ( + BuildCompleted, + BuildFailure, + BuildRefused, + CalibrationProgress, + EventRecord, + StageEvent, + StageProgress, + StageTransition, + TelemetryCommand, + TelemetrySettings, + telemetry_message, ) -from microcosm_emitter.telemetry.serialization import wire_value +from microcosm_emitter_interface.telemetry import TelemetryRun as TelemetryRun class Transport(Protocol): @@ -45,30 +52,6 @@ def send(self, message: Mapping[str, Any]) -> None: ... def close(self) -> None: ... -@dataclass(frozen=True) -class TelemetryRun: - """Identity registered with the hosted collector for one build.""" - - run_id: str - country_code: str - pipeline: str - candidate_id: str | None = None - release_id: str | None = None - run_kind: str = "build" - producer_id: str = field(default_factory=lambda: uuid.uuid4().hex) - - def as_registration(self) -> dict[str, Any]: - return { - "run_id": self.run_id, - "producer_id": self.producer_id, - "country_code": self.country_code, - "pipeline": self.pipeline, - "candidate_id": self.candidate_id, - "release_id": self.release_id, - "run_kind": self.run_kind, - } - - class LocalTelemetryEmitter: """Send bounded local requests; telemetry processing runs in the service.""" @@ -102,17 +85,15 @@ def start( ) -> LocalTelemetryEmitter: """Start the installed service; return a disabled adapter on failure.""" try: + settings = TelemetrySettings( + run=run, + identity={} if identity is None else identity, + development_collector_url=development_collector_url, + heartbeat_seconds=heartbeat_seconds, + ) handle = start_service( TELEMETRY_MODULE, - wire_value( - { - "registration": run.as_registration(), - "spool_path": str(spool_path), - "identity": identity or {}, - "development_collector_url": development_collector_url, - "heartbeat_seconds": heartbeat_seconds, - } - ), + settings.configuration(Path(spool_path)), startup_timeout=startup_timeout_seconds, ) except Exception as error: @@ -137,11 +118,11 @@ def start( def available(self) -> bool: return self._transport is not None and not self._closed - def _request(self, command: str, **arguments: Any) -> None: + def _request(self, command: TelemetryCommand) -> None: if not self.available: return try: - self._transport.send(wire_value({"command": command, **arguments})) + self._transport.send(telemetry_message(command)) except Exception as error: self._warn(error) @@ -162,12 +143,13 @@ def emit( details: Mapping[str, Any] | None = None, ) -> None: self._request( - COMMAND_EMIT, - event_type=event_type, - stage_id=stage_id, - status=status, - message=message, - details=details or {}, + EventRecord( + event_type, + status, + stage_id, + message, + {} if details is None else details, + ) ) def stage( @@ -178,13 +160,7 @@ def stage( message: str | None = None, **details: Any, ) -> None: - self._request( - COMMAND_STAGE, - stage_id=stage_id, - status=status, - message=message, - details=details, - ) + self._request(StageEvent(stage_id, status, message, details)) def transition_stage( self, @@ -194,13 +170,7 @@ def transition_stage( message: str | None = None, **details: Any, ) -> None: - self._request( - COMMAND_TRANSITION_STAGE, - stage_id=stage_id, - status=status, - message=message, - details=details, - ) + self._request(StageTransition(stage_id, status, message, details)) def progress( self, @@ -211,20 +181,13 @@ def progress( unit: str | None = None, **details: Any, ) -> None: - self._request( - COMMAND_PROGRESS, - stage_id=stage_id, - done=done, - total=total, - unit=unit, - details=details, - ) + self._request(StageProgress(stage_id, done, total, unit, details)) def calibration_progress(self, event: Mapping[str, Any]) -> None: - self._request(COMMAND_CALIBRATION_PROGRESS, event=event) + self._request(CalibrationProgress(event)) def transition_calibration_progress(self, event: Mapping[str, Any]) -> None: - self._request(COMMAND_TRANSITION_CALIBRATION_PROGRESS, event=event) + self._request(CalibrationProgress(event, transition=True)) def fail( self, @@ -241,12 +204,13 @@ def fail( self._warn(formatting_error) message = None self._request( - COMMAND_FAIL, - message=message, - error_type=type(error).__name__, - failed_during=failed_during, - failure_class=failure_class, - **({"error_code": error_code} if error_code is not None else {}), + BuildFailure( + error_type=type(error).__name__, + message=message, + failed_during=failed_during, + failure_class=failure_class, + error_code=error_code, + ) ) finally: if self._close_service_on_finish: @@ -262,12 +226,15 @@ def block( ) -> None: """Ask the service to finish a run refused by its validation checks.""" try: + # The interface names these fields for what they hold: the + # refusing validation checks, how many failed, and each status. self._request( - COMMAND_BLOCK, - phase=phase, - blocking_gate_ids=blocking_gate_ids, - blocking_failure_count=blocking_failure_count, - gate_statuses=gate_statuses, + BuildRefused( + phase=phase, + refusing_check_ids=blocking_gate_ids, + refusing_check_count=blocking_failure_count, + check_statuses=gate_statuses, + ) ) finally: if self._close_service_on_finish: @@ -275,7 +242,7 @@ def block( def complete(self) -> None: try: - self._request(COMMAND_COMPLETE) + self._request(BuildCompleted()) finally: if self._close_service_on_finish: self.close() diff --git a/src/microcosm_emitter/telemetry/constants.py b/src/microcosm_emitter/telemetry/constants.py index 8a3ea561..6d0c7a43 100644 --- a/src/microcosm_emitter/telemetry/constants.py +++ b/src/microcosm_emitter/telemetry/constants.py @@ -2,8 +2,11 @@ from typing import Final +from microcosm_emitter_interface.telemetry import ( + DEFAULT_HEARTBEAT_SECONDS as DEFAULT_HEARTBEAT_SECONDS, +) + TELEMETRY_MODULE: Final = "telemetry" -DEFAULT_HEARTBEAT_SECONDS: Final = 60.0 SERVICE_START_WARNING: Final = "Local telemetry emitter service failed to start ({error_type}); the build will continue." QUEUE_WARNING: Final = ( "Telemetry could not be queued ({error_type}); the build will continue." diff --git a/src/microcosm_emitter/telemetry/protocol.py b/src/microcosm_emitter/telemetry/protocol.py index 41a75324..2998d793 100644 --- a/src/microcosm_emitter/telemetry/protocol.py +++ b/src/microcosm_emitter/telemetry/protocol.py @@ -3,12 +3,13 @@ from __future__ import annotations from types import MappingProxyType -from typing import Final, Literal +from typing import Final -type TelemetryEventType = Literal[ - "run", "stage", "progress", "calibration", "heartbeat" -] -type TelemetryStatus = Literal["started", "progress", "completed", "failed", "blocked"] +# The interface package defines these for both kinds of client. +from microcosm_emitter_interface.telemetry import ( + TelemetryEventType as TelemetryEventType, +) +from microcosm_emitter_interface.telemetry import TelemetryStatus as TelemetryStatus TELEMETRY_SCHEMA_VERSION: Final = 1 TELEMETRY_IDENTIFIER_PATTERN: Final = r"^[A-Za-z0-9][A-Za-z0-9._:-]{0,191}$" diff --git a/src/microcosm_emitter/telemetry/requests.py b/src/microcosm_emitter/telemetry/requests.py index 039c2bd6..59d751a8 100644 --- a/src/microcosm_emitter/telemetry/requests.py +++ b/src/microcosm_emitter/telemetry/requests.py @@ -2,6 +2,10 @@ from typing import Final +from microcosm_emitter_interface.telemetry import ( + DEFAULT_FAILURE_CLASS as DEFAULT_FAILURE_CLASS, +) + COMMAND_EMIT: Final = "emit" COMMAND_STAGE: Final = "stage" COMMAND_TRANSITION_STAGE: Final = "transition_stage" @@ -11,4 +15,3 @@ COMMAND_FAIL: Final = "fail" COMMAND_BLOCK: Final = "block" COMMAND_COMPLETE: Final = "complete" -DEFAULT_FAILURE_CLASS: Final = "build_failure" diff --git a/src/microcosm_emitter/telemetry/serialization.py b/src/microcosm_emitter/telemetry/serialization.py index da4873af..f1267867 100644 --- a/src/microcosm_emitter/telemetry/serialization.py +++ b/src/microcosm_emitter/telemetry/serialization.py @@ -1,24 +1,7 @@ -"""Convert Python request arguments to JSON without interpreting telemetry.""" +"""Convert Python request arguments to JSON without interpreting telemetry. -import math -from collections.abc import Mapping -from pathlib import Path -from typing import Any +The conversion is the interface package's, so the in-process client and +``microcosm_emitter_interface`` send identical messages. +""" - -def wire_value(value: Any) -> Any: - """Preserve values where possible, including scientific-library scalars.""" - if isinstance(value, Mapping): - return {str(key): wire_value(item) for key, item in value.items()} - if isinstance(value, (list, tuple)): - return [wire_value(item) for item in value] - if isinstance(value, Path): - return str(value) - if isinstance(value, float): - return value if math.isfinite(value) else None - if isinstance(value, (str, int, bool)) or value is None: - return value - item = getattr(value, "item", None) - if callable(item): - return wire_value(item()) - return str(value) +from microcosm_emitter_interface.serialization import wire_value as wire_value diff --git a/tests/interface/conftest.py b/tests/interface/conftest.py new file mode 100644 index 00000000..8d1f57a4 --- /dev/null +++ b/tests/interface/conftest.py @@ -0,0 +1,76 @@ +"""Loopback services shared by the interface tests; nothing leaves this machine.""" + +import json +import stat +import sys +import threading +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path + +import pytest + + +@pytest.fixture +def loopback_collector(): + """A fake collector that records each request's path and bearer token.""" + received_requests = [] + + class Handler(BaseHTTPRequestHandler): + def do_POST(self): + payload = json.loads(self.rfile.read(int(self.headers["Content-Length"]))) + received_requests.append( + (self.path, self.headers["Authorization"], payload) + ) + if self.path.endswith("/exchange"): + status_code, response = ( + 200, + {"access_token": "collector-session-for-tests", "expires_in": 3600}, + ) + elif self.path == "/v1/runs": + status_code, response = 201, {"registered": True} + else: + status_code, response = ( + 202, + {"accepted": len(payload["events"]), "duplicates": 0}, + ) + body = json.dumps(response).encode() + self.send_response(status_code) + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def log_message(self, *args): + pass + + server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + thread = threading.Thread(target=server.serve_forever) + thread.start() + try: + yield f"http://127.0.0.1:{server.server_port}", received_requests + finally: + server.shutdown() + thread.join(timeout=5) + server.server_close() + + +@pytest.fixture +def emitter_command_script(tmp_path): + """Write an executable stand-in for the installed command.""" + + def write(name: str, python_source: str) -> Path: + path = tmp_path / "commands" / name + path.parent.mkdir(exist_ok=True) + path.write_text(f"#!{sys.executable}\n{python_source}") + path.chmod(path.stat().st_mode | stat.S_IXUSR) + return path + + return write + + +@pytest.fixture +def isolated_path(monkeypatch, tmp_path): + """A PATH holding only one fresh directory.""" + directory = tmp_path / "path-directory" + directory.mkdir() + monkeypatch.setenv("PATH", str(directory)) + return directory diff --git a/tests/interface/test_client_errors.py b/tests/interface/test_client_errors.py new file mode 100644 index 00000000..d0ec57ec --- /dev/null +++ b/tests/interface/test_client_errors.py @@ -0,0 +1,254 @@ +"""No interface call raises into the build, whatever arguments it is given. + +Arguments are converted to JSON inside each call's error handling: a value that +cannot be converted makes the call report failure, with a reason, both with and +without a running emitter. +""" + +from pathlib import Path +from typing import Any + +import pytest + +from microcosm_emitter_interface.client import EmitterClient +from microcosm_emitter_interface.launcher import EmitterProcess +from microcosm_emitter_interface.telemetry import TelemetryRun, TelemetrySettings + + +class MultiElementArray: + """Behaves like a NumPy array or tensor with more than one element.""" + + def item(self): + raise ValueError("can only convert an array of size 1 to a Python scalar") + + +def self_referencing_mapping() -> dict[str, Any]: + mapping: dict[str, Any] = {} + mapping["itself"] = mapping + return mapping + + +def deeply_nested_list(depth: int = 100_000) -> list[Any]: + nested: list[Any] = [] + for _ in range(depth): + nested = [nested] + return nested + + +class UnresolvablePath: + """A path-like argument whose conversion fails.""" + + def __fspath__(self): + raise OSError("synthetic unresolvable path") + + +class RecordingTransport: + def __init__(self): + self.requests: list[tuple[str, str, dict[str, Any]]] = [] + + def send(self, module, message, *, timeout=None): + self.requests.append(("message", module, dict(message))) + + def query(self, module, message, *, timeout=None): + self.requests.append(("query", module, dict(message))) + return {} + + def close(self): + pass + + +def running_client() -> tuple[EmitterClient, RecordingTransport]: + transport = RecordingTransport() + process = EmitterProcess( + process=None, + transport=transport, + runtime_directory=Path("/nonexistent"), + description={"modules": ["orrery", "telemetry"], "protocol_versions": [1]}, + ) + return EmitterClient(process), transport + + +UNCONVERTIBLE_DETAILS = { + "multi-element array": lambda: MultiElementArray(), + "self-referencing mapping": self_referencing_mapping, + "deep nesting": deeply_nested_list, +} + + +@pytest.mark.parametrize( + "make_value", UNCONVERTIBLE_DETAILS.values(), ids=list(UNCONVERTIBLE_DETAILS) +) +def test_unconvertible_telemetry_arguments_report_failure_with_an_emitter( + make_value, capsys +): + client, transport = running_client() + assert client.telemetry.stage("compile", value=make_value()) is False + assert client.telemetry.calibration_progress({"loss": make_value()}) is False + assert transport.requests == [] + from microcosm_emitter_interface.client import RequestFailure + + failure = client.last_failure + assert isinstance(failure, RequestFailure) + assert (failure.kind, failure.reason) == ("telemetry", "invalid_arguments") + assert failure.error_type in {"ValueError", "RecursionError"} + warning = capsys.readouterr().err + assert warning.count("did not accept a telemetry request") == 1 + assert "invalid_arguments" in warning + assert "synthetic" not in warning and "scalar" not in warning + + +@pytest.mark.parametrize( + "make_value", UNCONVERTIBLE_DETAILS.values(), ids=list(UNCONVERTIBLE_DETAILS) +) +def test_unconvertible_telemetry_arguments_report_failure_without_an_emitter( + make_value, +): + client = EmitterClient() + assert client.telemetry.stage("compile", value=make_value()) is False + assert ( + client.telemetry.emit( + event_type="progress", status="progress", details={"value": make_value()} + ) + is False + ) + assert client.last_failure is not None + assert client.last_failure.reason == "emitter_unavailable" + + +def test_unconvertible_refusal_and_failure_arguments_report_failure(): + client, transport = running_client() + assert ( + client.telemetry.refuse( + phase="preflight", refusing_check_ids=MultiElementArray() + ) + is False + ) + assert ( + client.telemetry.fail(RuntimeError("x"), failed_during=MultiElementArray()) + is False + ) + assert transport.requests == [] + + +def test_an_emitter_that_raises_an_unexpected_error_is_a_failure_not_a_raise(): + client, transport = running_client() + + def unexpected(*args, **kwargs): + raise RecursionError("synthetic") + + transport.send = unexpected + transport.query = unexpected + assert client.telemetry.complete() is False + assert client.last_failure.reason == "not_accepted" + assert client.orrery_receipt("fixture-publication")["status"] == "failed" + + +def test_successful_call_after_a_failure_keeps_the_last_failure_until_the_next(): + client, transport = running_client() + assert client.telemetry.stage("compile", value=MultiElementArray()) is False + assert client.telemetry.complete() is True + assert transport.requests[-1] == ("message", "telemetry", {"command": "complete"}) + + +class ReceiptlessGraphTransport: + """The in-process client's transport, answering without a receipt.""" + + def request(self, message, *, timeout=None): + return {"receipt": None} + + def close(self): + pass + + +def test_an_orrery_receipt_reports_the_error_code_of_its_own_call(): + client, transport = running_client() + transport.query = lambda module, message, *, timeout=None: {"receipt": None} + assert client.telemetry.stage("compile", value=MultiElementArray()) is False + assert client.last_failure.reason == "invalid_arguments" + + receipt = client.orrery_receipt("a-publication-this-spool-never-queued") + + assert receipt["status"] == "failed" + assert receipt["error_code"] == "local_service_unavailable" + from microcosm_emitter.orrery.client import GraphPublicationClient + + in_process_client = GraphPublicationClient(ReceiptlessGraphTransport()) + assert receipt == in_process_client.receipt("a-publication-this-spool-never-queued") + + +def test_an_orrery_retry_reports_the_error_code_of_its_own_call(): + client, transport = running_client() + transport.query = lambda module, message, *, timeout=None: {"receipt": None} + assert client.orrery_inventory(UnresolvablePath(), {"graph.json": "graph"}) is None + assert client.last_failure.reason == "invalid_arguments" + + receipt = client.retry_orrery_inputs("a-publication-this-spool-never-queued") + + assert receipt["error_code"] == "local_service_unavailable" + + +def test_an_orrery_receipt_without_a_status_is_an_unavailable_receipt(): + client, transport = running_client() + transport.query = lambda module, message, *, timeout=None: { + "receipt": {"publication_id": "fixture-publication"} + } + + receipt = client.orrery_receipt("fixture-publication") + retried = client.retry_orrery_inputs("fixture-publication", wait_seconds=1) + handed_over = client.hand_over_orrery_inputs( + Path("/tmp"), + {"version": 1, "publication_id": "fixture-publication", "files": []}, + wait_seconds=1, + ) + + for answered in (receipt, retried, handed_over): + assert answered["status"] == "failed" + assert answered["error_code"] == "local_service_unavailable" + + +def test_unconvertible_orrery_arguments_report_failure(): + client, transport = running_client() + assert client.orrery_inventory(UnresolvablePath(), {"graph.json": "graph"}) is None + receipt = client.hand_over_orrery_inputs( + UnresolvablePath(), + {"version": 1, "publication_id": "fixture-publication", "files": []}, + wait_seconds=0, + ) + assert receipt["status"] == "failed" + assert receipt["publication_id"] == "fixture-publication" + assert receipt["error_code"] == "invalid_arguments" + assert client.last_failure.reason == "invalid_arguments" + missing_identifier = client.hand_over_orrery_inputs( + Path("/tmp"), {"version": 1, "files": []}, wait_seconds=0 + ) + assert missing_identifier["status"] == "failed" + assert missing_identifier["publication_id"] is None + assert all( + request[2].get("command") != "inventory" for request in transport.requests + ) + + +def test_unconvertible_startup_settings_start_an_unavailable_client(tmp_path, capsys): + settings = TelemetrySettings( + run=TelemetryRun("bad-settings", "UK", "test"), + identity={"accelerator": MultiElementArray()}, + ) + client = EmitterClient.start( + telemetry=settings, spool_path=tmp_path / "events.sqlite3" + ) + assert not client.available + assert client.startup_error is not None + assert client.startup_error.reason == "invalid_configuration" + assert client.telemetry.complete() is False + warning = capsys.readouterr().err + assert "invalid_configuration" in warning + assert "scalar" not in warning + + +def test_unconvertible_spool_path_starts_an_unavailable_client(): + client = EmitterClient.start( + telemetry=TelemetrySettings(run=TelemetryRun("bad-path", "UK", "test")), + spool_path=UnresolvablePath(), + ) + assert not client.available + assert client.startup_error.reason == "invalid_configuration" diff --git a/tests/interface/test_client_messages.py b/tests/interface/test_client_messages.py new file mode 100644 index 00000000..71680e3f --- /dev/null +++ b/tests/interface/test_client_messages.py @@ -0,0 +1,197 @@ +"""Each typed call sends exactly the message its protocol fixture records.""" + +from collections.abc import Callable +from pathlib import Path +from typing import Any + +import pytest + +from microcosm_emitter_interface.client import EmitterClient +from microcosm_emitter_interface.launcher import EmitterProcess +from microcosm_emitter_interface.orrery import OrrerySettings +from microcosm_emitter_interface.protocol_fixtures import ( + ProtocolFixture, + materialize, + protocol_fixtures, +) +from microcosm_emitter_interface.telemetry import TelemetryRun, TelemetrySettings + + +class RecordingTransport: + """Records requests and answers queries like an emitter that queued nothing.""" + + def __init__(self): + self.requests: list[tuple[str, str, dict[str, Any]]] = [] + + def send(self, module, message, *, timeout=None): + self.requests.append(("message", module, dict(message))) + + def query(self, module, message, *, timeout=None): + self.requests.append(("query", module, dict(message))) + if message.get("command") == "inventory": + return {"inventory": None} + return { + "receipt": { + "version": 1, + "publication_id": message.get("publication_id") + or message["inventory"]["publication_id"], + "status": "pending", + "error_code": None, + "attempts": 1, + } + } + + def close(self): + pass + + +@pytest.fixture +def recording_client(): + transport = RecordingTransport() + process = EmitterProcess( + process=None, + transport=transport, + runtime_directory=Path("/nonexistent"), + description={"modules": ["orrery", "telemetry"], "protocol_versions": [1]}, + ) + return EmitterClient(process), transport + + +def fixture_run() -> TelemetryRun: + return TelemetryRun( + "fixture-run", "UK", "fixture-pipeline", producer_id="fixture-producer" + ) + + +def fixture_inventory(directory: Path) -> dict[str, Any]: + fixture = fixtures_by_name()["orrery-enqueue"] + return materialize(fixture, directory=directory, spool_path=Path("unused"))[ + "inventory" + ] + + +def fixtures_by_name() -> dict[str, ProtocolFixture]: + return {fixture.name: fixture for fixture in protocol_fixtures()} + + +SENT_BY: dict[str, Callable[[EmitterClient, Path], Any]] = { + "telemetry-emit": lambda client, directory: client.telemetry.emit( + event_type="progress", + status="progress", + stage_id="compile", + message="Compiling.", + details={"step": 1}, + ), + "telemetry-stage": lambda client, directory: client.telemetry.stage( + "compile", rows=3 + ), + "telemetry-transition-stage": lambda client, directory: ( + client.telemetry.transition_stage("calibrating", message="Calibrating.") + ), + "telemetry-progress": lambda client, directory: client.telemetry.progress( + "calibrating", done=2, total=10, unit="epochs", phase="solve" + ), + "telemetry-calibration-progress": lambda client, directory: ( + client.telemetry.calibration_progress( + {"kind": "calibration_epoch", "epoch": 10, "epochs": 100, "loss": 0.25} + ) + ), + "telemetry-transition-calibration-progress": lambda client, directory: ( + client.telemetry.transition_calibration_progress( + {"kind": "calibration_epoch", "epoch": 20, "epochs": 100, "loss": 0.125} + ) + ), + "telemetry-fail": lambda client, directory: client.telemetry.fail( + RuntimeError("Synthetic failure."), + failed_during="calibrating", + error_code="BUILD_FAILED", + ), + "telemetry-fail-without-code": lambda client, directory: client.telemetry.fail( + MemoryError("Synthetic failure."), failure_class="oom" + ), + "telemetry-refusal": lambda client, directory: client.telemetry.refuse( + phase="preflight", + refusing_check_ids=["fixture-check"], + refusing_check_count=1, + check_statuses={"fixture-check": "failed"}, + ), + "telemetry-complete": lambda client, directory: client.telemetry.complete(), + "orrery-inventory": lambda client, directory: client.orrery_inventory( + directory, + {"graph.orrery.json": "graph", "graph.schema.json": "schema"}, + publication_id="fixture-publication", + ), + "orrery-enqueue": lambda client, directory: client.hand_over_orrery_inputs( + directory, fixture_inventory(directory), wait_seconds=0 + ), + "orrery-receipt": lambda client, directory: client.orrery_receipt( + "fixture-publication" + ), + "orrery-retry": lambda client, directory: client.retry_orrery_inputs( + "fixture-publication" + ), +} + +CONFIGURED_BY: dict[str, Callable[[Path], dict[str, Any]]] = { + "telemetry-configuration": lambda spool_path: TelemetrySettings( + run=fixture_run(), + identity={"host": "fixture"}, + development_collector_url="http://127.0.0.1:1", + ).configuration(spool_path), + "orrery-configuration": lambda spool_path: OrrerySettings( + development_collector_url="http://127.0.0.1:1", + publication_origin="http://127.0.0.1:1", + ).configuration(spool_path), +} + + +#: Host fixtures: the launch and each socket envelope, sent in +#: test_host_protocol.py. +HOST_FIXTURES = { + "host-close", + "host-hello", + "host-launch", + "host-message-envelope", + "host-ping", + "host-query-envelope", +} + + +def test_every_fixture_has_a_typed_call_that_produces_it(): + covered = set(SENT_BY) | set(CONFIGURED_BY) | HOST_FIXTURES + assert {fixture.name for fixture in protocol_fixtures()} == covered + + +@pytest.mark.parametrize("name", sorted(SENT_BY)) +def test_typed_call_sends_the_recorded_message(recording_client, tmp_path, name): + client, transport = recording_client + fixture = fixtures_by_name()[name] + directory = tmp_path / "inputs" + SENT_BY[name](client, directory) + expected = materialize(fixture, directory=directory.resolve(), spool_path=tmp_path) + assert transport.requests[0] == (fixture.kind, fixture.module, expected) + + +@pytest.mark.parametrize("name", sorted(CONFIGURED_BY)) +def test_module_settings_produce_the_recorded_configuration(tmp_path, name): + fixture = fixtures_by_name()[name] + spool_path = tmp_path / "events.sqlite3" + assert CONFIGURED_BY[name](spool_path) == materialize( + fixture, directory=tmp_path, spool_path=spool_path + ) + + +def test_calls_for_a_module_the_emitter_did_not_load_are_not_sent(tmp_path): + transport = RecordingTransport() + client = EmitterClient( + EmitterProcess( + process=None, + transport=transport, + runtime_directory=Path("/nonexistent"), + description={"modules": ["orrery"], "protocol_versions": [1]}, + ) + ) + assert client.telemetry.complete() is False + assert transport.requests == [] + receipt = client.orrery_receipt("fixture-publication") + assert receipt["status"] == "pending" diff --git a/tests/interface/test_host_protocol.py b/tests/interface/test_host_protocol.py new file mode 100644 index 00000000..2c447f08 --- /dev/null +++ b/tests/interface/test_host_protocol.py @@ -0,0 +1,177 @@ +"""The launcher and the socket transport send exactly the recorded host fixtures. + +These fixtures pin what an older interface release sends to whichever emitter +is installed: the command's arguments, the configuration object on standard +input and each socket request's envelope. An argument-parser or envelope change +made on both sides at once would pass every other test while breaking builds +that still run an older interface. +""" + +import json +import os +import shutil +import socket +import tempfile +import threading +from collections.abc import Callable +from pathlib import Path +from typing import Any + +import pytest + +from microcosm_emitter_interface.client import EmitterClient +from microcosm_emitter_interface.launcher import EmitterProcess +from microcosm_emitter_interface.orrery import OrrerySettings +from microcosm_emitter_interface.protocol_fixtures import ( + ProtocolFixture, + materialize, + protocol_fixtures, +) +from microcosm_emitter_interface.transport import SocketTransport + +HELLO_ANSWER = { + "emitter_version": "0.0.0", + "protocol_versions": [1], + "modules": ["orrery", "telemetry"], + "skipped_modules": [], +} + + +def fixtures_by_name() -> dict[str, ProtocolFixture]: + return {fixture.name: fixture for fixture in protocol_fixtures()} + + +class RecordingSocketServer: + """A Unix socket that records each request line and answers like a host.""" + + def __init__(self, socket_path: Path): + self.socket_path = socket_path + self.requests: list[bytes] = [] + self._server = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + self._server.bind(str(socket_path)) + self._server.listen(4) + self._server.settimeout(0.1) + self._stopped = threading.Event() + self._thread = threading.Thread(target=self._serve) + self._thread.start() + + def _serve(self): + while not self._stopped.is_set(): + try: + connection, _ = self._server.accept() + except TimeoutError: + continue + with connection: + connection.settimeout(2) + request = bytearray() + while not request.endswith(b"\n"): + part = connection.recv(65_536) + if not part: + break + request.extend(part) + self.requests.append(bytes(request)) + action = json.loads(request)["action"] + if action == "hello": + answer = json.dumps(HELLO_ANSWER).encode() + b"\n" + elif action == "query": + answer = b'{"receipt": null}\n' + else: + answer = b"ok\n" + connection.sendall(answer) + + def close(self): + self._stopped.set() + self._thread.join(timeout=5) + self._server.close() + + +@pytest.fixture +def recording_server(): + # Short paths are necessary for macOS's Unix-socket path limit. + directory = Path(tempfile.mkdtemp(prefix="pe-fixture-", dir="/tmp")) + server = RecordingSocketServer(directory / "service.sock") + try: + yield server + finally: + server.close() + shutil.rmtree(directory, ignore_errors=True) + + +def client_for(transport: SocketTransport) -> EmitterClient: + return EmitterClient( + EmitterProcess( + process=None, + transport=transport, + runtime_directory=Path("/nonexistent"), + description=dict(HELLO_ANSWER), + ) + ) + + +ENVELOPE_SENT_BY: dict[str, Callable[[SocketTransport], Any]] = { + "host-ping": lambda transport: transport.ping(timeout=2), + "host-hello": lambda transport: transport.hello(timeout=2), + "host-message-envelope": lambda transport: client_for( + transport + ).telemetry.complete(), + "host-query-envelope": lambda transport: client_for(transport).orrery_receipt( + "fixture-publication" + ), + "host-close": lambda transport: transport.close(), +} + + +def test_every_host_fixture_has_a_test_that_sends_it(): + host_fixtures = { + fixture.name for fixture in protocol_fixtures() if fixture.module == "host" + } + assert host_fixtures == set(ENVELOPE_SENT_BY) | {"host-launch"} + + +@pytest.mark.parametrize("name", sorted(ENVELOPE_SENT_BY)) +def test_socket_request_has_the_recorded_envelope(recording_server, name): + fixture = fixtures_by_name()[name] + transport = SocketTransport(recording_server.socket_path, timeout=2) + ENVELOPE_SENT_BY[name](transport) + (request,) = recording_server.requests + # One JSON object per request, ended by a newline. + assert request.endswith(b"\n") and request.count(b"\n") == 1 + assert json.loads(request) == fixture.body + + +def test_launcher_passes_the_recorded_arguments_and_standard_input( + tmp_path, emitter_command_script +): + record_path = tmp_path / "launch-record.json" + recorder = emitter_command_script( + "recording-emitter", + "import json, sys\n" + "standard_input = json.loads(sys.stdin.read())\n" + f"with open({str(record_path)!r}, 'w') as record:\n" + " json.dump({'arguments': sys.argv[1:]," + " 'standard_input': standard_input}, record)\n" + "sys.exit(3)\n", + ) + spool_path = tmp_path / "events.sqlite3" + client = EmitterClient.start( + orrery=OrrerySettings( + development_collector_url="http://127.0.0.1:1", + publication_origin="http://127.0.0.1:1", + ), + spool_path=spool_path, + command=recorder, + startup_timeout=10, + ) + assert client.startup_error is not None + assert client.startup_error.reason == "child_exit" + recorded = json.loads(record_path.read_text()) + socket_path = Path(recorded["arguments"][2]) + assert socket_path.name == "service.sock" + expected = materialize( + fixtures_by_name()["host-launch"], + directory=tmp_path, + spool_path=spool_path, + socket_path=socket_path, + parent_pid=os.getpid(), + ) + assert recorded == expected diff --git a/tests/interface/test_in_process_clients.py b/tests/interface/test_in_process_clients.py new file mode 100644 index 00000000..fa473b9f --- /dev/null +++ b/tests/interface/test_in_process_clients.py @@ -0,0 +1,231 @@ +"""The emitter's in-process clients share the interface's definitions. + +`microcosm_emitter.telemetry.client` and `microcosm_emitter.orrery.client` +build their messages, run identity, receipts and value conversion with the +interface's code, so the two kinds of client send the same messages and handle +unconvertible arguments the same way: they report failure, never raise. +""" + +from collections.abc import Callable +from pathlib import Path +from typing import Any + +import pytest + +import microcosm_emitter_interface.orrery as interface_orrery +import microcosm_emitter_interface.serialization as interface_serialization +import microcosm_emitter_interface.telemetry as interface_telemetry +from microcosm_emitter_interface.protocol_fixtures import materialize, protocol_fixtures + + +class MultiElementArray: + def item(self): + raise ValueError("can only convert an array of size 1 to a Python scalar") + + +class UnresolvablePath: + def __fspath__(self): + raise OSError("synthetic unresolvable path") + + def resolve(self): + raise OSError("synthetic unresolvable path") + + +def test_in_process_clients_reuse_the_interface_definitions(): + from microcosm_emitter.orrery import client as orrery_client + from microcosm_emitter.orrery import contracts as orrery_contracts + from microcosm_emitter.telemetry import client as telemetry_client + from microcosm_emitter.telemetry import serialization + + assert serialization.wire_value is interface_serialization.wire_value + assert telemetry_client.TelemetryRun is interface_telemetry.TelemetryRun + for name in ( + "PublicationFileRole", + "PublicationFile", + "PublicationInventory", + "PublicationReceipt", + ): + assert getattr(orrery_contracts, name) is getattr(interface_orrery, name) + from microcosm_emitter_interface import default_spool_path + + assert orrery_client.default_spool_path is default_spool_path + + +class RecordingTelemetryTransport: + def __init__(self): + self.requests: list[dict[str, Any]] = [] + self.closed = False + + def send(self, message): + self.requests.append(dict(message)) + + def close(self): + self.closed = True + + +def in_process_telemetry(): + from microcosm_emitter.telemetry.client import LocalTelemetryEmitter, TelemetryRun + + transport = RecordingTelemetryTransport() + emitter = LocalTelemetryEmitter( + run=TelemetryRun("fixture-run", "UK", "fixture-pipeline"), + transport=transport, + close_service_on_finish=False, + ) + return emitter, transport + + +#: The in-process call equivalent to each telemetry fixture's typed call. +IN_PROCESS_SENDS: dict[str, Callable[[Any], None]] = { + "telemetry-emit": lambda emitter: emitter.emit( + event_type="progress", + status="progress", + stage_id="compile", + message="Compiling.", + details={"step": 1}, + ), + "telemetry-stage": lambda emitter: emitter.stage("compile", rows=3), + "telemetry-transition-stage": lambda emitter: emitter.transition_stage( + "calibrating", message="Calibrating." + ), + "telemetry-progress": lambda emitter: emitter.progress( + "calibrating", done=2, total=10, unit="epochs", phase="solve" + ), + "telemetry-calibration-progress": lambda emitter: emitter.calibration_progress( + {"kind": "calibration_epoch", "epoch": 10, "epochs": 100, "loss": 0.25} + ), + "telemetry-transition-calibration-progress": lambda emitter: ( + emitter.transition_calibration_progress( + {"kind": "calibration_epoch", "epoch": 20, "epochs": 100, "loss": 0.125} + ) + ), + "telemetry-fail": lambda emitter: emitter.fail( + RuntimeError("Synthetic failure."), + failed_during="calibrating", + error_code="BUILD_FAILED", + ), + "telemetry-fail-without-code": lambda emitter: emitter.fail( + MemoryError("Synthetic failure."), failure_class="oom" + ), + "telemetry-refusal": lambda emitter: emitter.block( + phase="preflight", + blocking_gate_ids=["fixture-check"], + blocking_failure_count=1, + gate_statuses={"fixture-check": "failed"}, + ), + "telemetry-complete": lambda emitter: emitter.complete(), +} + + +def test_every_telemetry_message_fixture_has_an_in_process_call(): + assert set(IN_PROCESS_SENDS) == { + fixture.name + for fixture in protocol_fixtures() + if fixture.module == "telemetry" and fixture.kind == "message" + } + + +@pytest.mark.parametrize("name", sorted(IN_PROCESS_SENDS)) +def test_in_process_telemetry_sends_the_recorded_message(tmp_path, name): + (fixture,) = [fixture for fixture in protocol_fixtures() if fixture.name == name] + emitter, transport = in_process_telemetry() + IN_PROCESS_SENDS[name](emitter) + assert transport.requests == [ + materialize(fixture, directory=tmp_path, spool_path=tmp_path) + ] + + +def test_in_process_telemetry_reports_unconvertible_arguments(capsys): + emitter, transport = in_process_telemetry() + emitter.stage("compile", value=MultiElementArray()) + emitter.block(phase="preflight", blocking_gate_ids=MultiElementArray()) + assert transport.requests == [] + assert capsys.readouterr().err.count("could not be queued") == 1 + + +class RecordingGraphTransport: + def __init__(self, answer: Callable[[dict[str, Any]], dict[str, Any]]): + self.answer = answer + self.requests: list[dict[str, Any]] = [] + + def request(self, message, *, timeout=None): + self.requests.append(dict(message)) + return self.answer(dict(message)) + + def close(self): + pass + + +def test_in_process_orrery_client_reports_unconvertible_arguments(): + from microcosm_emitter.orrery.client import GraphPublicationClient + + transport = RecordingGraphTransport(lambda message: {"receipt": None}) + client = GraphPublicationClient(transport) + assert client.inventory(UnresolvablePath(), {"graph.json": "graph"}) is None + receipt = client.enqueue( + UnresolvablePath(), + {"version": 1, "publication_id": "fixture-publication", "files": []}, + ) + assert receipt["status"] == "failed" + assert receipt["publication_id"] == "fixture-publication" + assert receipt["error_code"] == "invalid_arguments" + missing_identifier = client.enqueue(Path("/tmp"), {"version": 1, "files": []}) + assert missing_identifier["status"] == "failed" + assert missing_identifier["publication_id"] is None + assert transport.requests == [] + + +def test_in_process_orrery_client_reports_an_unexpected_transport_error(): + from microcosm_emitter.orrery.client import GraphPublicationClient + + def unexpected(message): + raise RecursionError("synthetic") + + client = GraphPublicationClient(RecordingGraphTransport(unexpected)) + receipt = client.receipt("fixture-publication") + assert receipt["status"] == "failed" + assert receipt["error_code"] == "local_service_unavailable" + + +def test_in_process_orrery_client_sends_the_recorded_queries(tmp_path): + from microcosm_emitter.orrery.client import GraphPublicationClient + + fixtures = { + fixture.name: fixture + for fixture in protocol_fixtures() + if fixture.module == "orrery" and fixture.kind == "query" + } + directory = tmp_path / "inputs" + inventory = materialize( + fixtures["orrery-enqueue"], directory=directory, spool_path=tmp_path + )["inventory"] + transport = RecordingGraphTransport( + lambda message: { + "receipt": { + "version": 1, + "publication_id": "fixture-publication", + "status": "pending", + "error_code": None, + "attempts": 1, + } + } + ) + client = GraphPublicationClient(transport) + client.inventory( + directory, + {"graph.orrery.json": "graph", "graph.schema.json": "schema"}, + publication_id="fixture-publication", + ) + client.enqueue(directory, inventory) + client.receipt("fixture-publication") + client.retry("fixture-publication") + expected = [ + materialize(fixtures[name], directory=directory.resolve(), spool_path=tmp_path) + for name in ( + "orrery-inventory", + "orrery-enqueue", + "orrery-receipt", + "orrery-retry", + ) + ] + assert transport.requests == expected diff --git a/tests/interface/test_launcher.py b/tests/interface/test_launcher.py new file mode 100644 index 00000000..f0d15120 --- /dev/null +++ b/tests/interface/test_launcher.py @@ -0,0 +1,240 @@ +"""The interface starts the installed emitter command as the build's own child.""" + +import os +import shutil +import time + +import psutil +import pytest + +from microcosm_emitter_interface.client import EmitterClient +from microcosm_emitter_interface.launcher import ( + COMMAND_ENVIRONMENT_VARIABLE, + DEFAULT_COMMAND, + EmitterStartupError, + resolve_command, + start_emitter, +) +from microcosm_emitter_interface.orrery import OrrerySettings +from microcosm_emitter_interface.protocol import PROTOCOL_VERSION +from microcosm_emitter_interface.telemetry import TelemetryRun, TelemetrySettings + +INSTALLED_COMMAND = shutil.which(DEFAULT_COMMAND) +FAKE_HF_TOKEN = "hf_fake_value_from_hf_token" +FAKE_HUGGINGFACE_TOKEN = "hf_fake_value_from_huggingface_token" +FAKE_SAVED_LOGIN_TOKEN = "hf_fake_value_from_saved_login" + + +def idle_orrery() -> OrrerySettings: + return OrrerySettings( + development_collector_url="http://127.0.0.1:1", + publication_origin="http://127.0.0.1:1", + ) + + +def test_explicit_command_wins_over_the_environment_and_the_path( + isolated_path, emitter_command_script, monkeypatch +): + explicit = emitter_command_script("explicit-emitter", "") + configured = emitter_command_script("configured-emitter", "") + on_path = isolated_path / DEFAULT_COMMAND + shutil.copy(explicit, on_path) + monkeypatch.setenv(COMMAND_ENVIRONMENT_VARIABLE, str(configured)) + assert resolve_command(explicit) == str(explicit) + + +def test_environment_variable_names_the_command_when_no_argument_does( + isolated_path, emitter_command_script, monkeypatch +): + configured = emitter_command_script("configured-emitter", "") + shutil.copy(configured, isolated_path / DEFAULT_COMMAND) + monkeypatch.setenv(COMMAND_ENVIRONMENT_VARIABLE, str(configured)) + assert resolve_command() == str(configured) + + +def test_installed_command_on_the_path_is_the_default( + isolated_path, emitter_command_script, monkeypatch +): + monkeypatch.delenv(COMMAND_ENVIRONMENT_VARIABLE, raising=False) + on_path = isolated_path / DEFAULT_COMMAND + shutil.copy(emitter_command_script("installed", ""), on_path) + assert resolve_command() == str(on_path) + + +def test_missing_command_is_reported_without_starting_anything( + isolated_path, monkeypatch +): + monkeypatch.delenv(COMMAND_ENVIRONMENT_VARIABLE, raising=False) + with pytest.raises(EmitterStartupError) as failure: + resolve_command() + assert failure.value.reason == "command_not_found" + + +def test_client_without_an_installed_emitter_is_unavailable_and_never_raises( + tmp_path, isolated_path, monkeypatch, capsys +): + monkeypatch.delenv(COMMAND_ENVIRONMENT_VARIABLE, raising=False) + client = EmitterClient.start( + telemetry=TelemetrySettings(run=TelemetryRun("no-emitter", "UK", "test")), + spool_path=tmp_path / "events.sqlite3", + ) + assert not client.available + assert client.startup_error is not None + assert client.startup_error.reason == "command_not_found" + client.telemetry.stage("compile") + client.telemetry.complete() + client.close() + assert "the build will continue" in capsys.readouterr().err + + +def test_emitter_runs_as_a_child_of_the_build_under_the_same_user(tmp_path): + assert INSTALLED_COMMAND is not None + process = start_emitter( + {"orrery": idle_orrery().configuration(tmp_path / "events.sqlite3")}, + command=INSTALLED_COMMAND, + ) + try: + child = psutil.Process(process.process.pid) + assert child.ppid() == os.getpid() + assert child.uids().real == os.getuid() + assert "spool_path" not in " ".join(child.cmdline()) + assert process.description["modules"] == ["orrery"] + assert PROTOCOL_VERSION in process.description["protocol_versions"] + finally: + process.close() + assert process.process.wait(timeout=20) == 0 + + +@pytest.mark.parametrize( + ("variables", "saved_login", "expected_token"), + [ + ( + {"HF_TOKEN": FAKE_HF_TOKEN, "HUGGINGFACE_TOKEN": FAKE_HUGGINGFACE_TOKEN}, + FAKE_SAVED_LOGIN_TOKEN, + FAKE_HF_TOKEN, + ), + ( + {"HUGGINGFACE_TOKEN": FAKE_HUGGINGFACE_TOKEN}, + FAKE_SAVED_LOGIN_TOKEN, + FAKE_HUGGINGFACE_TOKEN, + ), + ({}, FAKE_SAVED_LOGIN_TOKEN, FAKE_SAVED_LOGIN_TOKEN), + ], + ids=["hf-token-first", "huggingface-token-second", "saved-login-last"], +) +def test_child_finds_the_build_hugging_face_credential_in_the_same_order( + tmp_path, monkeypatch, loopback_collector, variables, saved_login, expected_token +): + """The token reaches the child only by inheriting the build's environment. + + The client passes no credential and no environment: HF_TOKEN, then + HUGGINGFACE_TOKEN, then the token saved by `hf auth login` + (core/auth.py), each set only in this build process. + """ + address, received_requests = loopback_collector + for name, value in variables.items(): + monkeypatch.setenv(name, value) + token_path = os.environ["HF_TOKEN_PATH"] + os.makedirs(os.path.dirname(token_path), exist_ok=True) + with open(token_path, "w") as token_file: + token_file.write(saved_login) + client = EmitterClient.start( + telemetry=TelemetrySettings( + run=TelemetryRun("credential-order", "UK", "test"), + development_collector_url=address, + ), + spool_path=tmp_path / "events.sqlite3", + command=INSTALLED_COMMAND, + ) + assert client.available, client.startup_error + try: + client.telemetry.stage("compile") + client.telemetry.complete() + deadline = time.monotonic() + 15 + while not received_requests and time.monotonic() < deadline: + time.sleep(0.05) + finally: + client.close() + exchange_path, authorization, _ = received_requests[0] + assert exchange_path == "/v1/auth/huggingface/exchange" + assert authorization == f"Bearer {expected_token}" + + +def test_emitter_that_does_not_accept_this_protocol_version_is_stopped( + tmp_path, emitter_command_script +): + future_only = emitter_command_script( + "future-emitter", + "import sys\n" + "from microcosm_emitter.host.__main__ import main\n" + "description = {'emitter_version': '99.0.0', 'protocol_versions': [99]}\n" + "sys.exit(main(sys.argv[2:], description=description))\n", + ) + with pytest.raises(EmitterStartupError) as failure: + start_emitter( + {"orrery": idle_orrery().configuration(tmp_path / "events.sqlite3")}, + command=future_only, + ) + assert failure.value.reason == "unsupported_protocol_version" + + +def test_emitter_that_exits_reports_only_its_exit_code( + tmp_path, emitter_command_script +): + exits = emitter_command_script( + "exiting-emitter", + "import sys\nsys.stdin.read()\nprint('private detail', file=sys.stderr)\n" + "sys.exit(3)\n", + ) + with pytest.raises(EmitterStartupError) as failure: + start_emitter( + {"orrery": {"spool_path": "private-configuration-value"}}, + command=exits, + startup_timeout=10, + ) + assert failure.value.reason == "child_exit" + assert failure.value.exit_code == 3 + assert "private" not in str(failure.value) + assert failure.value.__cause__ is None and failure.value.__context__ is None + + +def test_close_stops_the_emitter_and_removes_its_runtime_directory(tmp_path): + process = start_emitter( + {"orrery": idle_orrery().configuration(tmp_path / "events.sqlite3")}, + command=INSTALLED_COMMAND, + ) + process.close() + assert process.process.wait(timeout=20) == 0 + deadline = time.monotonic() + 5 + while process.runtime_directory.exists() and time.monotonic() < deadline: + time.sleep(0.05) + assert not process.runtime_directory.exists() + + +class AnsweringTransport: + """A socket transport stand-in that answers ``hello`` with a fixed value.""" + + def __init__(self, description): + self.description = description + + def hello(self, *, timeout=None): + return self.description + + +@pytest.mark.parametrize( + "description", + [ + {"protocol_versions": [PROTOCOL_VERSION], "modules": [{"name": "orrery"}]}, + {"protocol_versions": [PROTOCOL_VERSION], "modules": ["orrery", 3]}, + {"protocol_versions": [PROTOCOL_VERSION, {"version": 2}], "modules": []}, + ], + ids=["module-objects", "module-number", "version-object"], +) +def test_a_hello_answer_with_unusable_values_is_refused(description): + """A later emitter's differently shaped answer stops at startup, never in a + call: every call reads the module names as a set of strings.""" + from microcosm_emitter_interface.launcher import _accepted_description + + with pytest.raises(EmitterStartupError) as failure: + _accepted_description(AnsweringTransport(description), time.monotonic() + 5) + assert failure.value.reason == "unsupported_protocol_version" diff --git a/tests/interface/test_protocol_fixtures.py b/tests/interface/test_protocol_fixtures.py new file mode 100644 index 00000000..01480ff7 --- /dev/null +++ b/tests/interface/test_protocol_fixtures.py @@ -0,0 +1,317 @@ +"""This emitter accepts every protocol fixture the interface package ships. + +Builds keep whichever emitter is installed, so every emitter release must +accept every protocol version that interface releases still in use send. The +fixtures are those messages; released fixtures are never edited, only added. +""" + +import json +import os +import shutil +import socket +import subprocess +import tempfile +import threading +import time +from pathlib import Path + +import pytest + +from microcosm_emitter.compatibility import SUPPORTED_PROTOCOL_VERSIONS +from microcosm_emitter.host.contracts import ModuleContext +from microcosm_emitter.host.runtime import ServiceRuntime +from microcosm_emitter.orrery.service.module import OrreryModule +from microcosm_emitter.telemetry.service.module import TelemetryModule +from microcosm_emitter_interface.launcher import DEFAULT_COMMAND, start_emitter +from microcosm_emitter_interface.protocol_fixtures import ( + fixture_digests, + materialize, + protocol_fixtures, +) + +#: SHA-256 of each released fixture file. A changed digest means a released +#: message changed shape: add a fixture or a protocol version instead. +RELEASED_FIXTURE_DIGESTS = { + "protocol-v1/host-close.json": ( + "68fa81b4116852f91f3c0dab3716a1974a8150039fb2b238f79efc094d9b0f6f" + ), + "protocol-v1/host-hello.json": ( + "36fcac377b69a9103b7c1cfd884d2ff667e030463edc9da349a379bb4868b927" + ), + "protocol-v1/host-launch.json": ( + "cb67b2cc3e8e709ca177c0f05b7b412fec4fbf7ffb1ae9b5028e3c3cbcb15a1c" + ), + "protocol-v1/host-message-envelope.json": ( + "21b5e6b22019cd82b2e4b57363c123ed2628bee86f4f4ee1f08c3ca7bac6cc12" + ), + "protocol-v1/host-ping.json": ( + "b4b6ba4668a29fcb79160a245f3b92b0191cd98e9a611a2713f65523e40434a4" + ), + "protocol-v1/host-query-envelope.json": ( + "dd364ff5c91f1a81778245a8453c1b0eee8434c9824898f36327204e032d22e8" + ), + "protocol-v1/orrery-configuration.json": ( + "956b5e1ca4c65a9fc0282f648b45d00ae697ac72e50147269e365a52d16f141a" + ), + "protocol-v1/orrery-enqueue.json": ( + "e70c33e089b7361dc01e0448dd34458c8ca8c606ea8c552af191c9b08cb11c5b" + ), + "protocol-v1/orrery-inventory.json": ( + "8cba4ec76bec8db21f1e158bf5da4eef23ca799f6730012051280d33763092f5" + ), + "protocol-v1/orrery-receipt.json": ( + "cbc7ed6d1776af8082d6def5f920d547400c7da073febea89364150ccd4e7a4b" + ), + "protocol-v1/orrery-retry.json": ( + "acfa79ba0221ff3786e61cdad699124e81cf825cc4b1364895a3679e839ade49" + ), + "protocol-v1/telemetry-calibration-progress.json": ( + "52c2396dd12f9e0cb65e429139d0bd1f2fa6e5adf9c007fd513161de560775c4" + ), + "protocol-v1/telemetry-complete.json": ( + "8f32aa105366fc544a3dd85d3e330c8602feb2f770bb196d49d09a4126747a05" + ), + "protocol-v1/telemetry-configuration.json": ( + "fd2ae2786543b8d9239f4f5a6604440daada9ab89ec974abc858b4847c3a563b" + ), + "protocol-v1/telemetry-emit.json": ( + "0ae388f58fae80267391360c30b21bf392072eea7158dda465630911e3d919c3" + ), + "protocol-v1/telemetry-fail-without-code.json": ( + "ad55024a6bf0fae3583245b74b83671ea652dbb5ad48763d2e1e341c116e5fdb" + ), + "protocol-v1/telemetry-fail.json": ( + "f509a443a361dfa5133fa32667b88a2e6be8f0ef29f8facfc03136e1af215b56" + ), + "protocol-v1/telemetry-progress.json": ( + "fcb9f3b69b2c4e941b9a7f4af300a87781861d6b1b69acdeb1ba2f3fb7d52166" + ), + "protocol-v1/telemetry-refusal.json": ( + "33c6bc9750b9e342a8a86e7cc9c25c23a76a03419b6bdb1690bfb9ec3353c82c" + ), + "protocol-v1/telemetry-stage.json": ( + "083a6a88e30092b45a75911d8cd80e7f8079a7715d2a5870215203ec15c642ec" + ), + "protocol-v1/telemetry-transition-calibration-progress.json": ( + "9236f9bc9c0cf771fc67af01f1b99606be28a8c49d142709aeb6f3cad1bc46da" + ), + "protocol-v1/telemetry-transition-stage.json": ( + "50fcb5f0379579625db58daba44aab2d1a63c73c4d54738dfdf65d7b93d8c783" + ), +} + + +def fixtures_of(module: str, kind: str): + return sorted( + ( + fixture + for fixture in protocol_fixtures() + if fixture.module == module and fixture.kind == kind + ), + key=lambda fixture: (fixture.sequence, fixture.name), + ) + + +def configuration_of(module: str, tmp_path: Path) -> dict: + (fixture,) = fixtures_of(module, "configuration") + return materialize( + fixture, directory=tmp_path, spool_path=tmp_path / "events.sqlite3" + ) + + +def test_every_fixture_uses_a_protocol_version_this_emitter_accepts(): + fixtures = protocol_fixtures() + assert fixtures + assert {fixture.protocol_version for fixture in fixtures} <= set( + SUPPORTED_PROTOCOL_VERSIONS + ) + + +def test_released_fixtures_are_unchanged(): + assert fixture_digests() == RELEASED_FIXTURE_DIGESTS + + +@pytest.mark.parametrize( + "fixture", + fixtures_of("telemetry", "message"), + ids=lambda fixture: fixture.name, +) +def test_telemetry_module_records_each_message_fixture(tmp_path, fixture): + module = TelemetryModule( + configuration_of("telemetry", tmp_path), ModuleContext(os.getpid()) + ) + module.initialize() + try: + registration = module.registration + before = module.spool.batch(registration["run_id"], registration["producer_id"]) + module.handle_message( + materialize(fixture, directory=tmp_path, spool_path=tmp_path) + ) + after = module.spool.batch(registration["run_id"], registration["producer_id"]) + assert len(after) > len(before) + finally: + module.close(time.monotonic()) + + +def test_orrery_module_answers_each_query_fixture_in_order(tmp_path): + module = OrreryModule( + configuration_of("orrery", tmp_path), ModuleContext(os.getpid()) + ) + module.initialize() + directory = tmp_path / "inputs" + try: + for fixture in fixtures_of("orrery", "query"): + directory.mkdir(exist_ok=True) + for name, text in fixture.files.items(): + (directory / name).write_text(text) + response = module.query( + materialize(fixture, directory=directory, spool_path=tmp_path) + ) + assert set(fixture.response_keys) <= set(response), fixture.name + receipt = response.get("receipt") + if receipt is not None: + assert receipt["status"] == "pending", fixture.name + finally: + module.close(time.monotonic()) + + +def test_installed_command_answers_the_hello_fixture(tmp_path): + """The answer has at least the recorded keys; a newer emitter may add more.""" + (fixture,) = fixtures_of("host", "hello") + process = start_emitter( + {"orrery": configuration_of("orrery", tmp_path)}, + command=shutil.which(DEFAULT_COMMAND), + ) + try: + assert set(fixture.response_keys) <= set(process.description) + finally: + process.close() + assert process.process.wait(timeout=20) == 0 + + +def raw_request(socket_path: Path, body: dict, *, timeout: float = 5.0) -> bytes: + with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as peer: + peer.settimeout(timeout) + peer.connect(str(socket_path)) + peer.sendall(json.dumps(body).encode() + b"\n") + response = bytearray() + while not response.endswith(b"\n"): + part = peer.recv(65_536) + if not part: + break + response.extend(part) + return bytes(response) + + +@pytest.fixture +def socket_directory(): + # Short paths are necessary for macOS's Unix-socket path limit. + directory = Path(tempfile.mkdtemp(prefix="pe-fixture-", dir="/tmp")) + try: + yield directory + finally: + shutil.rmtree(directory, ignore_errors=True) + + +def test_installed_command_starts_from_the_launch_fixture(tmp_path, socket_directory): + """The recorded arguments and standard input start this emitter's host.""" + (launch,) = fixtures_of("host", "launch") + socket_path = socket_directory / "service.sock" + launched = materialize( + launch, + directory=tmp_path, + spool_path=tmp_path / "events.sqlite3", + socket_path=socket_path, + parent_pid=os.getpid(), + ) + process = subprocess.Popen( + [shutil.which(DEFAULT_COMMAND), *launched["arguments"]], + stdin=subprocess.PIPE, + stdout=subprocess.DEVNULL, + start_new_session=True, + ) + try: + process.stdin.write(json.dumps(launched["standard_input"]).encode()) + process.stdin.close() + deadline = time.monotonic() + 15 + while time.monotonic() < deadline: + assert process.poll() is None, "the host refused the launch fixture" + try: + if raw_request(socket_path, {"action": "ping"}) == b"ok\n": + break + except OSError: + time.sleep(0.05) + hello = json.loads(raw_request(socket_path, {"action": "hello"})) + assert hello["modules"] == sorted(launched["standard_input"]) + assert raw_request(socket_path, {"action": "close"}) == b"ok\n" + assert process.wait(timeout=20) == 0 + finally: + if process.poll() is None: + process.kill() + process.wait(timeout=5) + + +class RecordingModule: + """Records the messages and queries the host routes to it.""" + + def __init__(self): + self.messages: list[dict] = [] + self.queries: list[dict] = [] + + def initialize(self): + pass + + def handle_message(self, message): + self.messages.append(dict(message)) + + def query(self, message): + self.queries.append(dict(message)) + return {"receipt": None} + + def tick(self, now): + pass + + def parent_exited(self): + pass + + def close(self, deadline): + pass + + +def test_host_answers_each_envelope_fixture(socket_directory): + """The host parses each recorded envelope and routes it to its module.""" + modules = {"orrery": RecordingModule(), "telemetry": RecordingModule()} + socket_path = socket_directory / "service.sock" + runtime = ServiceRuntime( + socket_path, + modules=modules, + is_parent_alive=lambda: True, + tick_seconds=0.01, + description={"modules": sorted(modules)}, + ) + thread = threading.Thread(target=runtime.run) + thread.start() + try: + deadline = time.monotonic() + 5 + while not socket_path.exists() and time.monotonic() < deadline: + time.sleep(0.01) + envelopes = fixtures_of("host", "envelope") + assert envelopes + for fixture in envelopes: + response = raw_request(socket_path, fixture.body) + if fixture.response == "acknowledgement": + assert response == b"ok\n", fixture.name + else: + assert fixture.response == "answer", fixture.name + assert isinstance(json.loads(response), dict), fixture.name + body = fixture.body + if body["action"] == "message": + assert modules[body["module"]].messages == [body["message"]] + elif body["action"] == "query": + assert modules[body["module"]].queries == [body["message"]] + assert envelopes[-1].body == {"action": "close"} + thread.join(timeout=10) + assert not thread.is_alive(), "the close envelope did not stop the host" + finally: + runtime.stop() + thread.join(timeout=10) diff --git a/tests/test_boundaries.py b/tests/test_boundaries.py index cc575a9e..785b9365 100644 --- a/tests/test_boundaries.py +++ b/tests/test_boundaries.py @@ -121,3 +121,41 @@ def test_database_code_has_no_raw_sql_or_manual_schema_creation(): alias.name in {"sqlite3", "psycopg", "psycopg2"} for alias in node.names ), path + + +INTERFACE_SOURCE = ROOT / "packages/microcosm-emitter-interface/src" + + +def test_interface_imports_only_the_standard_library_and_itself(): + allowed = set(sys.stdlib_module_names) | {"microcosm_emitter_interface"} + paths = list((INTERFACE_SOURCE / "microcosm_emitter_interface").rglob("*.py")) + assert paths + for path in paths: + for name in imports(path): + assert name.split(".")[0] in allowed, (path, name) + + +def test_emitter_host_does_not_import_the_interface(): + for path in (ROOT / "src/microcosm_emitter/host").rglob("*.py"): + assert not any( + name.split(".")[0] == "microcosm_emitter_interface" + for name in imports(path) + ), path + + +def test_interface_imports_have_no_runtime_side_effects(): + script = """ +import sys +sys.dont_write_bytecode = True +def audit(event, args): + if event in {"socket.connect", "socket.bind", "subprocess.Popen", "os.mkdir"}: + raise AssertionError(event) +sys.addaudithook(audit) +import microcosm_emitter_interface +import microcosm_emitter_interface.client +import microcosm_emitter_interface.launcher +import microcosm_emitter_interface.protocol_fixtures +assert not any(name.startswith("microcosm_emitter.") for name in sys.modules) +assert "microcosm_emitter" not in sys.modules +""" + subprocess.run([sys.executable, "-c", script], check=True) diff --git a/tests/test_package_names.py b/tests/test_package_names.py index 9decf043..7cbe9742 100644 --- a/tests/test_package_names.py +++ b/tests/test_package_names.py @@ -9,15 +9,16 @@ EXPECTED_NAME = "microcosm-emitter" -def test_single_distribution_with_required_telemetry_dependencies(): +INTERFACE_NAME = "microcosm-emitter-interface" +INTERFACE_ROOT = ROOT / "packages" / INTERFACE_NAME +SOURCE_URL = "https://github.com/PolicyEngine/microcosm-emitter" + + +def test_emitter_requires_telemetry_and_its_own_interface_release(): config = tomllib.loads((ROOT / "pyproject.toml").read_text()) project = config["project"] assert project["name"] == EXPECTED_NAME - assert project["urls"]["Source"] == ( - "https://github.com/PolicyEngine/microcosm-emitter" - ) - assert not list((ROOT / "packages").glob("*/pyproject.toml")) - assert "workspace" not in config.get("tool", {}).get("uv", {}) + assert project["urls"]["Source"] == SOURCE_URL assert "optional-dependencies" not in project assert set(project["dependencies"]) == { "alembic>=1.13.3,<2", @@ -25,7 +26,28 @@ def test_single_distribution_with_required_telemetry_dependencies(): "huggingface-hub>=0.20,<2", "psutil>=6,<8", "httpx>=0.27,<1", + # The emitter runs the checks and fixtures of the same release. + f"{INTERFACE_NAME}=={project['version']}", + } + + +def test_interface_is_the_only_other_distribution_and_has_no_dependencies(): + config = tomllib.loads((ROOT / "pyproject.toml").read_text()) + assert sorted( + path.relative_to(ROOT).as_posix() + for path in (ROOT / "packages").glob("*/pyproject.toml") + ) == [f"packages/{INTERFACE_NAME}/pyproject.toml"] + assert config["tool"]["uv"]["workspace"] == { + "members": [f"packages/{INTERFACE_NAME}"] } + assert config["tool"]["uv"]["sources"] == {INTERFACE_NAME: {"workspace": True}} + interface = tomllib.loads((INTERFACE_ROOT / "pyproject.toml").read_text())[ + "project" + ] + assert interface["name"] == INTERFACE_NAME + assert interface["version"] == config["project"]["version"] + assert interface["dependencies"] == [] + assert interface["urls"]["Source"] == SOURCE_URL def test_installed_distribution_includes_host_telemetry_and_migrations(): @@ -38,6 +60,8 @@ def test_installed_distribution_includes_host_telemetry_and_migrations(): requirements = emitter.requires assert all("extra ==" not in requirement for requirement in requirements) assert not any("microcosm-provider" in requirement for requirement in requirements) + assert f"{INTERFACE_NAME}=={project['version']}" in requirements + assert distribution(INTERFACE_NAME).version == project["version"] modules = list(entry_points(group="microcosm.emitter.modules", name="telemetry")) assert len(modules) == 1 assert modules[0].value == "microcosm_emitter.telemetry.service:create_module" diff --git a/tests/test_workflows.py b/tests/test_workflows.py index 30a70dc3..812bd202 100644 --- a/tests/test_workflows.py +++ b/tests/test_workflows.py @@ -11,6 +11,8 @@ ROOT = Path(__file__).resolve().parents[1] RELEASE_COMMIT = "Update microcosm-emitter" VERSIONED_FILE = "pyproject.toml" +INTERFACE_VERSIONED_FILE = "packages/microcosm-emitter-interface/pyproject.toml" +VERSIONED_FILES = (VERSIONED_FILE, INTERFACE_VERSIONED_FILE) def workflow_paths(): @@ -30,7 +32,9 @@ def bump_version(): def release_tree(root, *fragments): - shutil.copy(ROOT / VERSIONED_FILE, root / VERSIONED_FILE) + for versioned_file in VERSIONED_FILES: + (root / versioned_file).parent.mkdir(parents=True, exist_ok=True) + shutil.copy(ROOT / versioned_file, root / versioned_file) (root / "changelog.d").mkdir() (root / "changelog.d/.gitkeep").touch() for fragment in fragments: @@ -112,12 +116,89 @@ def test_only_publication_holds_oidc_and_only_versioning_holds_secrets(): assert text.count("secrets.") == 2 +#: The artifacts the verification workflow uploads for each distribution, in +#: the order PyPI must receive them: the emitter requires the interface of +#: the same version, so the interface is uploaded first. +PUBLISHED_ARTIFACTS = ( + ("interface-distribution", "dist/microcosm_emitter_interface-*"), + ("emitter-distribution", "dist/microcosm_emitter-*"), +) + + +def uploaded_artifacts(): + steps = workflow("ci.yml")["jobs"]["build-packages"]["steps"] + return { + step["with"]["name"]: step["with"] + for step in steps + if step.get("uses", "").startswith("actions/upload-artifact@") + } + + +def test_verification_uploads_each_distribution_as_its_own_artifact(): + artifacts = uploaded_artifacts() + # The collector job installs both wheels from the combined artifact. + assert artifacts["distributions"]["path"] == "dist/*" + for name, path in PUBLISHED_ARTIFACTS: + assert artifacts[name]["path"] == path + assert artifacts[name]["if-no-files-found"] == "error" + + +def test_emitter_artifact_pattern_excludes_the_interface(): + from fnmatch import fnmatch + + emitter_pattern = dict(PUBLISHED_ARTIFACTS)["emitter-distribution"] + interface_pattern = dict(PUBLISHED_ARTIFACTS)["interface-distribution"] + emitter_files = [ + "dist/microcosm_emitter-1.0.0-py3-none-any.whl", + "dist/microcosm_emitter-1.0.0.tar.gz", + ] + interface_files = [ + "dist/microcosm_emitter_interface-1.0.0-py3-none-any.whl", + "dist/microcosm_emitter_interface-1.0.0.tar.gz", + ] + assert all(fnmatch(name, emitter_pattern) for name in emitter_files) + assert not any(fnmatch(name, emitter_pattern) for name in interface_files) + assert all(fnmatch(name, interface_pattern) for name in interface_files) + assert not any(fnmatch(name, interface_pattern) for name in emitter_files) + + def test_pypi_receives_only_verified_artifacts(): publish = workflow("push.yaml")["jobs"]["publish"] - assert publish["steps"][0]["with"]["name"] == "distributions" + downloads = [ + step["with"]["name"] + for step in publish["steps"] + if step.get("uses", "").startswith("actions/download-artifact@") + ] + assert downloads == [name for name, _ in PUBLISHED_ARTIFACTS] + assert set(downloads) <= set(uploaded_artifacts()) assert not any("run" in step for step in publish["steps"]) +def test_publication_uploads_the_interface_first_and_can_be_rerun(): + """A failed or partial upload must leave a release that a rerun completes. + + The interface goes first, so PyPI never offers an emitter whose required + interface is missing. Files already on PyPI are skipped, so rerunning the + job after a partial upload uploads only what is missing instead of + stopping on "File already exists". + """ + steps = workflow("push.yaml")["jobs"]["publish"]["steps"] + sequence = [] + for step in steps: + uses = step.get("uses", "") + if uses.startswith("actions/download-artifact@"): + sequence.append(("download", step["with"]["name"], step["with"]["path"])) + elif uses.startswith("pypa/gh-action-pypi-publish@"): + assert step["with"]["skip-existing"] == "true" + sequence.append(("upload", step["with"]["packages-dir"])) + assert sequence == [ + ("download", "interface-distribution", "dist/interface"), + ("upload", "dist/interface/"), + ("download", "emitter-distribution", "dist/emitter"), + ("upload", "dist/emitter/"), + ] + + @pytest.mark.parametrize( ("types", "expected"), [ @@ -132,21 +213,67 @@ def test_fragment_types_choose_the_bump(bump_version, types, expected): assert bump_version["next_version"]("1.4.2", types) == expected -def test_release_rewrites_only_the_version_line(tmp_path, bump_version): +def changed_lines(versioned_file, root): + original = (ROOT / versioned_file).read_text().splitlines() + updated = (root / versioned_file).read_text().splitlines() + assert len(updated) == len(original) + return [ + (original_line, updated_line) + for original_line, updated_line in zip(original, updated, strict=True) + if original_line != updated_line + ] + + +def test_release_rewrites_both_versions_and_the_interface_pin_only( + tmp_path, bump_version +): root = release_tree(tmp_path, "1.added.md", "repair.fixed.md") current, upcoming = bump_version["plan_release"](root) assert upcoming == bump_version["next_version"](current, {"added", "fixed"}) bump_version["write_version"](root, current, upcoming) assert bump_version["current_version"](root) == upcoming - original = (ROOT / VERSIONED_FILE).read_text().splitlines() - updated = (root / VERSIONED_FILE).read_text().splitlines() - assert len(updated) == len(original) - changed = [ - (original_line, updated_line) - for original_line, updated_line in zip(original, updated, strict=True) - if original_line != updated_line + assert changed_lines(VERSIONED_FILE, root) == [ + (f'version = "{current}"', f'version = "{upcoming}"'), + ( + f' "microcosm-emitter-interface=={current}",', + f' "microcosm-emitter-interface=={upcoming}",', + ), ] - assert changed == [(f'version = "{current}"', f'version = "{upcoming}"')] + assert changed_lines(INTERFACE_VERSIONED_FILE, root) == [ + (f'version = "{current}"', f'version = "{upcoming}"') + ] + + +def test_release_refuses_diverged_distribution_versions(tmp_path, bump_version): + root = release_tree(tmp_path, "1.fixed.md") + interface = root / INTERFACE_VERSIONED_FILE + interface.write_text( + re.sub( + r'^version = "[^"]+"$', + 'version = "9.9.9"', + interface.read_text(), + count=1, + flags=re.MULTILINE, + ) + ) + with pytest.raises(ValueError, match="same version"): + bump_version["plan_release"](root) + + +def test_release_refuses_an_interface_pin_that_differs_from_the_version( + tmp_path, bump_version +): + root = release_tree(tmp_path, "1.fixed.md") + pyproject = root / VERSIONED_FILE + pyproject.write_text( + re.sub( + r'"microcosm-emitter-interface==[^"]+"', + '"microcosm-emitter-interface>=0.1"', + pyproject.read_text(), + ) + ) + with pytest.raises(ValueError, match="pin"): + bump_version["plan_release"](root) @pytest.mark.parametrize( @@ -160,25 +287,35 @@ def test_release_refuses_missing_or_untyped_fragments( bump_version["plan_release"](root) +@pytest.mark.parametrize("versioned_file", VERSIONED_FILES) @pytest.mark.parametrize( "edit_pyproject", [remove_version_line, add_tool_version_line], ids=["missing", "duplicated"], ) def test_release_refuses_a_missing_or_duplicated_version_line( - tmp_path, bump_version, edit_pyproject + tmp_path, bump_version, edit_pyproject, versioned_file ): root = release_tree(tmp_path, "1.fixed.md") - pyproject = root / VERSIONED_FILE + pyproject = root / versioned_file pyproject.write_text(edit_pyproject(pyproject.read_text())) with pytest.raises(ValueError, match="one version line"): bump_version["plan_release"](root) -def test_build_verifies_one_distribution_without_a_workspace(): +def test_build_verifies_both_distributions_from_one_script(): script = (ROOT / "tools/check-artifacts.sh").read_text() assert script.count("uv build ") == 1 - assert "--package" not in script - assert "--clear" in script + assert "uv build --all-packages --out-dir dist" in script + # Each package's build would clear the shared output directory, so one + # build could delete the other's artifacts; clear it once, beforehand. + assert "--clear" not in script + assert script.index("rm -rf dist") < script.index("uv build ") for path in workflow_paths(): - assert "--all-packages" not in path.read_text() + assert "uv build" not in path.read_text() + + +def test_installed_check_keeps_the_interface_apart_from_the_emitter(): + script = (ROOT / "tools/check-installed.py").read_text() + assert "microcosm_emitter_interface-" in script + assert "tools/installed-interface-smoke.py" in script diff --git a/tools/bump-version.py b/tools/bump-version.py index d31a7c7a..0f32d59b 100644 --- a/tools/bump-version.py +++ b/tools/bump-version.py @@ -1,8 +1,9 @@ """Choose the next release version from pending changelog fragments. A breaking fragment bumps the major version, an added or removed fragment the -minor version, and any other fragment the patch version. The version is read -from and written to the single version line in pyproject.toml. +minor version, and any other fragment the patch version. Both distributions +share one version: each pyproject.toml holds exactly one version line, and the +emitter pins the interface release of the same version. Prints the next version. With --check, writes nothing. """ @@ -18,7 +19,12 @@ + ", ".join(FRAGMENT_TYPES) ) VERSIONED_FILE = "pyproject.toml" +INTERFACE_VERSIONED_FILE = "packages/microcosm-emitter-interface/pyproject.toml" +VERSIONED_FILES = (VERSIONED_FILE, INTERFACE_VERSIONED_FILE) PYPROJECT_VERSION = re.compile(r'^version = "(\d+\.\d+\.\d+)"$', re.MULTILINE) +INTERFACE_PIN = re.compile( + r'^ "microcosm-emitter-interface==([^"]+)",$', re.MULTILINE +) def fragment_types(changelog_dir: Path) -> set[str]: @@ -51,29 +57,52 @@ def next_version(version: str, types: set[str]) -> str: return f"{major}.{minor}.{patch + 1}" -def current_version(root: Path) -> str: - versions = PYPROJECT_VERSION.findall((root / VERSIONED_FILE).read_text()) +def _file_version(root: Path, versioned_file: str) -> str: + versions = PYPROJECT_VERSION.findall((root / versioned_file).read_text()) if len(versions) != 1: raise ValueError( - f"Expected one version line in {VERSIONED_FILE}, found {versions}." + f"Expected one version line in {versioned_file}, found {versions}." ) return versions[0] +def current_version(root: Path) -> str: + versions = { + versioned_file: _file_version(root, versioned_file) + for versioned_file in VERSIONED_FILES + } + if len(set(versions.values())) != 1: + raise ValueError(f"Both distributions must have the same version: {versions}.") + version = versions[VERSIONED_FILE] + pins = INTERFACE_PIN.findall((root / VERSIONED_FILE).read_text()) + if pins != [version]: + raise ValueError( + f"{VERSIONED_FILE} must pin microcosm-emitter-interface=={version} " + f"exactly once, found {pins}." + ) + return version + + def plan_release(root: Path) -> tuple[str, str]: current = current_version(root) return current, next_version(current, fragment_types(root / "changelog.d")) def write_version(root: Path, current: str, upcoming: str) -> None: - path = root / VERSIONED_FILE - path.write_text( - PYPROJECT_VERSION.sub( + for versioned_file in VERSIONED_FILES: + path = root / versioned_file + text = PYPROJECT_VERSION.sub( lambda match: match.group(0).replace(current, upcoming), path.read_text(), count=1, ) - ) + if versioned_file == VERSIONED_FILE: + text = INTERFACE_PIN.sub( + lambda match: match.group(0).replace(current, upcoming), + text, + count=1, + ) + path.write_text(text) def main(arguments: list[str]) -> None: diff --git a/tools/check-artifacts.sh b/tools/check-artifacts.sh index 5d18b72e..1dbeb242 100644 --- a/tools/check-artifacts.sh +++ b/tools/check-artifacts.sh @@ -3,8 +3,12 @@ set -euo pipefail root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" cd "$root" -# Build the wheel from its source archive to check that both libraries and -# Alembic migrations are included. Remove artifacts from earlier builds first. -uv build --out-dir dist --clear +# Build each distribution's wheel from its source archive to check that the +# libraries, Alembic migrations and protocol fixtures are included. Remove +# artifacts from earlier builds first, once: with --all-packages, uv's option +# to clear the output directory empties it for each package, so one build +# could delete the other's artifacts. +rm -rf dist +uv build --all-packages --out-dir dist uv run --no-sync twine check --strict dist/* uv run --no-sync python tools/check-installed.py diff --git a/tools/check-installed.py b/tools/check-installed.py index 011d9795..6ae4aa4e 100644 --- a/tools/check-installed.py +++ b/tools/check-installed.py @@ -1,4 +1,9 @@ -"""Verify clean, non-editable installs away from the source checkout.""" +"""Verify clean, non-editable installs away from the source checkout. + +Two environments mirror production: the emitter's own, as +`uv tool install microcosm-emitter` creates it, and a build's, which holds only +the interface package and starts the emitter's command as its child. +""" import os import subprocess @@ -7,25 +12,43 @@ from pathlib import Path ROOT = Path(__file__).resolve().parents[1] +EMITTER_PREFIX = "microcosm_emitter-" +INTERFACE_PREFIX = "microcosm_emitter_interface-" def run(*arguments, **kwargs): subprocess.run(arguments, check=True, **kwargs) +def only(paths: list[Path], prefix: str) -> Path: + matches = [path for path in paths if path.name.startswith(prefix)] + assert len(matches) == 1, f"Expected one {prefix}* artifact, found {matches}" + return matches[0] + + +def create_environment(directory: Path, *wheels: Path) -> Path: + run("uv", "venv", "--python", sys.executable, str(directory)) + interpreter = directory / "bin/python" + run("uv", "pip", "install", "--python", str(interpreter), *map(str, wheels)) + return interpreter + + def main(): with tempfile.TemporaryDirectory(prefix="pe-wheel-") as directory: workspace = Path(directory) - interpreter = workspace / "venv/bin/python" - run("uv", "venv", "--python", sys.executable, str(workspace / "venv")) wheels = list((ROOT / "dist").glob("*.whl")) sources = list((ROOT / "dist").glob("*.tar.gz")) - assert len(wheels) == len(sources) == 1, ( - "Expected exactly one emitter wheel and source archive" + assert len(wheels) == len(sources) == 2, ( + "Expected an emitter and an interface wheel and source archive" + ) + for prefix in (EMITTER_PREFIX, INTERFACE_PREFIX): + only(sources, prefix) + emitter_wheel = only(wheels, EMITTER_PREFIX) + interface_wheel = only(wheels, INTERFACE_PREFIX) + emitter_interpreter = create_environment( + workspace / "emitter-tool", emitter_wheel, interface_wheel ) - assert wheels[0].name.startswith("microcosm_emitter-") - assert sources[0].name.startswith("microcosm_emitter-") - run("uv", "pip", "install", "--python", str(interpreter), str(wheels[0])) + build_interpreter = create_environment(workspace / "build", interface_wheel) environment = { key: value for key, value in os.environ.items() @@ -35,18 +58,27 @@ def main(): "HUGGINGFACE_TOKEN", "HUGGING_FACE_HUB_TOKEN", "PYTHONPATH", + "MICROCOSM_EMITTER_COMMAND", } } environment.update( HF_HOME=str(workspace / "hf"), HF_TOKEN_PATH=str(workspace / "hf/token") ) run( - str(interpreter), + str(emitter_interpreter), "-I", str(ROOT / "tools/installed-smoke.py"), cwd=workspace, env=environment, ) + run( + str(build_interpreter), + "-I", + str(ROOT / "tools/installed-interface-smoke.py"), + str(workspace / "emitter-tool/bin/microcosm-emitter"), + cwd=workspace, + env=environment, + ) if __name__ == "__main__": diff --git a/tools/installed-interface-smoke.py b/tools/installed-interface-smoke.py new file mode 100644 index 00000000..0e2b9adb --- /dev/null +++ b/tools/installed-interface-smoke.py @@ -0,0 +1,56 @@ +"""Start the separately installed emitter from a build environment without it.""" + +import sys +from importlib.metadata import distribution +from importlib.util import find_spec +from pathlib import Path + + +def main(emitter_command: str) -> None: + assert find_spec("microcosm_emitter") is None, "the build must not hold it" + interface = distribution("microcosm-emitter-interface") + assert not interface.requires, interface.requires + from microcosm_emitter_interface import ( + EmitterClient, + OrrerySettings, + TelemetryRun, + TelemetrySettings, + ) + from microcosm_emitter_interface.protocol_fixtures import protocol_fixtures + + assert protocol_fixtures(1), "protocol fixtures must ship in the wheel" + directory = Path.cwd() / "interface-graph" + directory.mkdir() + (directory / "graph.orrery.json").write_text('{"synthetic":true}') + client = EmitterClient.start( + telemetry=TelemetrySettings( + run=TelemetryRun("interface-wheel", "UK", "installed"), + development_collector_url="http://127.0.0.1:1", + ), + orrery=OrrerySettings( + development_collector_url="http://127.0.0.1:1", + publication_origin="http://127.0.0.1:1", + ), + spool_path=Path.cwd() / "interface.sqlite3", + command=emitter_command, + # A clean install compiles dependency bytecode on first use. + startup_timeout=15, + ) + assert client.available, client.startup_error + try: + assert client.modules == {"telemetry", "orrery"} + assert client.emitter_version == interface.version + assert client.telemetry.transition_stage("compile") + assert client.telemetry.complete() + inventory = client.orrery_inventory(directory, {"graph.orrery.json": "graph"}) + assert inventory is not None + receipt = client.hand_over_orrery_inputs(directory, inventory, wait_seconds=0) + assert receipt["status"] == "pending", receipt + finally: + process = client._process + client.close() + assert process is not None and process.process.wait(timeout=20) == 0 + + +if __name__ == "__main__": + main(sys.argv[1]) diff --git a/tools/update-version.sh b/tools/update-version.sh index f73adfe3..114f5995 100644 --- a/tools/update-version.sh +++ b/tools/update-version.sh @@ -1,9 +1,10 @@ #!/usr/bin/env bash set -euo pipefail -# Release the Python distribution: bump pyproject.toml from the pending fragment -# types, collect the fragments into CHANGELOG.md, and record the emitter's own -# new version in uv.lock for `uv sync --locked`. +# Release both Python distributions: bump both pyproject.toml files and the +# emitter's interface pin from the pending fragment types, collect the +# fragments into CHANGELOG.md, and record both new versions in uv.lock for +# `uv sync --locked`. version="$(uv run --no-project python tools/bump-version.py)" uvx --from "towncrier==26.9.0" towncrier build --yes --version "$version" uv lock diff --git a/uv.lock b/uv.lock index 386193ec..dc1afcbe 100644 --- a/uv.lock +++ b/uv.lock @@ -2,6 +2,12 @@ version = 1 revision = 5 requires-python = ">=3.13" +[manifest] +members = [ + "microcosm-emitter", + "microcosm-emitter-interface", +] + [[package]] name = "alembic" version = "1.20.0" @@ -606,6 +612,7 @@ dependencies = [ { name = "alembic" }, { name = "httpx" }, { name = "huggingface-hub" }, + { name = "microcosm-emitter-interface" }, { name = "psutil" }, { name = "sqlalchemy" }, ] @@ -624,6 +631,7 @@ requires-dist = [ { name = "alembic", specifier = ">=1.13.3,<2" }, { name = "httpx", specifier = ">=0.27,<1" }, { name = "huggingface-hub", specifier = ">=0.20,<2" }, + { name = "microcosm-emitter-interface", editable = "packages/microcosm-emitter-interface" }, { name = "psutil", specifier = ">=6,<8" }, { name = "sqlalchemy", specifier = ">=2,<3" }, ] @@ -637,6 +645,11 @@ dev = [ { name = "twine", specifier = ">=7,<8" }, ] +[[package]] +name = "microcosm-emitter-interface" +version = "0.4.0" +source = { editable = "packages/microcosm-emitter-interface" } + [[package]] name = "more-itertools" version = "11.1.0"