From b9e8b680feec9a6b4f7483d307bd96165c362087 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Fri, 9 Oct 2026 00:11:32 +0400 Subject: [PATCH 1/5] Separate telemetry and graph publication in one local provider service --- .github/workflows/ci.yml | 18 +- .github/workflows/publish.yml | 19 + .gitignore | 3 +- README.md | 87 +++- changelog.d/3.added.md | 1 + conftest.py | 2 + package.json | 15 + .../src/microcosm_provider_client/__main__.py | 22 +- .../src/microcosm_provider_client/client.py | 23 ++ .../microcosm_provider_client/contracts.py | 3 +- .../src/microcosm_provider_client/launcher.py | 13 +- .../src/microcosm_provider_client/runtime.py | 61 ++- .../tests/test_multiple_modules.py | 103 +++++ packages/provider-core/README.md | 11 + packages/provider-core/pyproject.toml | 18 + .../src/microcosm_provider_core/__init__.py | 1 + .../microcosm_provider_core}/alembic/env.py | 2 +- .../alembic/script.py.mako | 0 .../versions/v20261007_01_initial_spool.py | 0 .../v20261008_02_graph_publication.py | 27 ++ .../src/microcosm_provider_core/auth.py | 169 ++++++++ .../src/microcosm_provider_core}/constants.py | 0 .../src/microcosm_provider_core}/database.py | 2 +- .../microcosm_provider_core}/migrations.py | 2 +- .../src/microcosm_provider_core}/models.py | 17 +- packages/provider-orrery/README.md | 7 + packages/provider-orrery/pyproject.toml | 28 ++ .../src/microcosm_provider_orrery/__init__.py | 1 + .../src/microcosm_provider_orrery/cli.py | 58 +++ .../src/microcosm_provider_orrery/client.py | 47 +++ .../service/__init__.py | 8 + .../service/module.py | 55 +++ .../service/queue.py | 378 ++++++++++++++++++ packages/provider-orrery/tests/test_client.py | 35 ++ .../provider-orrery/tests/test_publication.py | 244 +++++++++++ packages/provider-telemetry/pyproject.toml | 2 + .../service/collector.py | 159 ++------ .../service/module.py | 26 +- .../service/spool.py | 16 +- .../provider-telemetry/tests/test_delivery.py | 28 +- .../tests/test_persistence.py | 7 +- tests/test_boundaries.py | 5 +- tests/test_package_names.py | 7 +- tests/test_queue_compatibility.py | 4 +- tests/test_workflows.py | 18 + tools/check-installed.py | 23 +- tools/check-reader-installed.ts | 65 +++ tools/check-reader.sh | 11 + tools/check-release.py | 5 +- tools/installed-smoke.py | 55 ++- tools/publish-reader.sh | 10 + typescript/orrery/README.md | 13 + typescript/orrery/bun.lock | 88 ++++ typescript/orrery/package.json | 46 +++ typescript/orrery/src/browser.ts | 98 +++++ typescript/orrery/src/contract.ts | 130 ++++++ typescript/orrery/src/server.ts | 52 +++ typescript/orrery/tests/contract.test.ts | 43 ++ typescript/orrery/tests/reader.test.ts | 160 ++++++++ typescript/orrery/tsconfig.build.json | 11 + typescript/orrery/tsconfig.json | 13 + uv.lock | 38 ++ 62 files changed, 2393 insertions(+), 220 deletions(-) create mode 100644 changelog.d/3.added.md create mode 100644 package.json create mode 100644 packages/provider-client/tests/test_multiple_modules.py create mode 100644 packages/provider-core/README.md create mode 100644 packages/provider-core/pyproject.toml create mode 100644 packages/provider-core/src/microcosm_provider_core/__init__.py rename packages/{provider-telemetry/src/microcosm_provider_telemetry/service => provider-core/src/microcosm_provider_core}/alembic/env.py (91%) rename packages/{provider-telemetry/src/microcosm_provider_telemetry/service => provider-core/src/microcosm_provider_core}/alembic/script.py.mako (100%) rename packages/{provider-telemetry/src/microcosm_provider_telemetry/service => provider-core/src/microcosm_provider_core}/alembic/versions/v20261007_01_initial_spool.py (100%) create mode 100644 packages/provider-core/src/microcosm_provider_core/alembic/versions/v20261008_02_graph_publication.py create mode 100644 packages/provider-core/src/microcosm_provider_core/auth.py rename packages/{provider-telemetry/src/microcosm_provider_telemetry/service => provider-core/src/microcosm_provider_core}/constants.py (100%) rename packages/{provider-telemetry/src/microcosm_provider_telemetry/service => provider-core/src/microcosm_provider_core}/database.py (93%) rename packages/{provider-telemetry/src/microcosm_provider_telemetry/service => provider-core/src/microcosm_provider_core}/migrations.py (97%) rename packages/{provider-telemetry/src/microcosm_provider_telemetry/service => provider-core/src/microcosm_provider_core}/models.py (82%) create mode 100644 packages/provider-orrery/README.md create mode 100644 packages/provider-orrery/pyproject.toml create mode 100644 packages/provider-orrery/src/microcosm_provider_orrery/__init__.py create mode 100644 packages/provider-orrery/src/microcosm_provider_orrery/cli.py create mode 100644 packages/provider-orrery/src/microcosm_provider_orrery/client.py create mode 100644 packages/provider-orrery/src/microcosm_provider_orrery/service/__init__.py create mode 100644 packages/provider-orrery/src/microcosm_provider_orrery/service/module.py create mode 100644 packages/provider-orrery/src/microcosm_provider_orrery/service/queue.py create mode 100644 packages/provider-orrery/tests/test_client.py create mode 100644 packages/provider-orrery/tests/test_publication.py create mode 100644 tools/check-reader-installed.ts create mode 100644 tools/check-reader.sh create mode 100644 tools/publish-reader.sh create mode 100644 typescript/orrery/README.md create mode 100644 typescript/orrery/bun.lock create mode 100644 typescript/orrery/package.json create mode 100644 typescript/orrery/src/browser.ts create mode 100644 typescript/orrery/src/contract.ts create mode 100644 typescript/orrery/src/server.ts create mode 100644 typescript/orrery/tests/contract.test.ts create mode 100644 typescript/orrery/tests/reader.test.ts create mode 100644 typescript/orrery/tsconfig.build.json create mode 100644 typescript/orrery/tsconfig.json diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e22dcfa9..1c6d8c91 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -85,13 +85,27 @@ jobs: path: dist - run: bash tools/check-collector.sh + reader: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2 + with: + bun-version: "1.4.2" + - run: bash tools/check-reader.sh + - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: orrery-reader + path: reader-dist/*.tgz + if-no-files-found: error + required: name: Required checks if: always() - needs: [quality, tests, artifacts, collector] + needs: [quality, tests, artifacts, collector, reader] runs-on: ubuntu-latest steps: - name: Require all validation jobs to succeed env: RESULTS: ${{ join(needs.*.result, ' ') }} - run: test "$RESULTS" = "success success success success" + run: test "$RESULTS" = "success success success success success" diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 231393fd..6a7b5bf1 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -40,3 +40,22 @@ jobs: - uses: pypa/gh-action-pypi-publish@dc37677b2e1c63e2034f94d8a5b11f265b73ba33 # release/v1 with: packages-dir: dist/ + + publish-reader: + needs: [verify, publish] + runs-on: ubuntu-latest + environment: pypi + permissions: + contents: read + id-token: write + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 + with: + node-version: "24" + registry-url: "https://registry.npmjs.org" + - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 + with: + name: orrery-reader + path: reader-dist + - run: bash tools/publish-reader.sh diff --git a/.gitignore b/.gitignore index f54261ab..85b6be4e 100644 --- a/.gitignore +++ b/.gitignore @@ -7,4 +7,5 @@ dist/ .coverage .collector/ .test-artifacts/ - +node_modules/ +*.tgz diff --git a/README.md b/README.md index 01092a31..10fe1a7b 100644 --- a/README.md +++ b/README.md @@ -1,37 +1,96 @@ # Microcosm local provider -Two Python distributions separate the local process host from telemetry behavior: +One local service process runs independent telemetry and completed-graph workers. +The workers share authentication and a durable database, but neither requires the +other to be active. Graph publication does not require a telemetry run ID. -- `microcosm-provider-client` owns subprocess startup, a private Unix socket, explicit module loading, parent-process monitoring, and shutdown. It has no telemetry, database, or authentication dependencies. -- `microcosm-provider-telemetry` provides the build-facing client and a separate service module. The service module owns resource sampling, SQLAlchemy ORM persistence, Alembic migrations, authentication, and collector delivery. +## Packages -The build process imports the client, not the service implementation. It supplies run metadata, its existing queue path, and build identity. Each build starts one local telemetry emitter service automatically. Imports alone create no processes, sockets, files, or network requests. +- `microcosm-provider-client`: subprocess lifecycle, private Unix socket, explicit + module selection, parent monitoring and bounded shutdown. No domain, database + or authentication imports. +- `microcosm-provider-core`: shared collector authentication, SQLAlchemy models + and Alembic migrations. The collector protocol is unchanged. +- `microcosm-provider-telemetry`: build-facing telemetry client and service module. +- `microcosm-provider-orrery`: explicit graph-file inventory, durable publication + queue, service module and `microcosm-provider-publish-graph` retry command. +- `@policyengine/microcosm-provider-orrery`: TypeScript graph readers. `/browser` + fetches and verifies the Orrery document, `/server` reads completed publications + through a supplied storage adapter, and `/contract` validates public metadata. + The React component, HTTP endpoints and storage credentials remain in the runs app. -## Delivery and persistence +Imports alone create no processes, sockets, files or network requests. Microcosm +supplies build identity and registers both workers using `start_services`. The +generic host creates separate workers and routes messages to the selected module; +one worker's slow HTTP request or failure does not stop the other worker. -The service commits each event before acknowledging its local socket message. The client bounds socket calls and reports telemetry failures without failing the build. Network requests run on the service worker, outside the message-handling lock. +## Persistence and delivery -The service uses the existing Hugging Face credential from the environment or local login cache. The collector validates PolicyEngine membership before issuing a short-lived write credential. Missing or rejected credentials permanently mark that producer's events as local-only. Transient collector failures retain events for retry. The production destination belongs to the package; development overrides accept loopback addresses only. +Telemetry commits each event before acknowledging its socket message. Missing or +rejected credentials mark that producer's telemetry as local-only, preserving the +existing policy. In contrast, graph jobs remain eligible for retry after missing, +rejected or temporarily unavailable credentials. They retain the exact inventory, +preserved file paths, upload progress and publication ID across restarts. The graph +client commits a job before waiting for the service; publication is immutable. +Signed storage uploads never receive the collector bearer credential. -The queue retains the current Microcosm Alembic revision, producer sequences, event IDs, and upload eligibility. Unversioned databases are rejected without schema modification. There is no raw SQL, automatic schema adoption, or parallel legacy implementation. The hosted collector and dashboard do not change. +The database retains Microcosm's existing revisions and event identities. Alembic +upgrades revision `20261007_01` to `20261008_02` by adding graph jobs. Unversioned +databases are rejected without modification; upgrade them using the existing +Microcosm implementation before migrating. No raw SQL or schema adoption is used. + +## Configuration + +No new runtime environment variables or secrets are introduced. The service reads +`HF_TOKEN`, then `HUGGINGFACE_TOKEN`, then the existing Hugging Face login cache. +It exchanges that credential with the existing collector; both workers share the +short-lived result. The tracked production destinations are in core constants and +the Orrery queue module. Collector development overrides accept loopback only. +`XDG_CACHE_HOME` optionally changes the existing cache root; the default database +is `~/.cache/microcosm/telemetry/events.sqlite3`. Tests replace credentials and +destinations with synthetic values and loopback or in-memory transports. + +The reader package needs no environment variables. Its server adapter receives +storage access from the runs app, which continues to own Vercel Blob configuration, +publication authorization and routes. This change does not modify the calibration +dashboard or deploy either hosted service. ## Development -Use Python 3.13 or 3.14 on Linux or macOS. +Use Python 3.13 or 3.14 on Linux or macOS, and Bun 1.4.2 for TypeScript. ```bash uv sync --all-packages --locked bash tools/check-quality.sh uv run --no-sync pytest -m 'not collector' bash tools/check-artifacts.sh +bash tools/check-reader.sh ``` -Tests isolate ambient credentials and use synthetic identities. The artifact check builds source archives and wheels, installs the runtime by itself, then installs telemetry and exercises the real service process and migrations. +Python artifact checks build and validate all four source archives and wheels, +install the generic runtime alone, then exercise installed telemetry and combined +telemetry/publication processes. Reader checks install the built tarball into a +separate directory and verify that the browser entry bundles without Node imports. +All new tests use local synthetic data. The pre-existing collector compatibility +job is retained; it requires a pinned checkout and disposable loopback PostgreSQL. -The collector compatibility job tests installed wheels against a pinned dashboard collector and disposable PostgreSQL. It covers the HTTP exchange, event ingestion, dashboard run documents, rejected identities, and duplicate delivery after a lost acknowledgement. Local execution requires that pinned collector checkout and a loopback test database; `tools/check-collector.sh` verifies these inputs. Never point this test at a deployed database. +## Release and deferred adoption -## Publication +All five packages use version `0.1.0`. A published GitHub release verifies tag +versions and main-branch ancestry, runs CI, then publishes exactly the tested +Python artifacts through the protected `pypi` environment. Only after that succeeds +does the reader job publish the tested npm tarball using OIDC and the same approval +environment. No long-lived registry token is committed or required by the workflow. -Both distributions use the same release version. A published GitHub release triggers version and main-branch ancestry checks, all CI checks, and artifact verification. Only then can the protected `pypi` environment approve publishing the exact tested artifacts. The publish job uses PyPI Trusted Publishing; no long-lived publishing token is supplied through CI. +The GitHub `pypi` environment and its approval/tag policy have been verified. Four +PyPI trusted-publisher bindings and the scoped npm package's first-publication and +trusted-publisher setup require registry-owner confirmation before release; those +external bindings have not been verified. The workflow cannot create them. Both +registries must bind `PolicyEngine/microcosm-local-provider`, `publish.yml` and +environment `pypi`. No release or consumer activation is authorized by this PR. -First publication requires two PyPI pending trusted publishers, one for each distribution, attached to this project's `publish.yml` workflow and `pypi` environment. That external setup must be verified before releasing. No release is created or merged automatically by this project. +The draft Microcosm and runs-app consumer PRs can pin this branch's exact Git commit +to install and test before publication. The root private npm manifest exists only +for that source-based test path. Before merging either consumer, publish these +packages, replace Git overrides with registry versions, regenerate lockfiles and +rerun its tests. Neither draft deploys, merges or activates automatically. diff --git a/changelog.d/3.added.md b/changelog.d/3.added.md new file mode 100644 index 00000000..6df38cd5 --- /dev/null +++ b/changelog.d/3.added.md @@ -0,0 +1 @@ +Add independent completed-graph publication and verified Orrery document readers alongside telemetry in the local provider. diff --git a/conftest.py b/conftest.py index cac810d2..aa2a8513 100644 --- a/conftest.py +++ b/conftest.py @@ -15,6 +15,7 @@ def isolate_credentials_and_network(tmp_path, monkeypatch): from huggingface_hub import constants monkeypatch.setattr(constants, "HF_TOKEN_PATH", str(hf_home / "token")) + from microcosm_provider_core import auth from microcosm_provider_telemetry.service import collector real_post = collector._http_post @@ -25,3 +26,4 @@ def loopback_post(url, *args, **kwargs): return real_post(url, *args, **kwargs) monkeypatch.setattr(collector, "_http_post", loopback_post) + monkeypatch.setattr(auth, "_http_post", loopback_post) diff --git a/package.json b/package.json new file mode 100644 index 00000000..711114cc --- /dev/null +++ b/package.json @@ -0,0 +1,15 @@ +{ + "name": "@policyengine/microcosm-provider-orrery", + "version": "0.1.0", + "private": true, + "type": "module", + "description": "Pinned-source development entry points; publish the built package from typescript/orrery", + "exports": { + "./contract": "./typescript/orrery/src/contract.ts", + "./browser": "./typescript/orrery/src/browser.ts", + "./server": "./typescript/orrery/src/server.ts" + }, + "peerDependencies": { + "@axiom-foundation/orrery": "0.6.1" + } +} diff --git a/packages/provider-client/src/microcosm_provider_client/__main__.py b/packages/provider-client/src/microcosm_provider_client/__main__.py index a0a2fbb0..ef7c521f 100644 --- a/packages/provider-client/src/microcosm_provider_client/__main__.py +++ b/packages/provider-client/src/microcosm_provider_client/__main__.py @@ -13,9 +13,11 @@ def main() -> int: parser = argparse.ArgumentParser() - parser.add_argument("--module", required=True) + selection = parser.add_mutually_exclusive_group(required=True) + selection.add_argument("--module") + selection.add_argument("--modules-json") parser.add_argument("--socket", type=Path, required=True) - parser.add_argument("--config-json", required=True) + parser.add_argument("--config-json", default="{}") parser.add_argument("--parent-pid", type=int, required=True) parser.add_argument("--parent-created-at", type=float, required=True) args = parser.parse_args() @@ -24,8 +26,20 @@ def main() -> int: if not isinstance(configuration, dict): raise ValueError("configuration must be an object") parent = ParentProcess(args.parent_pid, args.parent_created_at) - module = load_module(args.module, configuration, ModuleContext(args.parent_pid)) - ServiceRuntime(args.socket, module, is_parent_alive=parent.alive).run() + configurations = ( + json.loads(args.modules_json) + if args.modules_json + else {args.module: configuration} + ) + if not isinstance(configurations, dict) or not configurations: + raise ValueError("modules must be a non-empty object") + context = ModuleContext(args.parent_pid) + modules = {} + for name, settings in configurations.items(): + if not isinstance(name, str) or not isinstance(settings, dict): + raise ValueError("invalid module configuration") + modules[name] = load_module(name, settings, context) + ServiceRuntime(args.socket, modules=modules, is_parent_alive=parent.alive).run() return 0 except Exception as error: # Never include configuration or module exception payloads in stderr. diff --git a/packages/provider-client/src/microcosm_provider_client/client.py b/packages/provider-client/src/microcosm_provider_client/client.py index 2c259267..06688d14 100644 --- a/packages/provider-client/src/microcosm_provider_client/client.py +++ b/packages/provider-client/src/microcosm_provider_client/client.py @@ -1,5 +1,7 @@ """Bounded, private Unix-socket transport without domain-specific behavior.""" +from __future__ import annotations + import json import socket from pathlib import Path @@ -46,6 +48,10 @@ def _request(self, payload: JsonObject) -> None: def ping(self) -> None: self._request({"action": "ping"}) + def for_module(self, name: str) -> ModuleClient: + """Route messages without giving a component ownership of host shutdown.""" + return ModuleClient(self, name) + def send(self, message: JsonObject) -> None: if self.closed: raise LocalServiceError("client is closed") @@ -58,3 +64,20 @@ def close(self) -> None: self._request({"action": "close"}) finally: self.closed = True + + +class ModuleClient: + """One selected module's transport; the enclosing ServiceHandle owns shutdown.""" + + def __init__(self, client: SocketClient, name: str): + self.client, self.name = client, name + + def send(self, message: JsonObject) -> None: + if self.client.closed: + raise LocalServiceError("client is closed") + self.client._request( + {"action": "message", "module": self.name, "message": message} + ) + + def close(self) -> None: + """A domain component cannot close the shared host.""" diff --git a/packages/provider-client/src/microcosm_provider_client/contracts.py b/packages/provider-client/src/microcosm_provider_client/contracts.py index e7f2aa7c..a12f9ff7 100644 --- a/packages/provider-client/src/microcosm_provider_client/contracts.py +++ b/packages/provider-client/src/microcosm_provider_client/contracts.py @@ -1,7 +1,7 @@ """Interfaces shared by a local service host and its selected module.""" from collections.abc import Mapping -from dataclasses import dataclass +from dataclasses import dataclass, field from typing import Any, Protocol type JsonObject = Mapping[str, Any] @@ -12,6 +12,7 @@ class ModuleContext: """Identity of the process whose lifetime owns this service.""" parent_pid: int + services: dict[str, Any] = field(default_factory=dict) class ServiceModule(Protocol): diff --git a/packages/provider-client/src/microcosm_provider_client/launcher.py b/packages/provider-client/src/microcosm_provider_client/launcher.py index 46265e78..94e9678b 100644 --- a/packages/provider-client/src/microcosm_provider_client/launcher.py +++ b/packages/provider-client/src/microcosm_provider_client/launcher.py @@ -50,6 +50,13 @@ def _reap(self) -> None: def start_service( module: str, configuration: JsonObject, *, startup_timeout: float = STARTUP_TIMEOUT +) -> ServiceHandle: + """Start one selected module using the multi-module host.""" + return start_services({module: configuration}, startup_timeout=startup_timeout) + + +def start_services( + modules: JsonObject, *, startup_timeout: float = STARTUP_TIMEOUT ) -> ServiceHandle: """Return after readiness, or reap the child and remove temporary files.""" directory = None @@ -65,12 +72,10 @@ def start_service( sys.executable, "-m", MODULE_HOST, - "--module", - module, + "--modules-json", + json.dumps(modules, allow_nan=False), "--socket", str(path), - "--config-json", - json.dumps(configuration, allow_nan=False), "--parent-pid", str(parent.pid), "--parent-created-at", diff --git a/packages/provider-client/src/microcosm_provider_client/runtime.py b/packages/provider-client/src/microcosm_provider_client/runtime.py index 787a974b..d9f7a78e 100644 --- a/packages/provider-client/src/microcosm_provider_client/runtime.py +++ b/packages/provider-client/src/microcosm_provider_client/runtime.py @@ -1,4 +1,4 @@ -"""Host one explicitly selected service module for one parent process.""" +"""Host explicitly selected modules with independent periodic workers.""" import json import os @@ -26,28 +26,38 @@ class ServiceRuntime: def __init__( self, socket_path: Path, - module: ServiceModule, + module: ServiceModule | None = None, *, + modules: Mapping[str, ServiceModule] | None = None, is_parent_alive: Callable[[], bool], tick_seconds: float = TICK_SECONDS, drain_seconds: float = DRAIN_SECONDS, ): self.socket_path = socket_path + if (module is None) == (modules is None): + raise ValueError("Select one module or an explicit module mapping.") + self.modules = dict(modules) if modules is not None else {"default": module} + if not self.modules: + raise ValueError("At least one module is required.") self.module = module self.is_parent_alive = is_parent_alive self.tick_seconds = tick_seconds self.drain_seconds = drain_seconds self._stop = threading.Event() self._worker_error: Exception | None = None + self._failed: set[str] = set() def stop(self) -> None: self._stop.set() def run(self) -> None: - worker = None + workers = {} + initialized = [] bound = False try: - self.module.initialize() + for module in self.modules.values(): + initialized.append(module) + module.initialize() with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as server: # The launcher owns a fresh private directory. Never unlink an # existing socket which might belong to another running host. @@ -56,11 +66,17 @@ def run(self) -> None: os.chmod(self.socket_path, 0o600) server.listen(16) server.settimeout(SOCKET_TIMEOUT) - worker = threading.Thread(target=self._work, daemon=True) - worker.start() + for name, module in self.modules.items(): + worker = threading.Thread( + target=self._work, args=(name, module), daemon=True + ) + workers[name] = worker + worker.start() while not self._stop.is_set(): if not self.is_parent_alive(): - self.module.parent_exited() + for name, module in self.modules.items(): + if name not in self._failed: + module.parent_exited() break try: connection, _ = server.accept() @@ -75,12 +91,14 @@ def run(self) -> None: pass finally: self.stop() - if worker is not None: - worker.join(timeout=self.drain_seconds + SOCKET_TIMEOUT) + deadline = time.monotonic() + self.drain_seconds + SOCKET_TIMEOUT + for worker in workers.values(): + worker.join(timeout=max(0, deadline - time.monotonic())) if worker.is_alive(): self._worker_error = TimeoutError("Service shutdown timed out.") - else: - self.module.close(time.monotonic() + self.drain_seconds) + if not workers: + for module in initialized: + module.close(deadline) if bound: self.socket_path.unlink(missing_ok=True) if self._worker_error is not None: @@ -108,7 +126,12 @@ def _respond(self, connection: socket.socket) -> bytes: message = request.get("message") if not isinstance(message, Mapping): raise ValueError("message must be an object") - self.module.handle_message(message) + name = request.get("module") + if name is None and len(self.modules) == 1: + name = next(iter(self.modules)) + if name not in self.modules or name in self._failed: + raise ValueError("module is unavailable") + self.modules[name].handle_message(message) elif action == "close": self.stop() elif action != "ping": @@ -118,16 +141,20 @@ def _respond(self, connection: socket.socket) -> bytes: return ACK_ERROR return ACK_OK - def _work(self) -> None: + def _work(self, name: str, module: ServiceModule) -> None: try: while not self._stop.wait(self.tick_seconds): - self.module.tick(time.monotonic()) + module.tick(time.monotonic()) except Exception as error: self._worker_error = error - self.stop() + self._failed.add(name) + if len(self._failed) == len(self.modules): + self.stop() finally: try: - self.module.close(time.monotonic() + self.drain_seconds) + module.close(time.monotonic() + self.drain_seconds) except Exception as error: self._worker_error = error - self.stop() + self._failed.add(name) + if len(self._failed) == len(self.modules): + self.stop() diff --git a/packages/provider-client/tests/test_multiple_modules.py b/packages/provider-client/tests/test_multiple_modules.py new file mode 100644 index 00000000..bf5ab0f6 --- /dev/null +++ b/packages/provider-client/tests/test_multiple_modules.py @@ -0,0 +1,103 @@ +"""One local host routes named modules and gives each its own worker.""" + +import threading +import time + +from microcosm_provider_client.client import SocketClient +from microcosm_provider_client.runtime import ServiceRuntime + + +class Module: + def __init__(self): + self.messages = [] + self.entered = threading.Event() + self.release = threading.Event() + self.fail = False + + def initialize(self): + pass + + def handle_message(self, message): + self.messages.append(dict(message)) + + def tick(self, now): + self.entered.set() + if self.fail: + raise ValueError("synthetic failure") + self.release.wait(2) + + def parent_exited(self): + pass + + def close(self, deadline): + pass + + +def test_slow_module_does_not_block_another_worker_or_socket(tmp_path): + # macOS needs short socket paths; reuse the runtime suite's fixture. + import tempfile + from pathlib import Path + + with tempfile.TemporaryDirectory(prefix="provider-test-", dir="/tmp") as directory: + path = Path(directory) / "e.sock" + slow, other = Module(), Module() + other.release.set() + runtime = ServiceRuntime( + path, + modules={"publication": slow, "telemetry": other}, + is_parent_alive=lambda: True, + tick_seconds=0.01, + ) + thread = threading.Thread(target=runtime.run) + thread.start() + try: + assert slow.entered.wait(2) + assert other.entered.wait(2) + client = SocketClient(path) + client.for_module("telemetry").send({"number": 1}) + assert other.messages == [{"number": 1}] + assert slow.messages == [] + client.ping() + finally: + slow.release.set() + runtime.stop() + thread.join(3) + assert not thread.is_alive() + + +def test_one_worker_failure_does_not_stop_another_module(tmp_path): + import tempfile + from pathlib import Path + + with tempfile.TemporaryDirectory(prefix="provider-test-", dir="/tmp") as directory: + path = Path(directory) / "e.sock" + broken, healthy = Module(), Module() + broken.fail = True + healthy.release.set() + runtime = ServiceRuntime( + path, + modules={"broken": broken, "healthy": healthy}, + is_parent_alive=lambda: True, + tick_seconds=0.01, + ) + errors = [] + + def run(): + try: + runtime.run() + except Exception as error: + errors.append(error) + + thread = threading.Thread(target=run) + thread.start() + try: + assert broken.entered.wait(2) + assert healthy.entered.wait(2) + time.sleep(0.05) + SocketClient(path).for_module("healthy").send({"accepted": True}) + assert healthy.messages == [{"accepted": True}] + finally: + runtime.stop() + thread.join(3) + assert not thread.is_alive() + assert errors diff --git a/packages/provider-core/README.md b/packages/provider-core/README.md new file mode 100644 index 00000000..f7b5c78a --- /dev/null +++ b/packages/provider-core/README.md @@ -0,0 +1,11 @@ +# Microcosm provider core + +Shared short-lived collector sessions and one SQLAlchemy/Alembic database history +for independently scheduled telemetry and graph publication. The generic process +runtime does not depend on this distribution. Telemetry and graph clients can +import without loading authentication or persistence; domain service modules use +these implementations in the child process. The graph enqueue API lazily imports +the durable local queue to preserve jobs even when the child is unavailable. + +Existing versioned Microcosm spools upgrade without changing event or publication +identities. Unversioned queues are rejected rather than automatically adopted. diff --git a/packages/provider-core/pyproject.toml b/packages/provider-core/pyproject.toml new file mode 100644 index 00000000..1d131b6a --- /dev/null +++ b/packages/provider-core/pyproject.toml @@ -0,0 +1,18 @@ +[project] +name = "microcosm-provider-core" +version = "0.1.0" +description = "Shared collector sessions and Alembic-managed local provider persistence" +readme = "README.md" +requires-python = ">=3.13" +license = "MIT" +dependencies = ["alembic>=1.13.3,<2", "sqlalchemy>=2,<3", "huggingface-hub>=0.20,<2"] + +[project.urls] +Source = "https://github.com/PolicyEngine/microcosm-local-provider" + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[tool.hatch.build.targets.wheel] +packages = ["src/microcosm_provider_core"] diff --git a/packages/provider-core/src/microcosm_provider_core/__init__.py b/packages/provider-core/src/microcosm_provider_core/__init__.py new file mode 100644 index 00000000..b92eeaef --- /dev/null +++ b/packages/provider-core/src/microcosm_provider_core/__init__.py @@ -0,0 +1 @@ +"""Shared provider services; importing this package has no runtime side effects.""" diff --git a/packages/provider-telemetry/src/microcosm_provider_telemetry/service/alembic/env.py b/packages/provider-core/src/microcosm_provider_core/alembic/env.py similarity index 91% rename from packages/provider-telemetry/src/microcosm_provider_telemetry/service/alembic/env.py rename to packages/provider-core/src/microcosm_provider_core/alembic/env.py index 8313449b..49154c14 100644 --- a/packages/provider-telemetry/src/microcosm_provider_telemetry/service/alembic/env.py +++ b/packages/provider-core/src/microcosm_provider_core/alembic/env.py @@ -1,7 +1,7 @@ """Alembic environment for the local telemetry spool.""" from alembic import context -from microcosm_provider_telemetry.service.models import SpoolModel +from microcosm_provider_core.models import SpoolModel from sqlalchemy.engine import Connection _MISSING_CONNECTION_ERROR = ( diff --git a/packages/provider-telemetry/src/microcosm_provider_telemetry/service/alembic/script.py.mako b/packages/provider-core/src/microcosm_provider_core/alembic/script.py.mako similarity index 100% rename from packages/provider-telemetry/src/microcosm_provider_telemetry/service/alembic/script.py.mako rename to packages/provider-core/src/microcosm_provider_core/alembic/script.py.mako diff --git a/packages/provider-telemetry/src/microcosm_provider_telemetry/service/alembic/versions/v20261007_01_initial_spool.py b/packages/provider-core/src/microcosm_provider_core/alembic/versions/v20261007_01_initial_spool.py similarity index 100% rename from packages/provider-telemetry/src/microcosm_provider_telemetry/service/alembic/versions/v20261007_01_initial_spool.py rename to packages/provider-core/src/microcosm_provider_core/alembic/versions/v20261007_01_initial_spool.py diff --git a/packages/provider-core/src/microcosm_provider_core/alembic/versions/v20261008_02_graph_publication.py b/packages/provider-core/src/microcosm_provider_core/alembic/versions/v20261008_02_graph_publication.py new file mode 100644 index 00000000..14ae0318 --- /dev/null +++ b/packages/provider-core/src/microcosm_provider_core/alembic/versions/v20261008_02_graph_publication.py @@ -0,0 +1,27 @@ +"""Add durable graph publication jobs without changing event retention.""" + +import sqlalchemy as sa +from alembic import op + +revision = "20261008_02" +down_revision = "20261007_01" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.create_table( + "graph_publication_jobs", + sa.Column("publication_id", sa.Text(), primary_key=True), + sa.Column("inventory", sa.Text(), nullable=False), + sa.Column("directory", sa.Text(), nullable=False), + sa.Column("status", sa.Text(), nullable=False), + sa.Column("attempts", sa.Integer(), nullable=False), + sa.Column("next_attempt_at", sa.Float(), nullable=False), + sa.Column("lease_until", sa.Float(), nullable=False), + sa.Column("receipt", sa.Text(), nullable=False), + ) + + +def downgrade() -> None: + op.drop_table("graph_publication_jobs") diff --git a/packages/provider-core/src/microcosm_provider_core/auth.py b/packages/provider-core/src/microcosm_provider_core/auth.py new file mode 100644 index 00000000..e6589178 --- /dev/null +++ b/packages/provider-core/src/microcosm_provider_core/auth.py @@ -0,0 +1,169 @@ +"""HF-to-collector session shared by telemetry and graph publication.""" + +from __future__ import annotations + +import json +import os +import time +import urllib.error +import urllib.request +from collections.abc import Mapping +from http import HTTPStatus +from typing import Any +from urllib.parse import urlsplit + +from huggingface_hub import get_token + +from microcosm_provider_core.constants import ( + COLLECTOR_RESPONSE_TOO_LARGE_ERROR, + COLLECTOR_URL_HTTPS_ERROR, + COLLECTOR_URL_ORIGIN_ERROR, + DEFAULT_TOKEN_LIFETIME_SECONDS, + DEVELOPMENT_COLLECTOR_LOOPBACK_ERROR, + HTTP_TIMEOUT_SECONDS, + HTTP_USER_AGENT, + LOOPBACK_HOSTS, + MAX_HTTP_RESPONSE_BYTES, + MINIMUM_TOKEN_LIFETIME_SECONDS, + TOKEN_EXCHANGE_PATH, + TOKEN_REFRESH_MARGIN_SECONDS, +) + + +class _NoRedirectHandler(urllib.request.HTTPRedirectHandler): + """Keep bearer credentials on the explicitly configured origin.""" + + def redirect_request(self, req, fp, code, msg, headers, newurl): + return None + + +def _collector_origin(value: str, *, allow_loopback_http: bool = False) -> str: + parsed = urlsplit(value) + loopback = parsed.hostname in LOOPBACK_HOSTS + valid_scheme = parsed.scheme == "https" or ( + allow_loopback_http and loopback and parsed.scheme == "http" + ) + if not parsed.hostname or not valid_scheme: + raise ValueError(COLLECTOR_URL_HTTPS_ERROR) + if ( + parsed.username + or parsed.password + or parsed.query + or parsed.fragment + or parsed.path not in {"", "/"} + ): + raise ValueError(COLLECTOR_URL_ORIGIN_ERROR) + return value.rstrip("/") + + +def _development_collector_url(value: str) -> str: + parsed = urlsplit(value) + if parsed.hostname not in LOOPBACK_HOSTS: + raise ValueError(DEVELOPMENT_COLLECTOR_LOOPBACK_ERROR) + return _collector_origin(value, allow_loopback_http=True) + + +def _decode_response(body: bytes) -> dict[str, Any]: + try: + response = json.loads(body) if body else {} + except (json.JSONDecodeError, UnicodeDecodeError): + return {} + return response if isinstance(response, dict) else {} + + +def _http_post( + url: str, + payload: Mapping[str, Any], + bearer_token: str, + *, + timeout: float = HTTP_TIMEOUT_SECONDS, +) -> tuple[int, dict[str, Any]]: + request = urllib.request.Request( + url, + data=json.dumps(payload, separators=(",", ":")).encode(), + headers={ + "Authorization": f"Bearer {bearer_token}", + "Content-Type": "application/json", + "User-Agent": HTTP_USER_AGENT, + }, + method="POST", + ) + try: + opener = urllib.request.build_opener(_NoRedirectHandler) + with opener.open(request, timeout=timeout) as response: + body = response.read(MAX_HTTP_RESPONSE_BYTES + 1) + if len(body) > MAX_HTTP_RESPONSE_BYTES: + raise OSError(COLLECTOR_RESPONSE_TOO_LARGE_ERROR) + return response.status, _decode_response(body) + except urllib.error.HTTPError as error: + body = error.read(MAX_HTTP_RESPONSE_BYTES + 1) + if len(body) > MAX_HTTP_RESPONSE_BYTES: + return error.code, {} + return error.code, _decode_response(body) + + +def _huggingface_token() -> str | None: + return ( + os.environ.get("HF_TOKEN", "").strip() + or os.environ.get("HUGGINGFACE_TOKEN", "").strip() + or get_token() + ) + + +class CollectorSession: + """One thread-safe short-lived credential shared by independent publishers.""" + + def __init__(self, origin: str, *, post=None, credential=None): + import threading + + self.origin = origin + self._post = post or _http_post + self._credential = credential or _huggingface_token + self._session_token: tuple[str, float] | None = None + self._lock = threading.Lock() + + def invalidate(self) -> None: + with self._lock: + self._session_token = None + + def credential(self) -> tuple[str, str | None]: + with self._lock: + if ( + self._session_token is not None + and self._session_token[1] + > time.monotonic() + TOKEN_REFRESH_MARGIN_SECONDS + ): + return "ok", self._session_token[0] + hf_token = self._credential() + if not hf_token: + return "missing", None + try: + status, response = self._post( + self.origin + TOKEN_EXCHANGE_PATH, {}, hf_token + ) + except (OSError, TimeoutError): + return "unavailable", None + if status == HTTPStatus.OK and isinstance( + response.get("access_token"), str + ): + try: + expires_in = max( + MINIMUM_TOKEN_LIFETIME_SECONDS, + int(response.get("expires_in", DEFAULT_TOKEN_LIFETIME_SECONDS)), + ) + except (ValueError, TypeError): + return "unavailable", None + token = response["access_token"] + self._session_token = (token, time.monotonic() + expires_in) + return "ok", token + if status in {HTTPStatus.UNAUTHORIZED, HTTPStatus.FORBIDDEN}: + return "rejected", None + return "unavailable", None + + +def shared_session(services: dict, origin: str) -> CollectorSession: + """Share a session among explicit modules without adding auth to the host.""" + session = services.setdefault("collector_session", CollectorSession(origin)) + if not isinstance(session, CollectorSession) or session.origin != origin: + raise ValueError("Provider modules must use the same collector origin.") + return session diff --git a/packages/provider-telemetry/src/microcosm_provider_telemetry/service/constants.py b/packages/provider-core/src/microcosm_provider_core/constants.py similarity index 100% rename from packages/provider-telemetry/src/microcosm_provider_telemetry/service/constants.py rename to packages/provider-core/src/microcosm_provider_core/constants.py diff --git a/packages/provider-telemetry/src/microcosm_provider_telemetry/service/database.py b/packages/provider-core/src/microcosm_provider_core/database.py similarity index 93% rename from packages/provider-telemetry/src/microcosm_provider_telemetry/service/database.py rename to packages/provider-core/src/microcosm_provider_core/database.py index ad63035a..8416a5e5 100644 --- a/packages/provider-telemetry/src/microcosm_provider_telemetry/service/database.py +++ b/packages/provider-core/src/microcosm_provider_core/database.py @@ -8,7 +8,7 @@ from sqlalchemy.engine import Engine from sqlalchemy.orm import Session, sessionmaker -from microcosm_provider_telemetry.service.constants import ( +from microcosm_provider_core.constants import ( DATABASE_TIMEOUT_SECONDS, ) diff --git a/packages/provider-telemetry/src/microcosm_provider_telemetry/service/migrations.py b/packages/provider-core/src/microcosm_provider_core/migrations.py similarity index 97% rename from packages/provider-telemetry/src/microcosm_provider_telemetry/service/migrations.py rename to packages/provider-core/src/microcosm_provider_core/migrations.py index 1d94eb8d..cd6a2d8c 100644 --- a/packages/provider-telemetry/src/microcosm_provider_telemetry/service/migrations.py +++ b/packages/provider-core/src/microcosm_provider_core/migrations.py @@ -15,7 +15,7 @@ from sqlalchemy import inspect from sqlalchemy.engine import Connection, Engine -from microcosm_provider_telemetry.service.database import create_spool_engine +from microcosm_provider_core.database import create_spool_engine _MIGRATION_TARGET = "head" diff --git a/packages/provider-telemetry/src/microcosm_provider_telemetry/service/models.py b/packages/provider-core/src/microcosm_provider_core/models.py similarity index 82% rename from packages/provider-telemetry/src/microcosm_provider_telemetry/service/models.py rename to packages/provider-core/src/microcosm_provider_core/models.py index 8a5e70f3..5bd3e93b 100644 --- a/packages/provider-telemetry/src/microcosm_provider_telemetry/service/models.py +++ b/packages/provider-core/src/microcosm_provider_core/models.py @@ -6,6 +6,7 @@ from typing import Any from sqlalchemy import ( + Float, ForeignKeyConstraint, Index, Integer, @@ -15,7 +16,7 @@ from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column, relationship from sqlalchemy.types import TypeDecorator -from microcosm_provider_telemetry.service.constants import ( +from microcosm_provider_core.constants import ( UPLOAD_STATE_PENDING, ) @@ -128,3 +129,17 @@ class TelemetryEventRecord(SpoolModel): ) created_at: Mapped[str] = mapped_column(Text, nullable=False) run: Mapped[TelemetryRunRecord] = relationship(back_populates="events") + + +class GraphPublicationJob(SpoolModel): + """Durable artifact delivery, independent of event retention and run IDs.""" + + __tablename__ = "graph_publication_jobs" + publication_id: Mapped[str] = mapped_column(Text, primary_key=True) + inventory: Mapped[dict[str, Any]] = mapped_column(JsonObjectText, nullable=False) + directory: Mapped[str] = mapped_column(Text, nullable=False) + status: Mapped[str] = mapped_column(Text, nullable=False) + attempts: Mapped[int] = mapped_column(Integer, nullable=False) + next_attempt_at: Mapped[float] = mapped_column(Float, nullable=False) + lease_until: Mapped[float] = mapped_column(Float, nullable=False) + receipt: Mapped[dict[str, Any]] = mapped_column(JsonObjectText, nullable=False) diff --git a/packages/provider-orrery/README.md b/packages/provider-orrery/README.md new file mode 100644 index 00000000..79872bf1 --- /dev/null +++ b/packages/provider-orrery/README.md @@ -0,0 +1,7 @@ +# Microcosm provider Orrery publication + +Publishes preserved graph and execution-evidence files through the immutable +Microcosm runs API. Publication IDs do not require a telemetry run. Pending jobs +retain their exact inventory, bytes, leases and retry status independently of +telemetry retention. This package neither generates Microcosm graphs nor imports +Microcosm. HTTP delivery runs in the selected `orrery` service module. diff --git a/packages/provider-orrery/pyproject.toml b/packages/provider-orrery/pyproject.toml new file mode 100644 index 00000000..0f3bb9c2 --- /dev/null +++ b/packages/provider-orrery/pyproject.toml @@ -0,0 +1,28 @@ +[project] +name = "microcosm-provider-orrery" +version = "0.1.0" +description = "Durable immutable Orrery graph publication without telemetry registration" +readme = "README.md" +requires-python = ">=3.13" +license = "MIT" +dependencies = ["microcosm-provider-client==0.1.0", "microcosm-provider-core==0.1.0", "httpx>=0.27,<1"] + +[project.urls] +Source = "https://github.com/PolicyEngine/microcosm-local-provider" + +[project.scripts] +microcosm-provider-publish-graph = "microcosm_provider_orrery.cli:main" + +[project.entry-points."microcosm.provider.modules"] +orrery = "microcosm_provider_orrery.service:create_module" + +[tool.uv.sources] +microcosm-provider-client = { workspace = true } +microcosm-provider-core = { workspace = true } + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[tool.hatch.build.targets.wheel] +packages = ["src/microcosm_provider_orrery"] diff --git a/packages/provider-orrery/src/microcosm_provider_orrery/__init__.py b/packages/provider-orrery/src/microcosm_provider_orrery/__init__.py new file mode 100644 index 00000000..7e5fc1e6 --- /dev/null +++ b/packages/provider-orrery/src/microcosm_provider_orrery/__init__.py @@ -0,0 +1 @@ +"""Immutable graph publication; no Microcosm dependency or import side effects.""" diff --git a/packages/provider-orrery/src/microcosm_provider_orrery/cli.py b/packages/provider-orrery/src/microcosm_provider_orrery/cli.py new file mode 100644 index 00000000..d5eecd6e --- /dev/null +++ b/packages/provider-orrery/src/microcosm_provider_orrery/cli.py @@ -0,0 +1,58 @@ +"""Retry preserved graph uploads without registering a telemetry run.""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path + +from microcosm_provider_core.auth import ( + CollectorSession, + _collector_origin, +) +from microcosm_provider_core.constants import PRODUCTION_COLLECTOR_URL + +from microcosm_provider_orrery.client import default_spool_path +from microcosm_provider_orrery.service.queue import ( + DEFAULT_GRAPH_PUBLICATION_ORIGIN, + GraphPublicationDelivery, + GraphPublicationQueue, +) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--spool", type=Path, default=default_spool_path()) + parser.add_argument("--publication-id", required=True) + parser.add_argument( + "--directory", + type=Path, + help="Optional preserved bundle containing orrery.upload.json; re-enqueues after moving machines.", + ) + parser.add_argument("--origin", default=DEFAULT_GRAPH_PUBLICATION_ORIGIN) + args = parser.parse_args(argv) + queue = GraphPublicationQueue(args.spool) + if args.directory is not None: + inventory = json.loads((args.directory / "orrery.upload.json").read_text()) + if inventory["publication_id"] != args.publication_id: + parser.error("The preserved inventory has a different publication ID.") + queue.enqueue(args.directory, inventory) + if queue.receipt(args.publication_id) is None: + parser.error("Publication job not found; supply its preserved directory.") + queue.retry(args.publication_id) + session = CollectorSession(_collector_origin(PRODUCTION_COLLECTOR_URL)) + delivery = GraphPublicationDelivery( + queue, + credential=session.credential, + origin=args.origin, + invalidate_credential=session.invalidate, + ) + # Explicit retries process this job only, not unrelated pending jobs. + delivery.flush_once(publication_id=args.publication_id) + receipt = queue.receipt(args.publication_id) + print(json.dumps(receipt, sort_keys=True)) + return 0 if receipt["status"] == "published" else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/packages/provider-orrery/src/microcosm_provider_orrery/client.py b/packages/provider-orrery/src/microcosm_provider_orrery/client.py new file mode 100644 index 00000000..c0b3d13e --- /dev/null +++ b/packages/provider-orrery/src/microcosm_provider_orrery/client.py @@ -0,0 +1,47 @@ +"""Build-facing durable enqueue; all authentication and HTTP run in the service.""" + +import os +import time +from collections.abc import Callable +from pathlib import Path + + +def default_spool_path() -> Path: + """Keep Microcosm's existing cache path so queued jobs survive migration.""" + configured = os.environ.get("XDG_CACHE_HOME", "").strip() + root = Path(configured).expanduser() if configured else Path.home() / ".cache" + return root / "microcosm" / "telemetry" / "events.sqlite3" + + +def publication_inventory(directory: Path, roles, *, publication_id=None) -> dict: + """Hash an explicit set of public-safe graph files without network access.""" + from microcosm_provider_orrery.service.queue import ( + publication_inventory as inventory, + ) + + return inventory(directory, roles, publication_id=publication_id) + + +def publish_graph( + directory: Path, + inventory: dict, + *, + spool_path: Path | None = None, + available: Callable[[], bool] = lambda: False, + wait_seconds: float = 30.0, +) -> dict: + """Persist before waiting; unavailable services never lose a publication job.""" + from microcosm_provider_orrery.service.queue import GraphPublicationQueue + + queue = GraphPublicationQueue(spool_path or default_spool_path()) + try: + queue.enqueue(directory, inventory) + deadline = time.monotonic() + max(0, wait_seconds) + while available() and time.monotonic() < deadline: + receipt = queue.receipt(inventory["publication_id"]) + if receipt.get("attempts", 0) or receipt["status"] == "published": + return receipt + time.sleep(0.1) + return queue.receipt(inventory["publication_id"]) + finally: + queue.close() diff --git a/packages/provider-orrery/src/microcosm_provider_orrery/service/__init__.py b/packages/provider-orrery/src/microcosm_provider_orrery/service/__init__.py new file mode 100644 index 00000000..5826d70f --- /dev/null +++ b/packages/provider-orrery/src/microcosm_provider_orrery/service/__init__.py @@ -0,0 +1,8 @@ +"""Explicit graph-publication service module registration.""" + +from microcosm_provider_orrery.service.module import OrreryModule + + +def create_module(configuration, context): + """Construct a module without starting workers or performing network I/O.""" + return OrreryModule(configuration, context) diff --git a/packages/provider-orrery/src/microcosm_provider_orrery/service/module.py b/packages/provider-orrery/src/microcosm_provider_orrery/service/module.py new file mode 100644 index 00000000..c1e508e5 --- /dev/null +++ b/packages/provider-orrery/src/microcosm_provider_orrery/service/module.py @@ -0,0 +1,55 @@ +"""Completed graph delivery, independent of telemetry registration and retention.""" + +from pathlib import Path + +from microcosm_provider_client.contracts import JsonObject, ModuleContext +from microcosm_provider_core.auth import ( + _collector_origin, + _development_collector_url, + shared_session, +) +from microcosm_provider_core.constants import PRODUCTION_COLLECTOR_URL + +from microcosm_provider_orrery.service.queue import ( + DEFAULT_GRAPH_PUBLICATION_ORIGIN, + GraphPublicationDelivery, + GraphPublicationQueue, +) + + +class OrreryModule: + def __init__(self, configuration: JsonObject, context: ModuleContext): + self.configuration, self.context = configuration, context + self.queue = self.delivery = None + + def initialize(self) -> None: + self.queue = GraphPublicationQueue(Path(self.configuration["spool_path"])) + origin = ( + _development_collector_url(self.configuration["development_collector_url"]) + if self.configuration.get("development_collector_url") + else _collector_origin(PRODUCTION_COLLECTOR_URL) + ) + session = shared_session(self.context.services, origin) + self.delivery = GraphPublicationDelivery( + self.queue, + credential=session.credential, + invalidate_credential=session.invalidate, + origin=self.configuration.get( + "publication_origin", DEFAULT_GRAPH_PUBLICATION_ORIGIN + ), + ) + + def handle_message(self, message: JsonObject) -> None: + self.queue.enqueue(Path(message["directory"]), dict(message["inventory"])) + + def tick(self, now: float) -> None: + self.delivery.flush_once() + + def parent_exited(self) -> None: + """Preserved jobs remain eligible when a later host resumes delivery.""" + + def close(self, deadline: float) -> None: + if self.delivery is not None: + self.delivery.client.close() + if self.queue is not None: + self.queue.close() diff --git a/packages/provider-orrery/src/microcosm_provider_orrery/service/queue.py b/packages/provider-orrery/src/microcosm_provider_orrery/service/queue.py new file mode 100644 index 00000000..49a3ea91 --- /dev/null +++ b/packages/provider-orrery/src/microcosm_provider_orrery/service/queue.py @@ -0,0 +1,378 @@ +"""Durable, retryable immutable graph uploads owned by the emitter service. + +Graph bytes stay in preserved files, never in event details. Pending graph jobs +are not subject to the telemetry event queue's time or size pruning. +""" + +from __future__ import annotations + +import hashlib +import json +import re +import time +import uuid +from collections.abc import Callable, Mapping +from pathlib import Path +from urllib.parse import urlsplit + +import httpx +from microcosm_provider_core.database import ( + create_spool_engine, + create_spool_session_factory, +) +from microcosm_provider_core.migrations import upgrade_spool_database +from microcosm_provider_core.models import GraphPublicationJob +from sqlalchemy import select, update + +MAX_GRAPH_FILE_BYTES = 64 * 1024 * 1024 +MAX_GRAPH_TOTAL_BYTES = 512 * 1024 * 1024 +GRAPH_PUBLICATION_ACTION = "graph_publication" +GRAPH_PUBLICATION_WAIT_SECONDS = 30.0 +GRAPH_PUBLICATION_LEASE_SECONDS = 300.0 +GRAPH_PUBLICATION_STATUS_FILENAME = "publication.status.json" +DEFAULT_GRAPH_PUBLICATION_ORIGIN = "https://microcosm-runs.vercel.app" +GRAPH_PUBLICATION_ROLES = { + "graph": r"graph\.orrery\.json", + "schema": r"graph\.schema\.json", + "index": r"execution\.evidence\.json", + "declaration": r"evidence-[A-Za-z0-9_-]+-graph\.json", + "manifest": r"evidence-[A-Za-z0-9_-]+-manifest\.json", + "binding": r"evidence-[A-Za-z0-9_-]+-binding\.json", + "summaries": r"evidence-[A-Za-z0-9_-]+-summaries\.json", + "upstream": r"upstream-[a-f0-9]{64}\.json", +} + + +def _digest(payload: bytes) -> str: + return hashlib.sha256(payload).hexdigest() + + +def inventory_digest(inventory: Mapping) -> str: + """Digest the versioned field order shared with the publication API.""" + inventory = { + "version": inventory["version"], + "publication_id": inventory["publication_id"], + "files": [ + {key: item[key] for key in ("name", "role", "bytes", "sha256")} + for item in sorted(inventory["files"], key=lambda file: file["name"]) + ], + } + return _digest( + json.dumps(inventory, separators=(",", ":"), ensure_ascii=True).encode() + ) + + +def publication_inventory( + directory: Path, roles: Mapping[str, str], *, publication_id: str | None = None +) -> dict: + """Hash an explicit allowlist of files; publication identity is not a run ID.""" + files = [] + for name, role in sorted(roles.items()): + if role not in GRAPH_PUBLICATION_ROLES or not re.fullmatch( + GRAPH_PUBLICATION_ROLES[role], name + ): + raise ValueError("Ineligible graph publication filename or role.") + path = Path(directory) / name + if path.is_symlink() or not path.is_file(): + raise ValueError( + "Graph publication files must be regular, preserved files." + ) + size = path.stat().st_size + if not 0 < size <= MAX_GRAPH_FILE_BYTES: + raise ValueError("Graph publication file exceeds its size limit.") + files.append( + { + "name": name, + "role": role, + "bytes": size, + "sha256": _digest(path.read_bytes()), + } + ) + if ( + not files + or len(files) > 256 + or sum(f["bytes"] for f in files) > MAX_GRAPH_TOTAL_BYTES + or sum(f["role"] == "graph" for f in files) != 1 + ): + raise ValueError("Invalid graph publication inventory.") + id_ = publication_id or uuid.uuid4().hex + if not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_-]{0,127}", id_): + raise ValueError("Invalid graph publication ID.") + return {"version": 1, "publication_id": id_, "files": files} + + +class GraphPublicationQueue: + """SQLite job records with leases; retained until publication or operator action.""" + + def __init__(self, path: Path): + self.path = Path(path) + self.path.parent.mkdir(parents=True, exist_ok=True) + self._engine = create_spool_engine(self.path) + upgrade_spool_database(self._engine) + self._sessions = create_spool_session_factory(self._engine) + + def enqueue(self, directory: Path, inventory: dict) -> None: + """Durably register exact bytes without a telemetry run or credential.""" + directory = Path(directory).resolve() + expected = publication_inventory( + directory, + {f["name"]: f["role"] for f in inventory["files"]}, + publication_id=inventory["publication_id"], + ) + if expected != inventory: + raise ValueError("Graph publication digest mismatch.") + id_ = inventory["publication_id"] + with self._sessions.begin() as session: + existing = session.get(GraphPublicationJob, id_) + if existing is not None: + if existing.inventory != inventory or existing.directory != str( + directory + ): + raise ValueError( + "Graph publication identity conflicts with a queued job." + ) + return + session.add( + GraphPublicationJob( + publication_id=id_, + inventory=inventory, + directory=str(directory), + status="pending", + attempts=0, + next_attempt_at=0, + lease_until=0, + receipt={ + "version": 1, + "publication_id": id_, + "status": "pending", + "error_code": None, + }, + ) + ) + + def close(self) -> None: + """Release this handle's database connections without deleting jobs.""" + self._engine.dispose() + + def claim( + self, + *, + lease_seconds: float = GRAPH_PUBLICATION_LEASE_SECONDS, + publication_id: str | None = None, + ) -> dict | None: + """Atomically lease one eligible job across concurrent emitter processes.""" + now = time.time() + with self._sessions.begin() as session: + statement = select(GraphPublicationJob.publication_id) + if publication_id is not None: + statement = statement.where( + GraphPublicationJob.publication_id == publication_id + ) + id_ = session.scalar( + statement.where( + GraphPublicationJob.status == "pending", + GraphPublicationJob.next_attempt_at <= now, + GraphPublicationJob.lease_until <= now, + ).limit(1) + ) + if id_ is None: + return None + result = session.execute( + update(GraphPublicationJob) + .where( + GraphPublicationJob.publication_id == id_, + GraphPublicationJob.status == "pending", + GraphPublicationJob.lease_until <= now, + ) + .values(lease_until=now + lease_seconds) + ) + if not result.rowcount: + return None + job = session.get(GraphPublicationJob, id_) + return { + "publication_id": id_, + "directory": job.directory, + "inventory": job.inventory, + "lease_until": job.lease_until, + } + + def receipt(self, id_: str) -> dict | None: + """Read the latest local result; old HF receipt snapshots remain unchanged.""" + with self._sessions() as session: + job = session.get(GraphPublicationJob, id_) + return None if job is None else dict(job.receipt) + + def finish( + self, + id_: str, + *, + error_code: str | None = None, + url: str | None = None, + expected_lease: float | None = None, + ) -> None: + with self._sessions.begin() as session: + statement = update(GraphPublicationJob).where( + GraphPublicationJob.publication_id == id_, + GraphPublicationJob.status == "pending", + ) + if expected_lease is not None: + statement = statement.where( + GraphPublicationJob.lease_until == expected_lease + ) + job = session.scalar( + statement.values( + attempts=GraphPublicationJob.attempts + 1, + status="published" if url else "pending", + lease_until=0, + ).returning(GraphPublicationJob) + ) + # Completion is conditional in SQL, not a read-then-write check: + # expired workers cannot release another lease or replace success. + if job is None: + return + job.next_attempt_at = time.time() + min(300, 2 ** min(job.attempts, 8)) + job.receipt = { + "version": 1, + "publication_id": id_, + "status": job.status, + "error_code": error_code, + "url": url, + "attempts": job.attempts, + "inventory_sha256": inventory_digest(job.inventory), + } + # This is a separate latest-result file, not the build/HF receipt. + path = Path(job.directory) / GRAPH_PUBLICATION_STATUS_FILENAME + temporary = path.with_suffix(f".{uuid.uuid4().hex}.tmp") + temporary.write_text(json.dumps(job.receipt, sort_keys=True) + "\n") + temporary.replace(path) + + def retry(self, id_: str | None = None) -> None: + """Make preserved pending jobs eligible again, including missing credentials.""" + statement = update(GraphPublicationJob).where( + GraphPublicationJob.status == "pending" + ) + if id_ is not None: + statement = statement.where(GraphPublicationJob.publication_id == id_) + with self._sessions.begin() as session: + session.execute(statement.values(next_attempt_at=0)) + + +class GraphPublicationDelivery: + """Upload exact files with scoped signed URLs, then seal the publication.""" + + def __init__( + self, + queue: GraphPublicationQueue, + *, + credential: Callable[[], tuple[str, str | None]], + origin: str, + client: httpx.Client | None = None, + invalidate_credential: Callable[[], None] | None = None, + ): + parsed = urlsplit(origin) + if parsed.scheme != "https" and not ( + parsed.scheme == "http" + and parsed.hostname in {"localhost", "127.0.0.1", "::1"} + ): + raise ValueError( + "Graph publication origin must be HTTPS or local development." + ) + if ( + parsed.username + or parsed.password + or parsed.query + or parsed.fragment + or parsed.path not in {"", "/"} + ): + raise ValueError("Graph publication destination must be an origin.") + self.queue, self.credential, self.origin = queue, credential, origin.rstrip("/") + self.client = client or httpx.Client(timeout=60, follow_redirects=False) + self.invalidate_credential = invalidate_credential + + def flush_once(self, *, publication_id: str | None = None) -> bool: + """Try one job; failures retain it without changing telemetry eligibility.""" + job = self.queue.claim(publication_id=publication_id) + if job is None: + return False + id_, inventory, directory = ( + job["publication_id"], + job["inventory"], + Path(job["directory"]), + ) + error = "publication_unavailable" + lease = job["lease_until"] + try: + status, token = self.credential() + if token is None: + self.queue.finish( + id_, error_code=f"credential_{status}", expected_lease=lease + ) + return False + current = publication_inventory( + directory, + {f["name"]: f["role"] for f in inventory["files"]}, + publication_id=id_, + ) + if current != inventory: + raise ValueError("preserved_bytes_changed") + headers = {"Authorization": f"Bearer {token}"} + response = self.client.post( + self.origin + "/api/uploads/", json=inventory, headers=headers + ) + response.raise_for_status() + body = response.json() + uploads = body.get("uploads") + if not isinstance(uploads, list): + raise ValueError("invalid_upload_response") + files = {f["name"]: f for f in inventory["files"]} + for upload in uploads: + name = upload.get("name") + if name not in files: + raise ValueError("invalid_upload_response") + if upload.get("already_uploaded"): + continue + url = urlsplit(upload["url"]) + if ( + url.scheme != "https" + or not ( + (url.hostname == "vercel.com" and url.path == "/api/blob/") + or url.hostname == "blob.vercel-storage.com" + or (url.hostname or "").endswith(".blob.vercel-storage.com") + ) + or url.username + or url.password + ): + raise ValueError("invalid_upload_url") + with (directory / name).open("rb") as stream: + result = self.client.put( + upload["url"], + content=stream, + headers={ + "Content-Type": "application/json", + "Content-Length": str(files[name]["bytes"]), + }, + ) + result.raise_for_status() + response = self.client.post( + self.origin + "/api/publications/", json=inventory, headers=headers + ) + response.raise_for_status() + result = response.json() + if result.get("publication_id") != id_ or result.get( + "inventory_sha256" + ) != inventory_digest(inventory): + raise ValueError("invalid_publication_response") + self.queue.finish( + id_, url=f"{self.origin}/runs/{id_}/", expected_lease=lease + ) + return True + except httpx.HTTPStatusError as exception: + error = f"publication_http_{exception.response.status_code}" + if exception.response.status_code == 401 and self.invalidate_credential: + self.invalidate_credential() + except httpx.HTTPError: + error = "publication_network_error" + except (OSError, ValueError, KeyError, TypeError): + error = "publication_invalid_evidence_or_response" + self.queue.finish(id_, error_code=error, expected_lease=lease) + return False diff --git a/packages/provider-orrery/tests/test_client.py b/packages/provider-orrery/tests/test_client.py new file mode 100644 index 00000000..e9dc8822 --- /dev/null +++ b/packages/provider-orrery/tests/test_client.py @@ -0,0 +1,35 @@ +"""Public enqueue remains durable when no service or telemetry run exists.""" + +import json + +from microcosm_provider_core.auth import shared_session +from microcosm_provider_orrery.client import publication_inventory, publish_graph +from microcosm_provider_orrery.service.queue import GraphPublicationQueue + + +def test_public_client_persists_without_service_or_run(tmp_path): + directory = tmp_path / "graph" + directory.mkdir() + (directory / "graph.orrery.json").write_text(json.dumps({"nodes": []})) + inventory = publication_inventory( + directory, {"graph.orrery.json": "graph"}, publication_id="independent-graph" + ) + path = tmp_path / "jobs.sqlite3" + receipt = publish_graph(directory, inventory, spool_path=path, wait_seconds=0) + assert receipt["status"] == "pending" + assert receipt["publication_id"] == "independent-graph" + queue = GraphPublicationQueue(path) + try: + assert queue.receipt("independent-graph") == receipt + finally: + queue.close() + + +def test_modules_share_one_session_and_reject_conflicting_origins(): + import pytest + + services = {} + first = shared_session(services, "https://collector.example") + assert shared_session(services, "https://collector.example") is first + with pytest.raises(ValueError, match="same collector origin"): + shared_session(services, "https://different.example") diff --git a/packages/provider-orrery/tests/test_publication.py b/packages/provider-orrery/tests/test_publication.py new file mode 100644 index 00000000..033f43a1 --- /dev/null +++ b/packages/provider-orrery/tests/test_publication.py @@ -0,0 +1,244 @@ +"""Durable graph jobs use fake HTTP, independently of the visualization app.""" + +import json + +import httpx +import pytest +from microcosm_provider_orrery.service.queue import ( + GraphPublicationDelivery, + GraphPublicationQueue, + inventory_digest, + publication_inventory, +) + + +def source(tmp_path): + directory = tmp_path / "source" + directory.mkdir() + (directory / "graph.orrery.json").write_text('{"fixture":true}') + return directory, publication_inventory(directory, {"graph.orrery.json": "graph"}) + + +def test_jobs_survive_restart_without_run_registration_or_event_retention(tmp_path): + directory, inventory = source(tmp_path) + queue = GraphPublicationQueue(tmp_path / "jobs.sqlite3") + queue.enqueue(directory, inventory) + restarted = GraphPublicationQueue(queue.path) + job = restarted.claim() + assert job["inventory"] == inventory + assert "run_id" not in inventory + assert job["publication_id"] == inventory["publication_id"] + assert restarted.claim() is None + + +def test_queue_rejects_changed_bytes_and_conflicting_inventory(tmp_path): + directory, inventory = source(tmp_path) + queue = GraphPublicationQueue(tmp_path / "jobs.sqlite3") + (directory / "graph.orrery.json").write_text("changed") + with pytest.raises(ValueError, match="digest"): + queue.enqueue(directory, inventory) + with pytest.raises(ValueError): + publication_inventory(directory, {"private.h5": "graph"}) + + +def test_delivery_retries_missing_credentials_and_network_failure(tmp_path): + directory, inventory = source(tmp_path) + queue = GraphPublicationQueue(tmp_path / "jobs.sqlite3") + queue.enqueue(directory, inventory) + delivery = GraphPublicationDelivery( + queue, credential=lambda: ("missing", None), origin="https://runs.example" + ) + delivery.flush_once() + assert queue.receipt(inventory["publication_id"])["status"] == "pending" + queue.retry(inventory["publication_id"]) + client = httpx.Client( + transport=httpx.MockTransport( + lambda request: (_ for _ in ()).throw( + httpx.ConnectError("secret", request=request) + ) + ) + ) + GraphPublicationDelivery( + queue, + credential=lambda: ("ok", "session"), + origin="https://runs.example", + client=client, + ).flush_once() + receipt = queue.receipt(inventory["publication_id"]) + assert receipt["status"] == "pending" + assert "secret" not in json.dumps(receipt) + + +@pytest.mark.parametrize( + "upload_url", + [ + "https://store.blob.vercel-storage.com/upload?signed=credential", + "https://vercel.com/api/blob/?pathname=object&signed=credential", + ], +) +def test_delivery_uses_exact_bytes_and_does_not_forward_session_to_signed_upload( + tmp_path, + upload_url, +): + directory, inventory = source(tmp_path) + queue = GraphPublicationQueue(tmp_path / "jobs.sqlite3") + queue.enqueue(directory, inventory) + requests = [] + + def respond(request): + requests.append(request) + if request.url.path == "/api/uploads/": + return httpx.Response( + 200, + json={ + "uploads": [ + { + "name": "graph.orrery.json", + "url": upload_url, + "already_uploaded": False, + } + ] + }, + ) + if request.method == "PUT": + assert "authorization" not in request.headers + assert request.content == (directory / "graph.orrery.json").read_bytes() + return httpx.Response(200, json={}) + return httpx.Response( + 200, + json={ + "publication_id": inventory["publication_id"], + "inventory_sha256": inventory_digest(inventory), + }, + ) + + delivery = GraphPublicationDelivery( + queue, + credential=lambda: ("ok", "session"), + origin="https://runs.example", + client=httpx.Client(transport=httpx.MockTransport(respond)), + ) + delivery.flush_once() + receipt = queue.receipt(inventory["publication_id"]) + assert receipt["status"] == "published" + assert len(requests) == 3 + assert "signed=" not in json.dumps(receipt) + assert "session" not in json.dumps(receipt) + + +def test_queue_resumes_after_worker_exits_and_keeps_pending_jobs(tmp_path): + directory, inventory = source(tmp_path) + queue = GraphPublicationQueue(tmp_path / "jobs.sqlite3") + queue.enqueue(directory, inventory) + queue.claim(lease_seconds=0) + assert GraphPublicationQueue(queue.path).claim() is not None + + +def test_late_failed_attempt_cannot_replace_a_published_receipt(tmp_path): + directory, inventory = source(tmp_path) + queue = GraphPublicationQueue(tmp_path / "jobs.sqlite3") + queue.enqueue(directory, inventory) + queue.claim(lease_seconds=0) + replacement = GraphPublicationQueue(queue.path) + replacement.claim() + id_ = inventory["publication_id"] + replacement.finish(id_, url=f"https://runs.example/runs/{id_}/") + published = queue.receipt(id_) + queue.finish(id_, error_code="publication_network_error") + assert queue.receipt(id_) == published + assert json.loads((directory / "publication.status.json").read_bytes()) == published + + +def test_expired_worker_cannot_release_a_replacement_workers_lease(tmp_path): + directory, inventory = source(tmp_path) + queue = GraphPublicationQueue(tmp_path / "jobs.sqlite3") + queue.enqueue(directory, inventory) + expired = queue.claim(lease_seconds=0) + replacement = GraphPublicationQueue(queue.path) + current = replacement.claim() + id_ = inventory["publication_id"] + initial = queue.receipt(id_) + queue.finish( + id_, + error_code="publication_network_error", + expected_lease=expired["lease_until"], + ) + assert queue.receipt(id_) == initial + assert queue.claim() is None + replacement.finish( + id_, + url=f"https://runs.example/runs/{id_}/", + expected_lease=current["lease_until"], + ) + assert queue.receipt(id_)["status"] == "published" + + +def test_telemetry_pruning_cannot_remove_graph_jobs(tmp_path): + from microcosm_provider_telemetry.service.spool import EventSpool + + directory, inventory = source(tmp_path) + queue = GraphPublicationQueue(tmp_path / "jobs.sqlite3") + queue.enqueue(directory, inventory) + EventSpool(queue.path).prune() + assert ( + GraphPublicationQueue(queue.path).receipt(inventory["publication_id"])["status"] + == "pending" + ) + + +def test_rejected_graph_session_is_refreshed_without_discarding_the_job(tmp_path): + directory, inventory = source(tmp_path) + queue = GraphPublicationQueue(tmp_path / "jobs.sqlite3") + queue.enqueue(directory, inventory) + invalidations = [] + delivery = GraphPublicationDelivery( + queue, + credential=lambda: ("ok", "expired"), + invalidate_credential=lambda: invalidations.append(True), + origin="https://runs.example", + client=httpx.Client( + transport=httpx.MockTransport(lambda request: httpx.Response(401)) + ), + ) + assert not delivery.flush_once() + assert invalidations == [True] + assert queue.receipt(inventory["publication_id"])["status"] == "pending" + + +def test_retry_cli_reuses_identity_and_leaves_hf_snapshot_unchanged( + tmp_path, monkeypatch +): + from microcosm_provider_orrery import cli + + directory, inventory = source(tmp_path) + (directory / "orrery.upload.json").write_text(json.dumps(inventory)) + snapshot = directory / "orrery.publication.json" + snapshot.write_text('{"status":"pending"}') + calls = [] + + class Delivery: + def __init__(self, queue, **kwargs): + self.queue = queue + + def flush_once(self, *, publication_id): + calls.append(publication_id) + self.queue.finish( + publication_id, url="https://runs.example/runs/published/" + ) + + monkeypatch.setattr(cli, "GraphPublicationDelivery", Delivery) + assert ( + cli.main( + [ + "--spool", + str(tmp_path / "jobs.sqlite3"), + "--publication-id", + inventory["publication_id"], + "--directory", + str(directory), + ] + ) + == 0 + ) + assert calls == [inventory["publication_id"]] + assert snapshot.read_text() == '{"status":"pending"}' diff --git a/packages/provider-telemetry/pyproject.toml b/packages/provider-telemetry/pyproject.toml index 29658aa9..d7b0f01c 100644 --- a/packages/provider-telemetry/pyproject.toml +++ b/packages/provider-telemetry/pyproject.toml @@ -7,6 +7,7 @@ requires-python = ">=3.13" license = "MIT" dependencies = [ "microcosm-provider-client==0.1.0", + "microcosm-provider-core==0.1.0", "alembic>=1.13.3,<2", "sqlalchemy>=2,<3", "huggingface-hub>=0.20,<2", @@ -20,6 +21,7 @@ Source = "https://github.com/PolicyEngine/microcosm-local-provider" telemetry = "microcosm_provider_telemetry.service:create_module" [tool.uv.sources] +microcosm-provider-core = { workspace = true } microcosm-provider-client = { workspace = true } [build-system] diff --git a/packages/provider-telemetry/src/microcosm_provider_telemetry/service/collector.py b/packages/provider-telemetry/src/microcosm_provider_telemetry/service/collector.py index 169bc2c3..97c99a2e 100644 --- a/packages/provider-telemetry/src/microcosm_provider_telemetry/service/collector.py +++ b/packages/provider-telemetry/src/microcosm_provider_telemetry/service/collector.py @@ -1,126 +1,35 @@ -"""Collector authentication and best-effort event delivery.""" +"""Best-effort telemetry delivery; shared authentication lives outside this component.""" from __future__ import annotations -import json -import os import sys import time -import urllib.error -import urllib.request from collections.abc import Mapping from http import HTTPStatus from typing import Any -from urllib.parse import urlsplit -from huggingface_hub import get_token - -from microcosm_provider_telemetry.service.constants import ( - COLLECTOR_RESPONSE_TOO_LARGE_ERROR, - COLLECTOR_URL_HTTPS_ERROR, - COLLECTOR_URL_ORIGIN_ERROR, - DEFAULT_TOKEN_LIFETIME_SECONDS, - DEVELOPMENT_COLLECTOR_LOOPBACK_ERROR, - HTTP_TIMEOUT_SECONDS, - HTTP_USER_AGENT, +from microcosm_provider_core.auth import ( + CollectorSession, + _collector_origin, + _development_collector_url, + _http_post, + _huggingface_token, +) +from microcosm_provider_core.constants import ( INITIAL_RETRY_SECONDS, LOCAL_ONLY_MISSING_CREDENTIAL, LOCAL_ONLY_REJECTED_COLLECTOR_AUTHORIZATION, LOCAL_ONLY_REJECTED_CREDENTIAL, LOCAL_ONLY_REJECTED_REGISTRATION, - LOOPBACK_HOSTS, - MAX_HTTP_RESPONSE_BYTES, MAX_RETRY_SECONDS, - MINIMUM_TOKEN_LIFETIME_SECONDS, NO_CREDENTIAL_MESSAGE, PRODUCTION_COLLECTOR_URL, REJECTED_CREDENTIAL_MESSAGE, RUN_EVENTS_PATH_TEMPLATE, RUN_REGISTRATION_PATH, - TOKEN_EXCHANGE_PATH, - TOKEN_REFRESH_MARGIN_SECONDS, ) -from microcosm_provider_telemetry.service.spool import EventSpool - - -class _NoRedirectHandler(urllib.request.HTTPRedirectHandler): - """Keep bearer credentials on the explicitly configured origin.""" - - def redirect_request(self, req, fp, code, msg, headers, newurl): - return None - - -def _collector_origin(value: str, *, allow_loopback_http: bool = False) -> str: - parsed = urlsplit(value) - loopback = parsed.hostname in LOOPBACK_HOSTS - valid_scheme = parsed.scheme == "https" or ( - allow_loopback_http and loopback and parsed.scheme == "http" - ) - if not parsed.hostname or not valid_scheme: - raise ValueError(COLLECTOR_URL_HTTPS_ERROR) - if ( - parsed.username - or parsed.password - or parsed.query - or parsed.fragment - or parsed.path not in {"", "/"} - ): - raise ValueError(COLLECTOR_URL_ORIGIN_ERROR) - return value.rstrip("/") - - -def _development_collector_url(value: str) -> str: - parsed = urlsplit(value) - if parsed.hostname not in LOOPBACK_HOSTS: - raise ValueError(DEVELOPMENT_COLLECTOR_LOOPBACK_ERROR) - return _collector_origin(value, allow_loopback_http=True) - - -def _decode_response(body: bytes) -> dict[str, Any]: - try: - response = json.loads(body) if body else {} - except (json.JSONDecodeError, UnicodeDecodeError): - return {} - return response if isinstance(response, dict) else {} - -def _http_post( - url: str, - payload: Mapping[str, Any], - bearer_token: str, - *, - timeout: float = HTTP_TIMEOUT_SECONDS, -) -> tuple[int, dict[str, Any]]: - request = urllib.request.Request( - url, - data=json.dumps(payload, separators=(",", ":")).encode(), - headers={ - "Authorization": f"Bearer {bearer_token}", - "Content-Type": "application/json", - "User-Agent": HTTP_USER_AGENT, - }, - method="POST", - ) - try: - opener = urllib.request.build_opener(_NoRedirectHandler) - with opener.open(request, timeout=timeout) as response: - body = response.read(MAX_HTTP_RESPONSE_BYTES + 1) - if len(body) > MAX_HTTP_RESPONSE_BYTES: - raise OSError(COLLECTOR_RESPONSE_TOO_LARGE_ERROR) - return response.status, _decode_response(body) - except urllib.error.HTTPError as error: - body = error.read(MAX_HTTP_RESPONSE_BYTES + 1) - if len(body) > MAX_HTTP_RESPONSE_BYTES: - return error.code, {} - return error.code, _decode_response(body) - - -def _huggingface_token() -> str | None: - return ( - os.environ.get("HF_TOKEN", "").strip() - or os.environ.get("HUGGINGFACE_TOKEN", "").strip() - or get_token() - ) +from microcosm_provider_telemetry.service.spool import EventSpool def _registration_key(registration: Mapping[str, Any]) -> str: @@ -135,6 +44,7 @@ def __init__( spool: EventSpool, *, development_collector_url: str | None = None, + session: CollectorSession | None = None, ) -> None: self.collector_url = ( _development_collector_url(development_collector_url) @@ -142,7 +52,11 @@ def __init__( else _collector_origin(PRODUCTION_COLLECTOR_URL) ) self.spool = spool - self._session_token: tuple[str, float] | None = None + self.session = session or CollectorSession( + self.collector_url, + post=lambda *args: _http_post(*args), + credential=lambda: _huggingface_token(), + ) self._registered: set[str] = set() self._warned_no_token = False self._warned_denied: set[str] = set() @@ -181,7 +95,7 @@ def flush_once(self) -> bool: made_progress = True self._retry_seconds = INITIAL_RETRY_SECONDS elif status == HTTPStatus.UNAUTHORIZED: - self._session_token = None + self.session.invalidate() elif status == HTTPStatus.FORBIDDEN: self._make_local_only( registration, @@ -192,40 +106,21 @@ def flush_once(self) -> bool: return made_progress def _collector_token(self, registration: Mapping[str, Any]) -> str | None: - if ( - self._session_token is not None - and self._session_token[1] > time.monotonic() + TOKEN_REFRESH_MARGIN_SECONDS - ): - return self._session_token[0] - hf_token = _huggingface_token() - if not hf_token: + result, token = self.session_credential() + if result == "missing": if not self._warned_no_token: print(NO_CREDENTIAL_MESSAGE, file=sys.stderr, flush=True) self._warned_no_token = True self._make_local_only(registration, LOCAL_ONLY_MISSING_CREDENTIAL) - return None - try: - status, response = _http_post( - self.collector_url + TOKEN_EXCHANGE_PATH, - {}, - hf_token, - ) - except (OSError, TimeoutError): - self._defer_retry() - return None - if status == HTTPStatus.OK and isinstance(response.get("access_token"), str): - expires_in = max( - MINIMUM_TOKEN_LIFETIME_SECONDS, - int(response.get("expires_in", DEFAULT_TOKEN_LIFETIME_SECONDS)), - ) - token = response["access_token"] - self._session_token = (token, time.monotonic() + expires_in) - return token - if status in {HTTPStatus.UNAUTHORIZED, HTTPStatus.FORBIDDEN}: + elif result == "rejected": self._make_local_only(registration, LOCAL_ONLY_REJECTED_CREDENTIAL) - else: + elif result != "ok": self._defer_retry() - return None + return token + + def session_credential(self) -> tuple[str, str | None]: + """Compatibility accessor; authentication has no telemetry queue policy.""" + return self.session.credential() def _register(self, registration: Mapping[str, Any], token: str) -> bool: registration_key = _registration_key(registration) @@ -243,7 +138,7 @@ def _register(self, registration: Mapping[str, Any], token: str) -> bool: self._retry_seconds = INITIAL_RETRY_SECONDS return True if status == HTTPStatus.UNAUTHORIZED: - self._session_token = None + self.session.invalidate() elif status in {HTTPStatus.FORBIDDEN, HTTPStatus.CONFLICT}: self._make_local_only(registration, LOCAL_ONLY_REJECTED_REGISTRATION) else: diff --git a/packages/provider-telemetry/src/microcosm_provider_telemetry/service/module.py b/packages/provider-telemetry/src/microcosm_provider_telemetry/service/module.py index 60c095e0..1f604f3d 100644 --- a/packages/provider-telemetry/src/microcosm_provider_telemetry/service/module.py +++ b/packages/provider-telemetry/src/microcosm_provider_telemetry/service/module.py @@ -6,6 +6,18 @@ from collections.abc import Mapping from microcosm_provider_client.contracts import JsonObject, ModuleContext +from microcosm_provider_core.auth import ( + _collector_origin, + _development_collector_url, + shared_session, +) +from microcosm_provider_core.constants import ( + DEFAULT_HEARTBEAT_SECONDS, + DRAIN_RETRY_SECONDS, + FAILURE_CLASS_UNEXPECTED_PROCESS_EXIT, + MINIMUM_HEARTBEAT_SECONDS, + PRODUCTION_COLLECTOR_URL, +) from microcosm_provider_telemetry.protocol import ( EVENT_TYPE_HEARTBEAT, @@ -18,12 +30,6 @@ UNEXPECTED_PROCESS_EXIT_MESSAGE, ) from microcosm_provider_telemetry.service.collector import CollectorDelivery -from microcosm_provider_telemetry.service.constants import ( - DEFAULT_HEARTBEAT_SECONDS, - DRAIN_RETRY_SECONDS, - FAILURE_CLASS_UNEXPECTED_PROCESS_EXIT, - MINIMUM_HEARTBEAT_SECONDS, -) from microcosm_provider_telemetry.service.resources import ProcessTreeSampler from microcosm_provider_telemetry.service.spool import EventSpool from microcosm_provider_telemetry.service.timestamps import utc_now @@ -54,6 +60,14 @@ def initialize(self) -> None: self.spool = EventSpool(self.configuration["spool_path"]) self.delivery = CollectorDelivery( self.spool, + session=shared_session( + self.context.services, + _development_collector_url( + self.configuration["development_collector_url"] + ) + if self.configuration.get("development_collector_url") + else _collector_origin(PRODUCTION_COLLECTOR_URL), + ), development_collector_url=self.configuration.get( "development_collector_url" ), diff --git a/packages/provider-telemetry/src/microcosm_provider_telemetry/service/spool.py b/packages/provider-telemetry/src/microcosm_provider_telemetry/service/spool.py index 0082d8df..14c47f0b 100644 --- a/packages/provider-telemetry/src/microcosm_provider_telemetry/service/spool.py +++ b/packages/provider-telemetry/src/microcosm_provider_telemetry/service/spool.py @@ -10,11 +10,7 @@ from pathlib import Path from typing import Any -from sqlalchemy import func, select -from sqlalchemy.orm import Session - -from microcosm_provider_telemetry.protocol import TELEMETRY_SCHEMA_VERSION -from microcosm_provider_telemetry.service.constants import ( +from microcosm_provider_core.constants import ( BATCH_SIZE, MAX_QUEUED_BYTES, PRUNE_INTERVAL_SECONDS, @@ -22,18 +18,22 @@ UPLOAD_STATE_LOCAL_ONLY, UPLOAD_STATE_PENDING, ) -from microcosm_provider_telemetry.service.database import ( +from microcosm_provider_core.database import ( create_spool_engine, create_spool_session_factory, ) -from microcosm_provider_telemetry.service.migrations import ( +from microcosm_provider_core.migrations import ( upgrade_spool_database, ) -from microcosm_provider_telemetry.service.models import ( +from microcosm_provider_core.models import ( TelemetryEventRecord, TelemetryRunRecord, serialized_json_length, ) +from sqlalchemy import func, select +from sqlalchemy.orm import Session + +from microcosm_provider_telemetry.protocol import TELEMETRY_SCHEMA_VERSION from microcosm_provider_telemetry.service.timestamps import utc_now diff --git a/packages/provider-telemetry/tests/test_delivery.py b/packages/provider-telemetry/tests/test_delivery.py index 8267fa48..0b3179ab 100644 --- a/packages/provider-telemetry/tests/test_delivery.py +++ b/packages/provider-telemetry/tests/test_delivery.py @@ -4,6 +4,20 @@ from types import SimpleNamespace from alembic import command +from microcosm_provider_core.constants import ( + PRODUCTION_COLLECTOR_URL, +) +from microcosm_provider_core.database import ( + create_spool_engine, +) +from microcosm_provider_core.migrations import ( + alembic_config, + current_database_revision, + migration_head_revision, +) +from microcosm_provider_core.models import ( + serialized_json_length, +) from microcosm_provider_telemetry.client import ( TelemetryRun, ) @@ -19,20 +33,6 @@ from microcosm_provider_telemetry.service import resources as resources_module from microcosm_provider_telemetry.service import spool as spool_module from microcosm_provider_telemetry.service.collector import CollectorDelivery -from microcosm_provider_telemetry.service.constants import ( - PRODUCTION_COLLECTOR_URL, -) -from microcosm_provider_telemetry.service.database import ( - create_spool_engine, -) -from microcosm_provider_telemetry.service.migrations import ( - alembic_config, - current_database_revision, - migration_head_revision, -) -from microcosm_provider_telemetry.service.models import ( - serialized_json_length, -) from microcosm_provider_telemetry.service.spool import EventSpool diff --git a/packages/provider-telemetry/tests/test_persistence.py b/packages/provider-telemetry/tests/test_persistence.py index 6febc38d..96400949 100644 --- a/packages/provider-telemetry/tests/test_persistence.py +++ b/packages/provider-telemetry/tests/test_persistence.py @@ -4,8 +4,8 @@ import pytest from alembic import command -from microcosm_provider_telemetry.service.database import create_spool_engine -from microcosm_provider_telemetry.service.migrations import ( +from microcosm_provider_core.database import create_spool_engine +from microcosm_provider_core.migrations import ( alembic_config, current_database_revision, ) @@ -36,7 +36,7 @@ def test_migrations_match_models_and_reopening_preserves_sequences(tmp_path): ] == [1, 2] assert spool.batch("fixture-run", "producer-a")[0] == first assert first["event_id"] != second["event_id"] - assert current_database_revision(path) == "20261007_01" + assert current_database_revision(path) == "20261008_02" with ( spool._engine.begin() as connection, alembic_config(connection=connection) as config, @@ -81,5 +81,6 @@ def test_multiple_producers_and_local_only_survive_reopening(tmp_path): "alembic_version", "telemetry_runs", "telemetry_events", + "graph_publication_jobs", } spool.close() diff --git a/tests/test_boundaries.py b/tests/test_boundaries.py index 0bf2225b..97fbe402 100644 --- a/tests/test_boundaries.py +++ b/tests/test_boundaries.py @@ -19,6 +19,8 @@ def imports(path): def test_generic_runtime_has_no_domain_or_database_imports(): forbidden = ( "microcosm_provider_telemetry", + "microcosm_provider_core", + "microcosm_provider_orrery", "microcosm", "sqlalchemy", "alembic", @@ -61,13 +63,14 @@ def audit(event, args): import microcosm_provider_client import microcosm_provider_telemetry import microcosm_provider_telemetry.client +import microcosm_provider_orrery.client assert not any(name in sys.modules for name in ("sqlalchemy", "alembic", "huggingface_hub")) """ subprocess.run([sys.executable, "-c", script], check=True) def test_database_code_has_no_raw_sql_or_manual_schema_creation(): - for path in (ROOT / "packages/provider-telemetry/src").rglob("*.py"): + for path in (ROOT / "packages").glob("*/src/**/*.py"): tree = ast.parse(path.read_text()) for node in ast.walk(tree): if isinstance(node, ast.Call): diff --git a/tests/test_package_names.py b/tests/test_package_names.py index 7c51945d..5362a1b6 100644 --- a/tests/test_package_names.py +++ b/tests/test_package_names.py @@ -5,7 +5,12 @@ from pathlib import Path ROOT = Path(__file__).resolve().parents[1] -EXPECTED_NAMES = {"microcosm-provider-client", "microcosm-provider-telemetry"} +EXPECTED_NAMES = { + "microcosm-provider-client", + "microcosm-provider-core", + "microcosm-provider-telemetry", + "microcosm-provider-orrery", +} def test_distribution_names_and_source_links(): diff --git a/tests/test_queue_compatibility.py b/tests/test_queue_compatibility.py index 7ef54362..cf65857e 100644 --- a/tests/test_queue_compatibility.py +++ b/tests/test_queue_compatibility.py @@ -5,9 +5,9 @@ import shutil from pathlib import Path +from microcosm_provider_core.database import create_spool_engine +from microcosm_provider_core.models import TelemetryEventRecord from microcosm_provider_telemetry.service import spool as spool_module -from microcosm_provider_telemetry.service.database import create_spool_engine -from microcosm_provider_telemetry.service.models import TelemetryEventRecord from microcosm_provider_telemetry.service.spool import EventSpool from sqlalchemy import select from sqlalchemy.orm import Session diff --git a/tests/test_workflows.py b/tests/test_workflows.py index c1406eee..c48f9d3e 100644 --- a/tests/test_workflows.py +++ b/tests/test_workflows.py @@ -50,6 +50,7 @@ def test_all_required_checks_include_real_collector_and_matrix(): "tests", "artifacts", "collector", + "reader", } assert jobs["tests"]["strategy"]["matrix"] == { "os": ["ubuntu-latest", "macos-latest"], @@ -64,3 +65,20 @@ def test_release_tag_matches_both_package_versions(): validate("v0.1.0") with pytest.raises(AssertionError, match="Release tag"): validate("v9.9.9") + + +def test_reader_publishes_verified_tarball_after_python_with_oidc(): + jobs = workflow("publish.yml")["jobs"] + reader = jobs["publish-reader"] + assert reader["needs"] == ["verify", "publish"] + assert reader["environment"] == "pypi" + assert reader["permissions"]["id-token"] == "write" + download = next( + s for s in reader["steps"] if "download-artifact@" in s.get("uses", "") + ) + assert download["with"]["name"] == "orrery-reader" + assert reader["steps"][-1]["run"] == "bash tools/publish-reader.sh" + assert ( + workflow("ci.yml")["jobs"]["reader"]["steps"][-1]["with"]["name"] + == "orrery-reader" + ) diff --git a/tools/check-installed.py b/tools/check-installed.py index bd1b5301..d603b2d3 100644 --- a/tools/check-installed.py +++ b/tools/check-installed.py @@ -20,7 +20,9 @@ def main(): run("uv", "venv", "--python", sys.executable, str(workspace / "venv")) base = list((ROOT / "dist").glob("microcosm_provider_client-*.whl")) telemetry = list((ROOT / "dist").glob("microcosm_provider_telemetry-*.whl")) - assert len(base) == len(telemetry) == 1, ( + core = list((ROOT / "dist").glob("microcosm_provider_core-*.whl")) + orrery = list((ROOT / "dist").glob("microcosm_provider_orrery-*.whl")) + assert len(base) == len(telemetry) == len(core) == len(orrery) == 1, ( "Expected exactly one release of each distribution" ) run("uv", "pip", "install", "--python", str(interpreter), str(base[0])) @@ -31,7 +33,16 @@ def main(): "base", cwd=workspace, ) - run("uv", "pip", "install", "--python", str(interpreter), str(telemetry[0])) + run( + "uv", + "pip", + "install", + "--python", + str(interpreter), + str(core[0]), + str(telemetry[0]), + str(orrery[0]), + ) environment = { key: value for key, value in os.environ.items() @@ -54,6 +65,14 @@ def main(): cwd=workspace, env=environment, ) + run( + str(interpreter), + "-I", + str(ROOT / "tools/installed-smoke.py"), + "combined", + cwd=workspace, + env=environment, + ) if __name__ == "__main__": diff --git a/tools/check-reader-installed.ts b/tools/check-reader-installed.ts new file mode 100644 index 00000000..65296083 --- /dev/null +++ b/tools/check-reader-installed.ts @@ -0,0 +1,65 @@ +import { mkdtempSync, writeFileSync, readdirSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; + +const files = readdirSync("reader-dist").filter((name) => + name.endsWith(".tgz"), +); +if (files.length !== 1) throw new Error("Expected exactly one reader tarball."); +const workspace = mkdtempSync(join(tmpdir(), "provider-reader-")); +function run(args: string[]) { + const result = Bun.spawnSync(args, { + cwd: workspace, + stdout: "inherit", + stderr: "inherit", + }); + if (result.exitCode !== 0) throw new Error("Installed reader check failed."); +} +try { + writeFileSync( + join(workspace, "package.json"), + JSON.stringify({ + private: true, + type: "module", + dependencies: { + "@policyengine/microcosm-provider-orrery": resolve( + "reader-dist", + files[0], + ), + "@axiom-foundation/orrery": "0.6.1", + }, + }), + ); + run(["bun", "install", "--ignore-scripts"]); + writeFileSync( + join(workspace, "check.ts"), + ` +import { loadDocument } from "@policyengine/microcosm-provider-orrery/browser"; +import { createPublicationReader } from "@policyengine/microcosm-provider-orrery/server"; +const reader = createPublicationReader({read: async () => null}); +if (await reader.read("missing") !== null) throw new Error("Expected missing publication."); +try { + await loadDocument("missing", new AbortController().signal, async () => new Response("", {status: 404})); + throw new Error("Missing publication was accepted."); +} catch (error) { + if (!(error instanceof Error) || !error.message.includes("not found")) throw error; +} +`, + ); + run(["bun", "check.ts"]); + writeFileSync( + join(workspace, "browser.ts"), + 'export { loadDocument } from "@policyengine/microcosm-provider-orrery/browser";', + ); + run([ + "bun", + "build", + "browser.ts", + "--target", + "browser", + "--outfile", + "browser.js", + ]); +} finally { + rmSync(workspace, { recursive: true, force: true }); +} diff --git a/tools/check-reader.sh b/tools/check-reader.sh new file mode 100644 index 00000000..c438509b --- /dev/null +++ b/tools/check-reader.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash +set -euo pipefail +cd typescript/orrery +bun install --frozen-lockfile +bun run format:check +bun run typecheck +bun test +bun run build +bun pm pack --destination ../../reader-dist +cd ../.. +bun tools/check-reader-installed.ts diff --git a/tools/check-release.py b/tools/check-release.py index f226ed7f..c5dd08c9 100644 --- a/tools/check-release.py +++ b/tools/check-release.py @@ -1,5 +1,6 @@ """Refuse tags that do not describe both distribution versions.""" +import json import sys import tomllib from pathlib import Path @@ -12,10 +13,12 @@ def validate(tag: str) -> None: tomllib.loads(path.read_text())["project"] for path in sorted((ROOT / "packages").glob("*/pyproject.toml")) ] - assert len(projects) == 2 + assert len(projects) == 4 assert all(tag == f"v{project['version']}" for project in projects), ( "Release tag must match both distributions" ) + reader = json.loads((ROOT / "typescript/orrery/package.json").read_text()) + assert tag == f"v{reader['version']}", "Release tag must match the reader package" telemetry = next( project for project in projects diff --git a/tools/installed-smoke.py b/tools/installed-smoke.py index d15d23b4..ce293c1c 100644 --- a/tools/installed-smoke.py +++ b/tools/installed-smoke.py @@ -40,12 +40,12 @@ def telemetry_smoke(): assert emitter._handle.process.wait(timeout=10) == 0 finally: emitter.close() - from microcosm_provider_telemetry.service.migrations import ( + from microcosm_provider_core.migrations import ( current_database_revision, ) from microcosm_provider_telemetry.service.spool import EventSpool - assert current_database_revision(path) == "20261007_01" + assert current_database_revision(path) == "20261008_02" spool = EventSpool(path) try: assert not spool.has_deliverable() @@ -56,5 +56,54 @@ def telemetry_smoke(): spool.close() +def combined_smoke(): + from microcosm_provider_client.launcher import start_services + from microcosm_provider_orrery.client import publication_inventory, publish_graph + from microcosm_provider_orrery.service.queue import GraphPublicationQueue + from microcosm_provider_telemetry.client import LocalTelemetryEmitter, TelemetryRun + + path = Path.cwd() / "combined.sqlite3" + run = TelemetryRun("combined-test", "GB", "installed") + handle = start_services( + { + "telemetry": { + "registration": run.as_registration(), + "spool_path": str(path), + "development_collector_url": "http://127.0.0.1:1", + }, + "orrery": { + "spool_path": str(path), + "development_collector_url": "http://127.0.0.1:1", + }, + } + ) + directory = Path.cwd() / "graph" + directory.mkdir() + (directory / "graph.orrery.json").write_text('{"synthetic":true}') + inventory = publication_inventory(directory, {"graph.orrery.json": "graph"}) + try: + emitter = LocalTelemetryEmitter( + run=run, transport=handle.client.for_module("telemetry") + ) + emitter.stage("compile") + emitter.complete() + handle.client.ping() # Completing telemetry must not close publication. + receipt = publish_graph(directory, inventory, spool_path=path, wait_seconds=0) + assert receipt["status"] == "pending" and "run_id" not in inventory + handle.client.for_module("orrery").send( + {"directory": str(directory), "inventory": inventory} + ) + finally: + handle.close() + assert handle.process.wait(timeout=20) == 0 + queue = GraphPublicationQueue(path) + try: + assert queue.receipt(inventory["publication_id"])["status"] == "pending" + finally: + queue.close() + + if __name__ == "__main__": - {"base": base_smoke, "telemetry": telemetry_smoke}[sys.argv[1]]() + {"base": base_smoke, "telemetry": telemetry_smoke, "combined": combined_smoke}[ + sys.argv[1] + ]() diff --git a/tools/publish-reader.sh b/tools/publish-reader.sh new file mode 100644 index 00000000..a8978e46 --- /dev/null +++ b/tools/publish-reader.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +set -euo pipefail +node -e 'const [major, minor, patch] = require("child_process").execFileSync("npm", ["--version"], {encoding:"utf8"}).trim().split(".").map(Number); if (major < 11 || (major === 11 && (minor < 5 || (minor === 5 && patch < 1)))) throw new Error("npm OIDC requires npm >=11.5.1");' +shopt -s nullglob +archives=(reader-dist/*.tgz) +if (("${#archives[@]}" != 1)); then + echo "Expected exactly one verified reader tarball." >&2 + exit 1 +fi +npm publish --access public --provenance "${archives[0]}" diff --git a/typescript/orrery/README.md b/typescript/orrery/README.md new file mode 100644 index 00000000..e7830bc6 --- /dev/null +++ b/typescript/orrery/README.md @@ -0,0 +1,13 @@ +# Orrery publication reads + +The package owns publication validation, server-side metadata/evidence reads and +browser-side bounded downloads, SHA-256 verification and Orrery parsing. +It does not contain React components, routes, storage credentials or a hosted API. + +Import `createPublicationReader` from the `/server` entry point with a storage +adapter. Import `loadDocument` from `/browser` in the runs app. Browser exports +never import Node cryptography or server credentials. Existing publication URLs +and the Orrery React component remain unchanged. + +Build and test with Bun. Package publication and consumer activation are separate +operations; do not activate consumers before verifying the published tarball. diff --git a/typescript/orrery/bun.lock b/typescript/orrery/bun.lock new file mode 100644 index 00000000..58fb6c7a --- /dev/null +++ b/typescript/orrery/bun.lock @@ -0,0 +1,88 @@ +{ + "lockfileVersion": 2, + "configVersion": 1, + "workspaces": { + "": { + "name": "@policyengine/microcosm-provider-orrery", + "devDependencies": { + "@axiom-foundation/orrery": "0.6.1", + "@types/bun": "^1.3.0", + "@types/node": "^24.0.0", + "prettier": "^3.6.0", + "typescript": "^5.9.0", + }, + "peerDependencies": { + "@axiom-foundation/orrery": "0.6.1", + }, + }, + }, + "packages": { + "@axiom-foundation/orrery": ["@axiom-foundation/orrery@0.6.1", "", { "dependencies": { "@dagrejs/dagre": "3.1.1", "@xyflow/react": "12.11.6" }, "peerDependencies": { "react": ">=18 <20", "react-dom": ">=18 <20" }, "bin": { "orrery": "dist/cli.js", "graph-explorer": "dist/cli.js" } }, "sha512-VHYAuBZ50KhSXou1ogGG5IBjG1ET3Mnr05rIwUtoUgmgFu7eg5iH9rlsLQ28KQI/Lkd6yBsdYUsE3wo7kyCkhA=="], + + "@dagrejs/dagre": ["@dagrejs/dagre@3.1.1", "", { "dependencies": { "@dagrejs/graphlib": "4.0.5" } }, "sha512-zroZB1dFOFiGgv4Xcrn1DckB1o4aOikPqD2NDQPV0WM//CXGcS6xiD0rNkqHmw6FEg4tabt4nxPLwgCWT+Vb2A=="], + + "@dagrejs/graphlib": ["@dagrejs/graphlib@4.0.5", "", {}, "sha512-7xrBTqIts3o+PMUZX97wSc+7TUbW+/rULzGNCTP6yooNVDXbzw4Wutg/H/xOutTB/c/k0YqOAavgPh4/Zk9PFA=="], + + "@types/bun": ["@types/bun@1.4.2", "", { "dependencies": { "bun-types": "1.4.2" } }, "sha512-GimotNn7+ZV0uVArItBbriZsR1oNf0+WTzPkdcFrzShI7k2norL0uzEaJT8T33dWr7O/c9ZDuAFQrctKCi72oQ=="], + + "@types/d3-color": ["@types/d3-color@3.1.3", "", {}, "sha512-iO90scth9WAbmgv7ogoq57O9YpKmFBbmoEoCHDB2xMBY0+/KVrqAaCDyCE16dUspeOvIxFFRI+0sEtqDqy2b4A=="], + + "@types/d3-drag": ["@types/d3-drag@3.0.7", "", { "dependencies": { "@types/d3-selection": "*" } }, "sha512-HE3jVKlzU9AaMazNufooRJ5ZpWmLIoc90A37WU2JMmeq28w1FQqCZswHZ3xR+SuxYftzHq6WU6KJHvqxKzTxxQ=="], + + "@types/d3-interpolate": ["@types/d3-interpolate@3.0.4", "", { "dependencies": { "@types/d3-color": "*" } }, "sha512-mgLPETlrpVV1YRJIglr4Ez47g7Yxjl1lj7YKsiMCb27VJH9W8NVM6Bb9d8kkpG/uAQS5AmbA48q2IAolKKo1MA=="], + + "@types/d3-selection": ["@types/d3-selection@3.0.12", "", {}, "sha512-Qe/KWYhEiIIxGs7HrAAjMfShxKldx19SJtr5zu53f3afPsdZNz7HHtdTLXo/kqeiWNXVycI24kSnfzBYkTzpgw=="], + + "@types/d3-transition": ["@types/d3-transition@3.0.9", "", { "dependencies": { "@types/d3-selection": "*" } }, "sha512-uZS5shfxzO3rGlu0cC3bjmMFKsXv+SmZZcgp0KD22ts4uGXp5EVYGzu/0YdwZeKmddhcAccYtREJKkPfXkZuCg=="], + + "@types/d3-zoom": ["@types/d3-zoom@3.0.9", "", { "dependencies": { "@types/d3-interpolate": "*", "@types/d3-selection": "*" } }, "sha512-0sE1406XBYJGiqD3AusTl9ZqC//2mIXix51tbom25gDCA8ri4xnSZg28CaSE8Srl6FClqABUKfsn0qghgdepMA=="], + + "@types/node": ["@types/node@24.19.1", "", { "dependencies": { "undici-types": ">=7.24.0 <7.24.7" } }, "sha512-aS3/DG0oM05K0RIXXP+hKjinGG5IgSSVGzswZxW3O0sS3pH4/fycXundUC9XsszgKCk4gHXylTEK6hyFxVxnoQ=="], + + "@xyflow/react": ["@xyflow/react@12.11.6", "", { "dependencies": { "@xyflow/system": "0.0.82", "classcat": "^5.0.3", "zustand": "^4.4.0" }, "peerDependencies": { "@types/react": ">=17", "@types/react-dom": ">=17", "react": ">=17", "react-dom": ">=17" }, "optionalPeers": ["@types/react", "@types/react-dom"] }, "sha512-9XsEJNHjatKYndszKTF/bsU7FOP9dJ6V/EQwzy3oMdtqgBuUq7BjKSwkEo+C7s4qHstHQfwwoHA3E8QfpPxZZQ=="], + + "@xyflow/system": ["@xyflow/system@0.0.82", "", { "dependencies": { "@types/d3-drag": "^3.0.7", "@types/d3-interpolate": "^3.0.4", "@types/d3-selection": "^3.0.10", "@types/d3-transition": "^3.0.8", "@types/d3-zoom": "^3.0.8", "d3-drag": "^3.0.0", "d3-interpolate": "^3.0.1", "d3-selection": "^3.0.0", "d3-zoom": "^3.0.0" } }, "sha512-4DKnL3CGtCGLRSmgDqaajRVgeksMXq/Yw4wPfdMfm7JvdIiWHGzVYOfFUztiATwdBXZqUU6HhehwUdbV9G23PQ=="], + + "bun-types": ["bun-types@1.4.2", "", { "dependencies": { "@types/node": "*" } }, "sha512-bxV1FgK7yBIzjRe5zBozIM4Bem11ZJcCXSrjWRG3YWLt8yFDePu4cLjpebO8OvPeIE9trbyPF4fuj3Cia4Fj3w=="], + + "classcat": ["classcat@5.0.5", "", {}, "sha512-JhZUT7JFcQy/EzW605k/ktHtncoo9vnyW/2GspNYwFlN1C/WmjuV/xtS04e9SOkL2sTdw0VAZ2UGCcQ9lR6p6w=="], + + "d3-color": ["d3-color@3.1.0", "", {}, "sha512-zg/chbXyeBtMQ1LbD/WSoW2DpC3I0mpmPdW+ynRTj/x2DAWYrIY7qeZIHidozwV24m4iavr15lNwIwLxRmOxhA=="], + + "d3-dispatch": ["d3-dispatch@3.0.1", "", {}, "sha512-rzUyPU/S7rwUflMyLc1ETDeBj0NRuHKKAcvukozwhshr6g6c5d8zh4c2gQjY2bZ0dXeGLWc1PF174P2tVvKhfg=="], + + "d3-drag": ["d3-drag@3.0.0", "", { "dependencies": { "d3-dispatch": "1 - 3", "d3-selection": "3" } }, "sha512-pWbUJLdETVA8lQNJecMxoXfH6x+mO2UQo8rSmZ+QqxcbyA3hfeprFgIT//HW2nlHChWeIIMwS2Fq+gEARkhTkg=="], + + "d3-ease": ["d3-ease@3.0.1", "", {}, "sha512-wR/XK3D3XcLIZwpbvQwQ5fK+8Ykds1ip7A2Txe0yxncXSdq1L9skcG7blcedkOX+ZcgxGAmLX1FrRGbADwzi0w=="], + + "d3-interpolate": ["d3-interpolate@3.0.1", "", { "dependencies": { "d3-color": "1 - 3" } }, "sha512-3bYs1rOD33uo8aqJfKP3JWPAibgw8Zm2+L9vBKEHJ2Rg+viTR7o5Mmv5mZcieN+FRYaAOWX5SJATX6k1PWz72g=="], + + "d3-selection": ["d3-selection@3.0.0", "", {}, "sha512-fmTRWbNMmsmWq6xJV8D19U/gw/bwrHfNXxrIN+HfZgnzqTHp9jOmKMhsTUjXOJnZOdZY9Q28y4yebKzqDKlxlQ=="], + + "d3-timer": ["d3-timer@3.0.1", "", {}, "sha512-ndfJ/JxxMd3nw31uyKoY2naivF+r29V+Lc0svZxe1JvvIRmi8hUsrMvdOwgS1o6uBHmiz91geQ0ylPP0aj1VUA=="], + + "d3-transition": ["d3-transition@3.0.1", "", { "dependencies": { "d3-color": "1 - 3", "d3-dispatch": "1 - 3", "d3-ease": "1 - 3", "d3-interpolate": "1 - 3", "d3-timer": "1 - 3" }, "peerDependencies": { "d3-selection": "2 - 3" } }, "sha512-ApKvfjsSR6tg06xrL434C0WydLr7JewBB3V+/39RMHsaXTOG0zmt/OAXeng5M5LBm0ojmxJrpomQVZ1aPvBL4w=="], + + "d3-zoom": ["d3-zoom@3.0.0", "", { "dependencies": { "d3-dispatch": "1 - 3", "d3-drag": "2 - 3", "d3-interpolate": "1 - 3", "d3-selection": "2 - 3", "d3-transition": "2 - 3" } }, "sha512-b8AmV3kfQaqWAuacbPuNbL6vahnOJflOhexLzMMNLga62+/nh0JzvJ0aO/5a5MVgUFGS7Hu1P9P03o3fJkDCyw=="], + + "prettier": ["prettier@3.9.9", "", { "bin": { "prettier": "bin/prettier.cjs" } }, "sha512-Z/CJHIkdujO/OtN7nXUii0Rf3VT5SRuhjBA82Xvu2XhBUgX3nhP67T0LHceBdQLex7OOFGTox+Q5Yg8Jk2Qivg=="], + + "react": ["react@19.3.0", "", {}, "sha512-E8LUcbtBWt20bbl2YoHfx4ZDBdxVTfOKtCZn9cDSJ4l6/nuoApcpIBcj47t2wZoVX8g2ZHuMHbiShgCR1T5Sog=="], + + "react-dom": ["react-dom@19.3.0", "", { "dependencies": { "scheduler": "^0.28.0" }, "peerDependencies": { "react": "^19.3.0" } }, "sha512-JDk8dgif51OjFoDE70+OT9ICyYr+69HlmihNwp1+Nsfbna3t5sIiCa9ZJktDmQ4/1b/rn26hIAR2uYXDMr5r0Q=="], + + "scheduler": ["scheduler@0.28.0", "", {}, "sha512-juorfCmIkIw8tT+p5BXSm6PJjQF/ycEYmKyzURCIt/RaZIhL+PulbQ9Yu2z1HdOJDdqDTlxA1+xKBmHXJsczAw=="], + + "typescript": ["typescript@5.9.3", "", { "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" } }, "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw=="], + + "undici-types": ["undici-types@7.24.6", "", {}, "sha512-WRNW+sJgj5OBN4/0JpHFqtqzhpbnV0GuB+OozA9gCL7a993SmU+1JBZCzLNxYsbMfIeDL+lTsphD5jN5N+n0zg=="], + + "use-sync-external-store": ["use-sync-external-store@1.7.0", "", { "peerDependencies": { "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0" } }, "sha512-6L+EeigHMQhdaIPNIFUKwfWJSwWFQ8gJbJ2DLOs5sDIegTwR9fRxvnM3uciHKjIZhFz+KAv2emhWMRvDmMcY8A=="], + + "zustand": ["zustand@4.5.7", "", { "dependencies": { "use-sync-external-store": "^1.2.2" }, "peerDependencies": { "@types/react": ">=16.8", "immer": ">=9.0.6", "react": ">=16.8" }, "optionalPeers": ["@types/react", "immer", "react"] }, "sha512-CHOUy7mu3lbD6o6LJLfllpjkzhHXSBlX8B9+qPddUsIfeF5S/UZ5q0kmCsnRqT1UHFQZchNFDDzMbQsuesHWlw=="], + + "bun-types/@types/node": ["@types/node@26.6.4", "", { "dependencies": { "undici-types": "~8.9.0" } }, "sha512-ldVPDCzj7fsaGZrLB0NuHuTvJcsNasysBAqMolr/cgxrLd1xbqxIr3XJiPnHHJUCxj5sNF1vnRj9aWnrVh5Jcg=="], + + "bun-types/@types/node/undici-types": ["undici-types@8.9.0", "", {}, "sha512-KTDyRTYX8sWmKXAikPHHSyc63CRPETMctyjKFupcC6OBLXT3xsN0e9aF7m+mIXutFWpUXuedtowG7iLOzp0kQg=="], + } +} diff --git a/typescript/orrery/package.json b/typescript/orrery/package.json new file mode 100644 index 00000000..9bbc55c5 --- /dev/null +++ b/typescript/orrery/package.json @@ -0,0 +1,46 @@ +{ + "name": "@policyengine/microcosm-provider-orrery", + "version": "0.1.0", + "description": "Validated immutable publication reads for Orrery applications", + "type": "module", + "license": "MIT", + "repository": { + "type": "git", + "url": "git+https://github.com/PolicyEngine/microcosm-local-provider.git", + "directory": "typescript/orrery" + }, + "files": [ + "dist", + "README.md" + ], + "exports": { + "./contract": { + "types": "./dist/contract.d.ts", + "import": "./dist/contract.js" + }, + "./browser": { + "types": "./dist/browser.d.ts", + "import": "./dist/browser.js" + }, + "./server": { + "types": "./dist/server.d.ts", + "import": "./dist/server.js" + } + }, + "scripts": { + "build": "tsc -p tsconfig.build.json", + "typecheck": "tsc --noEmit", + "test": "bun test", + "format:check": "prettier --check src tests *.json README.md" + }, + "peerDependencies": { + "@axiom-foundation/orrery": "0.6.1" + }, + "devDependencies": { + "@axiom-foundation/orrery": "0.6.1", + "@types/bun": "^1.3.0", + "@types/node": "^24.0.0", + "prettier": "^3.6.0", + "typescript": "^5.9.0" + } +} diff --git a/typescript/orrery/src/browser.ts b/typescript/orrery/src/browser.ts new file mode 100644 index 00000000..a42e98d6 --- /dev/null +++ b/typescript/orrery/src/browser.ts @@ -0,0 +1,98 @@ +import { parseGraphDocument } from "@axiom-foundation/orrery"; +import { + inventory, + publication, + publicationId, + MAX_FILE_BYTES, +} from "./contract.js"; + +export type Fetcher = ( + input: string | URL | Request, + init?: RequestInit, +) => Promise; + +async function boundedBytes( + response: Response, + maximum: number, +): Promise { + if (!response.body) throw new Error("The stored graph is unavailable."); + const reader = response.body.getReader(); + const chunks: Uint8Array[] = []; + let size = 0; + try { + while (true) { + const { value, done } = await reader.read(); + if (done) break; + size += value.byteLength; + if (size > maximum) + throw new Error("Response exceeds its publication size limit."); + chunks.push(value); + } + } finally { + await reader.cancel(); + } + const result = new Uint8Array(size); + let offset = 0; + for (const chunk of chunks) { + result.set(chunk, offset); + offset += chunk.byteLength; + } + return result; +} +async function digest(bytes: Uint8Array): Promise { + return Array.from( + new Uint8Array( + await crypto.subtle.digest("SHA-256", bytes as Uint8Array), + ), + (value) => value.toString(16).padStart(2, "0"), + ).join(""); +} +export async function loadDocument( + id: string, + signal: AbortSignal, + fetcher: Fetcher = fetch, +) { + publicationId(id); + const response = await fetcher(`/api/runs/${encodeURIComponent(id)}/`, { + signal, + redirect: "error", + credentials: "omit", + }); + if (response.status === 404) + throw new Error("This graph publication was not found."); + if (!response.ok) + throw new Error("Graph publication metadata is unavailable."); + const bytes = await boundedBytes(response, 256 * 1024); + const saved = publication( + JSON.parse(new TextDecoder("utf-8", { fatal: true }).decode(bytes)), + id, + ); + if ( + (await digest( + new TextEncoder().encode(JSON.stringify(inventory(saved))), + )) !== saved.inventory_sha256 + ) + throw new Error("Inventory digest does not match its publication."); + const file = saved.files.find((file) => file.role === "graph")!; + const graph = await fetcher(file.url, { + signal, + redirect: "error", + credentials: "omit", + }); + if (!graph.ok) throw new Error("The stored graph is unavailable."); + const payload = await boundedBytes( + graph, + Math.min(file.bytes, MAX_FILE_BYTES), + ); + if (payload.length !== file.bytes) + throw new Error("Graph size does not match its publication."); + const checksum = await digest(payload); + if (checksum !== file.sha256) + throw new Error("Graph digest does not match its publication."); + return { + document: parseGraphDocument( + JSON.parse(new TextDecoder("utf-8", { fatal: true }).decode(payload)), + ), + digest: checksum, + }; +} diff --git a/typescript/orrery/src/contract.ts b/typescript/orrery/src/contract.ts new file mode 100644 index 00000000..2774419b --- /dev/null +++ b/typescript/orrery/src/contract.ts @@ -0,0 +1,130 @@ +export const MAX_FILE_BYTES = 64 * 1024 * 1024; +export const MAX_TOTAL_BYTES = 512 * 1024 * 1024; +export const MAX_FILES = 256; +export type FileRole = + | "graph" + | "schema" + | "index" + | "declaration" + | "manifest" + | "binding" + | "summaries" + | "upstream"; +export interface RunFile { + name: string; + role: FileRole; + bytes: number; + sha256: string; +} +export interface Inventory { + version: 1; + publication_id: string; + files: RunFile[]; +} +export interface Publication extends Inventory { + inventory_sha256: string; + published_at: string; + files: (RunFile & { url: string })[]; +} +export class ApiError extends Error { + constructor( + public status: number, + public code: string, + ) { + super(code); + } +} +export function publicationId(value: unknown): string { + if ( + typeof value !== "string" || + !/^[A-Za-z0-9][A-Za-z0-9_-]{0,127}$/.test(value) + ) + throw new ApiError(400, "invalid_publication_id"); + return value; +} +const roleNames: Record = { + graph: /^graph\.orrery\.json$/, + schema: /^graph\.schema\.json$/, + index: /^execution\.evidence\.json$/, + declaration: /^evidence-[a-zA-Z0-9_-]+-graph\.json$/, + manifest: /^evidence-[a-zA-Z0-9_-]+-manifest\.json$/, + binding: /^evidence-[a-zA-Z0-9_-]+-binding\.json$/, + summaries: /^evidence-[a-zA-Z0-9_-]+-summaries\.json$/, + upstream: /^upstream-[a-f0-9]{64}\.json$/, +}; +export function inventory(raw: unknown): Inventory { + if (!raw || typeof raw !== "object") + throw new ApiError(400, "invalid_inventory"); + const data = raw as Record; + const id = publicationId(data.publication_id); + if ( + data.version !== 1 || + !Array.isArray(data.files) || + data.files.length === 0 || + data.files.length > MAX_FILES + ) + throw new ApiError(400, "invalid_inventory"); + const names = new Set(); + let total = 0; + const files: RunFile[] = data.files.map((raw) => { + if (!raw || typeof raw !== "object") + throw new ApiError(400, "invalid_file"); + const f = raw as RunFile; + if ( + !Object.hasOwn(roleNames, f.role) || + typeof f.name !== "string" || + !roleNames[f.role].test(f.name) || + names.has(f.name) || + !Number.isSafeInteger(f.bytes) || + f.bytes <= 0 || + f.bytes > MAX_FILE_BYTES || + typeof f.sha256 !== "string" || + !/^[a-f0-9]{64}$/.test(f.sha256) + ) + throw new ApiError(400, "invalid_file"); + names.add(f.name); + total += f.bytes; + return { name: f.name, role: f.role, bytes: f.bytes, sha256: f.sha256 }; + }); + if ( + total > MAX_TOTAL_BYTES || + files.filter((f) => f.role === "graph").length !== 1 + ) + throw new ApiError(400, "invalid_inventory"); + return { + version: 1, + publication_id: id, + files: files.sort((a, b) => + a.name < b.name ? -1 : a.name > b.name ? 1 : 0, + ), + }; +} + +export function publication(raw: unknown, expectedId: string): Publication { + const value = inventory(raw); + const saved = raw as Publication; + if ( + value.publication_id !== expectedId || + !/^[a-f0-9]{64}$/.test(saved.inventory_sha256 ?? "") || + typeof saved.published_at !== "string" || + !Number.isFinite(Date.parse(saved.published_at)) + ) + throw new ApiError(400, "invalid_publication"); + const urls = new Map(saved.files.map((file) => [file.name, file.url])); + return { + ...value, + inventory_sha256: saved.inventory_sha256, + published_at: saved.published_at, + files: value.files.map((file) => { + const url = new URL(urls.get(file.name)!); + if ( + url.protocol !== "https:" || + !url.hostname.endsWith(".public.blob.vercel-storage.com") || + url.username || + url.password + ) + throw new ApiError(400, "invalid_storage_url"); + return { ...file, url: url.toString() }; + }), + }; +} diff --git a/typescript/orrery/src/server.ts b/typescript/orrery/src/server.ts new file mode 100644 index 00000000..3dff69cf --- /dev/null +++ b/typescript/orrery/src/server.ts @@ -0,0 +1,52 @@ +import { createHash } from "node:crypto"; +import { + ApiError, + inventory, + publication, + publicationId, + type Inventory, +} from "./contract.js"; + +export function sha256(bytes: string | Uint8Array): string { + return createHash("sha256").update(bytes).digest("hex"); +} +export function inventoryDigest(value: Inventory): string { + return sha256(JSON.stringify(inventory(value))); +} +export function publicationPath(id: string): string { + return `publications/${publicationId(id)}.json`; +} +export function objectPath(value: Inventory, name: string): string { + return `objects/${value.publication_id}/${inventoryDigest(value)}/${name}`; +} +export interface PublicationStorage { + read( + path: string, + maximum: number, + ): Promise<{ bytes: Uint8Array; url: string } | null>; +} +export function createPublicationReader(storage: PublicationStorage) { + async function read(id: string) { + const object = await storage.read(publicationPath(id), 256 * 1024); + if (!object) return null; + if (object.bytes.byteLength > 256 * 1024) + throw new ApiError(400, "metadata_too_large"); + const value = publication( + JSON.parse( + new TextDecoder("utf-8", { fatal: true }).decode(object.bytes), + ), + id, + ); + if (inventoryDigest(value) !== value.inventory_sha256) + throw new ApiError(400, "inventory_digest_mismatch"); + return value; + } + return { + read, + async evidence(id: string, name: string) { + return ( + (await read(id))?.files.find((file) => file.name === name)?.url ?? null + ); + }, + }; +} diff --git a/typescript/orrery/tests/contract.test.ts b/typescript/orrery/tests/contract.test.ts new file mode 100644 index 00000000..eb3f19ae --- /dev/null +++ b/typescript/orrery/tests/contract.test.ts @@ -0,0 +1,43 @@ +import { describe, expect, test } from "bun:test"; +import { inventory, MAX_FILE_BYTES } from "../src/contract.js"; +import { inventoryDigest, objectPath } from "../src/server.js"; + +const graph = { + name: "graph.orrery.json", + role: "graph", + bytes: 100, + sha256: "a".repeat(64), +}; +const valid = { version: 1, publication_id: "uk-run-123", files: [graph] }; +describe("publication inventory", () => { + test("canonical order and content-addressed paths", () => { + const evidence = { + ...graph, + name: "execution.evidence.json", + role: "index", + }; + const a = inventory({ ...valid, files: [graph, evidence] }); + const b = inventory({ ...valid, files: [evidence, graph] }); + expect(inventoryDigest(a)).toBe(inventoryDigest(b)); + expect(objectPath(a, graph.name)).toBe( + `objects/uk-run-123/${inventoryDigest(a)}/graph.orrery.json`, + ); + }); + for (const value of [ + { ...valid, version: 2 }, + { ...valid, publication_id: "../bad" }, + { ...valid, files: [graph, graph] }, + { ...valid, files: [{ ...graph, name: "../graph.orrery.json" }] }, + { ...valid, files: [{ ...graph, bytes: MAX_FILE_BYTES + 1 }] }, + { ...valid, files: [{ ...graph, bytes: -1 }] }, + { ...valid, files: [{ ...graph, sha256: "wrong" }] }, + { ...valid, files: [{ ...graph, name: "private.h5", role: "manifest" }] }, + { + ...valid, + files: [{ ...graph, name: "unreviewed.json", role: "manifest" }], + }, + { ...valid, files: [] }, + ]) + test(`rejects ${JSON.stringify(value).slice(0, 100)}`, () => + expect(() => inventory(value)).toThrow()); +}); diff --git a/typescript/orrery/tests/reader.test.ts b/typescript/orrery/tests/reader.test.ts new file mode 100644 index 00000000..5bc02b59 --- /dev/null +++ b/typescript/orrery/tests/reader.test.ts @@ -0,0 +1,160 @@ +import { expect, test } from "bun:test"; +import { loadDocument, type Fetcher } from "../src/browser.js"; +import { + createPublicationReader, + inventoryDigest, + sha256, +} from "../src/server.js"; +import { inventory, type Publication } from "../src/contract.js"; + +const document = { + schemaVersion: "graph-explorer/v1", + id: "fixture", + title: "Fixture", + nodes: [], + edges: [], +}; +function fixture(payload = JSON.stringify(document)): Publication { + const value = inventory({ + version: 1, + publication_id: "graph", + files: [ + { + name: "graph.orrery.json", + role: "graph", + bytes: new TextEncoder().encode(payload).length, + sha256: sha256(payload), + }, + ], + }); + return { + ...value, + inventory_sha256: inventoryDigest(value), + published_at: "2026-10-08T00:00:00Z", + files: value.files.map((file) => ({ + ...file, + url: "https://example.public.blob.vercel-storage.com/graph.json", + })), + }; +} +function fetcher(saved: unknown, payload = JSON.stringify(document)): Fetcher { + let calls = 0; + return async (_url, init) => { + expect(init?.credentials).toBe("omit"); + expect(init?.redirect).toBe("error"); + return ++calls === 1 ? Response.json(saved) : new Response(payload); + }; +} +const signal = () => new AbortController().signal; +test("browser verifies metadata, exact bytes and graph parser", async () => { + const result = await loadDocument("graph", signal(), fetcher(fixture())); + expect(result.document.id).toBe("fixture"); + expect(result.digest).toBe(sha256(JSON.stringify(document))); +}); +test("missing publications return an explicit error", async () => { + await expect( + loadDocument( + "graph", + signal(), + async () => new Response("", { status: 404 }), + ), + ).rejects.toThrow("not found"); +}); +for (const field of [ + "publication_id", + "inventory_sha256", + "published_at", +] as const) { + test("rejects invalid " + field, async () => { + const saved = { ...fixture(), [field]: "invalid" }; + await expect( + loadDocument("graph", signal(), fetcher(saved)), + ).rejects.toThrow(); + }); +} +for (const url of [ + "http://example.public.blob.vercel-storage.com/x", + "https://example.com/x", + "https://user:secret@example.public.blob.vercel-storage.com/x", +]) { + test("rejects credential-bearing or non-storage URL " + url, async () => { + const saved = fixture(); + saved.files[0].url = url; + await expect( + loadDocument("graph", signal(), fetcher(saved)), + ).rejects.toThrow(); + }); +} +test("rejects modified graph bytes", async () => { + await expect( + loadDocument( + "graph", + signal(), + fetcher(fixture(), "x".repeat(JSON.stringify(document).length)), + ), + ).rejects.toThrow("digest"); +}); +test("bounds graph streaming before parsing", async () => { + await expect( + loadDocument( + "graph", + signal(), + fetcher(fixture(), JSON.stringify(document) + "extra"), + ), + ).rejects.toThrow("size limit"); +}); +test("bounds metadata streaming", async () => { + await expect( + loadDocument( + "graph", + signal(), + async () => new Response("x".repeat(256 * 1024 + 1)), + ), + ).rejects.toThrow("size limit"); +}); +test("rejects a valid but incorrect inventory digest", async () => { + await expect( + loadDocument( + "graph", + signal(), + fetcher({ ...fixture(), inventory_sha256: "a".repeat(64) }), + ), + ).rejects.toThrow("Inventory digest"); +}); +test("server reads only completed publications and listed evidence", async () => { + const saved = fixture(); + const reads: string[] = []; + const reader = createPublicationReader({ + async read(path, maximum) { + reads.push(path); + expect(maximum).toBe(256 * 1024); + return path === "publications/graph.json" + ? { + bytes: new TextEncoder().encode(JSON.stringify(saved)), + url: "unused", + } + : null; + }, + }); + expect(await reader.read("missing")).toBeNull(); + expect((await reader.read("graph"))?.publication_id).toBe("graph"); + expect(await reader.evidence("graph", "private.h5")).toBeNull(); + expect(await reader.evidence("graph", "graph.orrery.json")).toBe( + saved.files[0].url, + ); + expect(reads.every((path) => path.startsWith("publications/"))).toBe(true); +}); +test("server rejects corrupt stored metadata", async () => { + const saved = { ...fixture(), inventory_sha256: "a".repeat(64) }; + const reader = createPublicationReader({ + async read() { + return { + bytes: new TextEncoder().encode(JSON.stringify(saved)), + url: "unused", + }; + }, + }); + await expect(reader.read("graph")).rejects.toThrow( + "inventory_digest_mismatch", + ); +}); diff --git a/typescript/orrery/tsconfig.build.json b/typescript/orrery/tsconfig.build.json new file mode 100644 index 00000000..0f6eea2d --- /dev/null +++ b/typescript/orrery/tsconfig.build.json @@ -0,0 +1,11 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "noEmit": false, + "declaration": true, + "outDir": "dist", + "rootDir": "src" + }, + "include": ["src"], + "exclude": ["tests"] +} diff --git a/typescript/orrery/tsconfig.json b/typescript/orrery/tsconfig.json new file mode 100644 index 00000000..8c1f9cdc --- /dev/null +++ b/typescript/orrery/tsconfig.json @@ -0,0 +1,13 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "NodeNext", + "moduleResolution": "NodeNext", + "lib": ["ES2023", "DOM"], + "strict": true, + "skipLibCheck": true, + "types": ["bun"], + "noEmit": true + }, + "include": ["src", "tests"] +} diff --git a/uv.lock b/uv.lock index d092a9be..b35c626c 100644 --- a/uv.lock +++ b/uv.lock @@ -6,6 +6,8 @@ requires-python = ">=3.13" members = [ "microcosm-local-provider-workspace", "microcosm-provider-client", + "microcosm-provider-core", + "microcosm-provider-orrery", "microcosm-provider-telemetry", ] @@ -641,6 +643,40 @@ dependencies = [ [package.metadata] requires-dist = [{ name = "psutil", specifier = ">=6,<8" }] +[[package]] +name = "microcosm-provider-core" +version = "0.1.0" +source = { editable = "packages/provider-core" } +dependencies = [ + { name = "alembic" }, + { name = "huggingface-hub" }, + { name = "sqlalchemy" }, +] + +[package.metadata] +requires-dist = [ + { name = "alembic", specifier = ">=1.13.3,<2" }, + { name = "huggingface-hub", specifier = ">=0.20,<2" }, + { name = "sqlalchemy", specifier = ">=2,<3" }, +] + +[[package]] +name = "microcosm-provider-orrery" +version = "0.1.0" +source = { editable = "packages/provider-orrery" } +dependencies = [ + { name = "httpx" }, + { name = "microcosm-provider-client" }, + { name = "microcosm-provider-core" }, +] + +[package.metadata] +requires-dist = [ + { name = "httpx", specifier = ">=0.27,<1" }, + { name = "microcosm-provider-client", editable = "packages/provider-client" }, + { name = "microcosm-provider-core", editable = "packages/provider-core" }, +] + [[package]] name = "microcosm-provider-telemetry" version = "0.1.0" @@ -649,6 +685,7 @@ dependencies = [ { name = "alembic" }, { name = "huggingface-hub" }, { name = "microcosm-provider-client" }, + { name = "microcosm-provider-core" }, { name = "psutil" }, { name = "sqlalchemy" }, ] @@ -658,6 +695,7 @@ requires-dist = [ { name = "alembic", specifier = ">=1.13.3,<2" }, { name = "huggingface-hub", specifier = ">=0.20,<2" }, { name = "microcosm-provider-client", editable = "packages/provider-client" }, + { name = "microcosm-provider-core", editable = "packages/provider-core" }, { name = "psutil", specifier = ">=6,<8" }, { name = "sqlalchemy", specifier = ">=2,<3" }, ] From 6c93210a91679df73508a4541f393507df9bf8bb Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Fri, 9 Oct 2026 01:47:34 +0400 Subject: [PATCH 2/5] Type graph publication records and claimed upload jobs --- changelog.d/3-publication-types.changed.md | 1 + .../src/microcosm_provider_orrery/client.py | 26 +++++-- .../microcosm_provider_orrery/contracts.py | 57 ++++++++++++++ .../src/microcosm_provider_orrery/py.typed | 1 + .../service/queue.py | 75 ++++++++++++------- .../provider-orrery/tests/test_publication.py | 62 ++++++++++++++- 6 files changed, 188 insertions(+), 34 deletions(-) create mode 100644 changelog.d/3-publication-types.changed.md create mode 100644 packages/provider-orrery/src/microcosm_provider_orrery/contracts.py create mode 100644 packages/provider-orrery/src/microcosm_provider_orrery/py.typed diff --git a/changelog.d/3-publication-types.changed.md b/changelog.d/3-publication-types.changed.md new file mode 100644 index 00000000..eb6d1f10 --- /dev/null +++ b/changelog.d/3-publication-types.changed.md @@ -0,0 +1 @@ +Define explicit types for publication inventories and results, and return a frozen record when reserving a graph upload. diff --git a/packages/provider-orrery/src/microcosm_provider_orrery/client.py b/packages/provider-orrery/src/microcosm_provider_orrery/client.py index c0b3d13e..db31adb6 100644 --- a/packages/provider-orrery/src/microcosm_provider_orrery/client.py +++ b/packages/provider-orrery/src/microcosm_provider_orrery/client.py @@ -2,9 +2,15 @@ import os import time -from collections.abc import Callable +from collections.abc import Callable, Mapping from pathlib import Path +from microcosm_provider_orrery.contracts import ( + PublicationFileRole, + PublicationInventory, + PublicationReceipt, +) + def default_spool_path() -> Path: """Keep Microcosm's existing cache path so queued jobs survive migration.""" @@ -13,7 +19,12 @@ def default_spool_path() -> Path: return root / "microcosm" / "telemetry" / "events.sqlite3" -def publication_inventory(directory: Path, roles, *, publication_id=None) -> dict: +def publication_inventory( + directory: Path, + roles: Mapping[str, PublicationFileRole], + *, + publication_id: str | None = None, +) -> PublicationInventory: """Hash an explicit set of public-safe graph files without network access.""" from microcosm_provider_orrery.service.queue import ( publication_inventory as inventory, @@ -24,12 +35,12 @@ def publication_inventory(directory: Path, roles, *, publication_id=None) -> dic def publish_graph( directory: Path, - inventory: dict, + inventory: PublicationInventory, *, spool_path: Path | None = None, available: Callable[[], bool] = lambda: False, wait_seconds: float = 30.0, -) -> dict: +) -> PublicationReceipt: """Persist before waiting; unavailable services never lose a publication job.""" from microcosm_provider_orrery.service.queue import GraphPublicationQueue @@ -39,9 +50,14 @@ def publish_graph( deadline = time.monotonic() + max(0, wait_seconds) while available() and time.monotonic() < deadline: receipt = queue.receipt(inventory["publication_id"]) + assert receipt is not None, ( + "A queued graph publication must have a receipt." + ) if receipt.get("attempts", 0) or receipt["status"] == "published": return receipt time.sleep(0.1) - return queue.receipt(inventory["publication_id"]) + receipt = queue.receipt(inventory["publication_id"]) + assert receipt is not None, "A queued graph publication must have a receipt." + return receipt finally: queue.close() diff --git a/packages/provider-orrery/src/microcosm_provider_orrery/contracts.py b/packages/provider-orrery/src/microcosm_provider_orrery/contracts.py new file mode 100644 index 00000000..94f4452f --- /dev/null +++ b/packages/provider-orrery/src/microcosm_provider_orrery/contracts.py @@ -0,0 +1,57 @@ +"""Explicit data contracts for immutable graph publication.""" + +from dataclasses import dataclass +from pathlib import Path +from typing import Literal, NotRequired, TypedDict + +type PublicationFileRole = Literal[ + "graph", + "schema", + "index", + "declaration", + "manifest", + "binding", + "summaries", + "upstream", +] + + +class PublicationFile(TypedDict): + """One public file, identified by its exact bytes and allowed role.""" + + name: str + role: PublicationFileRole + bytes: int + sha256: str + + +class PublicationInventory(TypedDict): + """The versioned file inventory submitted to the publication API.""" + + version: Literal[1] + publication_id: str + files: list[PublicationFile] + + +class PublicationReceipt(TypedDict): + """Local publication result, including retry and build-snapshot fields.""" + + version: Literal[1] + publication_id: str | None + status: Literal["pending", "published", "skipped", "failed", "no_evidence"] + error_code: str | None + url: NotRequired[str | None] + attempts: NotRequired[int] + inventory_sha256: NotRequired[str] + preserved_directory: NotRequired[str] + recorded_execution: NotRequired[bool] + + +@dataclass(frozen=True) +class ClaimedGraphPublication: + """A job reserved until lease_until; the timestamp also identifies its lease.""" + + publication_id: str + directory: Path + inventory: PublicationInventory + lease_until: float diff --git a/packages/provider-orrery/src/microcosm_provider_orrery/py.typed b/packages/provider-orrery/src/microcosm_provider_orrery/py.typed new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/packages/provider-orrery/src/microcosm_provider_orrery/py.typed @@ -0,0 +1 @@ + diff --git a/packages/provider-orrery/src/microcosm_provider_orrery/service/queue.py b/packages/provider-orrery/src/microcosm_provider_orrery/service/queue.py index 49a3ea91..57285788 100644 --- a/packages/provider-orrery/src/microcosm_provider_orrery/service/queue.py +++ b/packages/provider-orrery/src/microcosm_provider_orrery/service/queue.py @@ -13,6 +13,7 @@ import uuid from collections.abc import Callable, Mapping from pathlib import Path +from typing import cast from urllib.parse import urlsplit import httpx @@ -24,6 +25,14 @@ from microcosm_provider_core.models import GraphPublicationJob from sqlalchemy import select, update +from microcosm_provider_orrery.contracts import ( + ClaimedGraphPublication, + PublicationFile, + PublicationFileRole, + PublicationInventory, + PublicationReceipt, +) + MAX_GRAPH_FILE_BYTES = 64 * 1024 * 1024 MAX_GRAPH_TOTAL_BYTES = 512 * 1024 * 1024 GRAPH_PUBLICATION_ACTION = "graph_publication" @@ -47,26 +56,34 @@ def _digest(payload: bytes) -> str: return hashlib.sha256(payload).hexdigest() -def inventory_digest(inventory: Mapping) -> str: +def inventory_digest(inventory: PublicationInventory) -> str: """Digest the versioned field order shared with the publication API.""" - inventory = { + ordered: PublicationInventory = { "version": inventory["version"], "publication_id": inventory["publication_id"], "files": [ - {key: item[key] for key in ("name", "role", "bytes", "sha256")} + { + "name": item["name"], + "role": item["role"], + "bytes": item["bytes"], + "sha256": item["sha256"], + } for item in sorted(inventory["files"], key=lambda file: file["name"]) ], } return _digest( - json.dumps(inventory, separators=(",", ":"), ensure_ascii=True).encode() + json.dumps(ordered, separators=(",", ":"), ensure_ascii=True).encode() ) def publication_inventory( - directory: Path, roles: Mapping[str, str], *, publication_id: str | None = None -) -> dict: + directory: Path, + roles: Mapping[str, PublicationFileRole], + *, + publication_id: str | None = None, +) -> PublicationInventory: """Hash an explicit allowlist of files; publication identity is not a run ID.""" - files = [] + files: list[PublicationFile] = [] for name, role in sorted(roles.items()): if role not in GRAPH_PUBLICATION_ROLES or not re.fullmatch( GRAPH_PUBLICATION_ROLES[role], name @@ -104,14 +121,14 @@ def publication_inventory( class GraphPublicationQueue: """SQLite job records with leases; retained until publication or operator action.""" - def __init__(self, path: Path): + def __init__(self, path: Path) -> None: self.path = Path(path) self.path.parent.mkdir(parents=True, exist_ok=True) self._engine = create_spool_engine(self.path) upgrade_spool_database(self._engine) self._sessions = create_spool_session_factory(self._engine) - def enqueue(self, directory: Path, inventory: dict) -> None: + def enqueue(self, directory: Path, inventory: PublicationInventory) -> None: """Durably register exact bytes without a telemetry run or credential.""" directory = Path(directory).resolve() expected = publication_inventory( @@ -159,8 +176,13 @@ def claim( *, lease_seconds: float = GRAPH_PUBLICATION_LEASE_SECONDS, publication_id: str | None = None, - ) -> dict | None: - """Atomically lease one eligible job across concurrent emitter processes.""" + ) -> ClaimedGraphPublication | None: + """Reserve one pending job whose retry delay and previous lease expired. + + A conditional database update allows only one current lease. Workers + that exit leave their jobs eligible again after lease expiry; completion + uses the returned lease timestamp to reject outdated workers. + """ now = time.time() with self._sessions.begin() as session: statement = select(GraphPublicationJob.publication_id) @@ -189,18 +211,19 @@ def claim( if not result.rowcount: return None job = session.get(GraphPublicationJob, id_) - return { - "publication_id": id_, - "directory": job.directory, - "inventory": job.inventory, - "lease_until": job.lease_until, - } + return ClaimedGraphPublication( + publication_id=id_, + directory=Path(job.directory), + inventory=cast(PublicationInventory, job.inventory), + lease_until=job.lease_until, + ) - def receipt(self, id_: str) -> dict | None: + def receipt(self, id_: str) -> PublicationReceipt | None: """Read the latest local result; old HF receipt snapshots remain unchanged.""" with self._sessions() as session: job = session.get(GraphPublicationJob, id_) - return None if job is None else dict(job.receipt) + # ORM JSON columns are generic at the persistence boundary. + return None if job is None else cast(PublicationReceipt, dict(job.receipt)) def finish( self, @@ -238,7 +261,9 @@ def finish( "error_code": error_code, "url": url, "attempts": job.attempts, - "inventory_sha256": inventory_digest(job.inventory), + "inventory_sha256": inventory_digest( + cast(PublicationInventory, job.inventory) + ), } # This is a separate latest-result file, not the build/HF receipt. path = Path(job.directory) / GRAPH_PUBLICATION_STATUS_FILENAME @@ -268,7 +293,7 @@ def __init__( origin: str, client: httpx.Client | None = None, invalidate_credential: Callable[[], None] | None = None, - ): + ) -> None: parsed = urlsplit(origin) if parsed.scheme != "https" and not ( parsed.scheme == "http" @@ -295,12 +320,12 @@ def flush_once(self, *, publication_id: str | None = None) -> bool: if job is None: return False id_, inventory, directory = ( - job["publication_id"], - job["inventory"], - Path(job["directory"]), + job.publication_id, + job.inventory, + job.directory, ) error = "publication_unavailable" - lease = job["lease_until"] + lease = job.lease_until try: status, token = self.credential() if token is None: diff --git a/packages/provider-orrery/tests/test_publication.py b/packages/provider-orrery/tests/test_publication.py index 033f43a1..69dce888 100644 --- a/packages/provider-orrery/tests/test_publication.py +++ b/packages/provider-orrery/tests/test_publication.py @@ -1,9 +1,11 @@ """Durable graph jobs use fake HTTP, independently of the visualization app.""" import json +from dataclasses import FrozenInstanceError import httpx import pytest +from microcosm_provider_orrery.contracts import ClaimedGraphPublication from microcosm_provider_orrery.service.queue import ( GraphPublicationDelivery, GraphPublicationQueue, @@ -25,12 +27,64 @@ def test_jobs_survive_restart_without_run_registration_or_event_retention(tmp_pa queue.enqueue(directory, inventory) restarted = GraphPublicationQueue(queue.path) job = restarted.claim() - assert job["inventory"] == inventory + assert isinstance(job, ClaimedGraphPublication) + assert job.inventory == inventory assert "run_id" not in inventory - assert job["publication_id"] == inventory["publication_id"] + assert job.publication_id == inventory["publication_id"] + assert job.directory == directory.resolve() assert restarted.claim() is None +def test_claim_is_a_frozen_record_and_expired_leases_can_be_reclaimed( + tmp_path, monkeypatch +): + from microcosm_provider_orrery.service.queue import time + + directory, inventory = source(tmp_path) + queue = GraphPublicationQueue(tmp_path / "jobs.sqlite3") + queue.enqueue(directory, inventory) + clock = [1000.0] + monkeypatch.setattr(time, "time", lambda: clock[0]) + claimed = queue.claim(lease_seconds=5) + assert isinstance(claimed, ClaimedGraphPublication) + assert claimed.lease_until == 1005.0 + with pytest.raises(FrozenInstanceError): + claimed.lease_until = 0 + assert GraphPublicationQueue(queue.path).claim() is None + clock[0] = 1005.0 + replacement = GraphPublicationQueue(queue.path).claim() + assert replacement.publication_id == claimed.publication_id + assert replacement.lease_until > claimed.lease_until + + +def test_claim_respects_retry_delay_and_published_jobs_are_not_reclaimed( + tmp_path, monkeypatch +): + from microcosm_provider_orrery.service.queue import time + + directory, inventory = source(tmp_path) + queue = GraphPublicationQueue(tmp_path / "jobs.sqlite3") + queue.enqueue(directory, inventory) + clock = [1000.0] + monkeypatch.setattr(time, "time", lambda: clock[0]) + claimed = queue.claim() + queue.finish( + claimed.publication_id, + error_code="publication_network_error", + expected_lease=claimed.lease_until, + ) + assert queue.claim() is None + clock[0] = 1002.0 + retried = queue.claim() + assert retried.publication_id == claimed.publication_id + queue.finish( + retried.publication_id, + url="https://runs.example/runs/published/", + expected_lease=retried.lease_until, + ) + assert queue.claim() is None + + def test_queue_rejects_changed_bytes_and_conflicting_inventory(tmp_path): directory, inventory = source(tmp_path) queue = GraphPublicationQueue(tmp_path / "jobs.sqlite3") @@ -161,14 +215,14 @@ def test_expired_worker_cannot_release_a_replacement_workers_lease(tmp_path): queue.finish( id_, error_code="publication_network_error", - expected_lease=expired["lease_until"], + expected_lease=expired.lease_until, ) assert queue.receipt(id_) == initial assert queue.claim() is None replacement.finish( id_, url=f"https://runs.example/runs/{id_}/", - expected_lease=current["lease_until"], + expected_lease=current.lease_until, ) assert queue.receipt(id_)["status"] == "published" From 06e47847983ee67ccdb6a596062251322d07f793 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Fri, 9 Oct 2026 01:51:06 +0400 Subject: [PATCH 3/5] Return a typed identifier from the conditional upload claim --- .../src/microcosm_provider_orrery/service/queue.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/packages/provider-orrery/src/microcosm_provider_orrery/service/queue.py b/packages/provider-orrery/src/microcosm_provider_orrery/service/queue.py index 57285788..11facd78 100644 --- a/packages/provider-orrery/src/microcosm_provider_orrery/service/queue.py +++ b/packages/provider-orrery/src/microcosm_provider_orrery/service/queue.py @@ -199,7 +199,7 @@ def claim( ) if id_ is None: return None - result = session.execute( + claimed_id = session.scalar( update(GraphPublicationJob) .where( GraphPublicationJob.publication_id == id_, @@ -207,10 +207,14 @@ def claim( GraphPublicationJob.lease_until <= now, ) .values(lease_until=now + lease_seconds) + .returning(GraphPublicationJob.publication_id) ) - if not result.rowcount: + if claimed_id is None: return None job = session.get(GraphPublicationJob, id_) + assert job is not None, ( + "A claimed publication must exist in its transaction." + ) return ClaimedGraphPublication( publication_id=id_, directory=Path(job.directory), From 018805010aefeed3c9cdf8a7a287fc465d6c3792 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Fri, 9 Oct 2026 19:03:52 +0400 Subject: [PATCH 4/5] Keep graph delivery running after missing files or queue errors A job whose preserved directory was removed no longer stops the Orrery module: it stays pending with preserved_files_missing and backs off, and claims take the earliest-due job first so other jobs proceed. The latest publication.status.json copy is written after the queue commit, so a missing directory cannot roll back the recorded result. The module's tick reports an unexpected local error once instead of letting the host stop the module. Type the module's queue and delivery attributes. Co-Authored-By: Claude Opus 5.5 --- changelog.d/3-publication-failures.fixed.md | 1 + .../service/module.py | 30 ++++++-- .../service/queue.py | 42 +++++++++--- .../provider-orrery/tests/test_publication.py | 68 +++++++++++++++++++ 4 files changed, 128 insertions(+), 13 deletions(-) create mode 100644 changelog.d/3-publication-failures.fixed.md diff --git a/changelog.d/3-publication-failures.fixed.md b/changelog.d/3-publication-failures.fixed.md new file mode 100644 index 00000000..ebbb7780 --- /dev/null +++ b/changelog.d/3-publication-failures.fixed.md @@ -0,0 +1 @@ +Keep graph publication running when a preserved directory is removed or the local queue reports an error, and record the latest status only after the queue commit. diff --git a/packages/provider-orrery/src/microcosm_provider_orrery/service/module.py b/packages/provider-orrery/src/microcosm_provider_orrery/service/module.py index c1e508e5..36d5e70c 100644 --- a/packages/provider-orrery/src/microcosm_provider_orrery/service/module.py +++ b/packages/provider-orrery/src/microcosm_provider_orrery/service/module.py @@ -1,6 +1,8 @@ """Completed graph delivery, independent of telemetry registration and retention.""" +import sys from pathlib import Path +from typing import cast from microcosm_provider_client.contracts import JsonObject, ModuleContext from microcosm_provider_core.auth import ( @@ -10,8 +12,10 @@ ) from microcosm_provider_core.constants import PRODUCTION_COLLECTOR_URL +from microcosm_provider_orrery.contracts import PublicationInventory from microcosm_provider_orrery.service.queue import ( DEFAULT_GRAPH_PUBLICATION_ORIGIN, + GRAPH_WORKER_ERROR_MESSAGE, GraphPublicationDelivery, GraphPublicationQueue, ) @@ -20,10 +24,13 @@ class OrreryModule: def __init__(self, configuration: JsonObject, context: ModuleContext): self.configuration, self.context = configuration, context - self.queue = self.delivery = None + self.queue: GraphPublicationQueue | None = None + self.delivery: GraphPublicationDelivery | None = None + self._warned = False def initialize(self) -> None: - self.queue = GraphPublicationQueue(Path(self.configuration["spool_path"])) + queue = GraphPublicationQueue(Path(self.configuration["spool_path"])) + self.queue = queue origin = ( _development_collector_url(self.configuration["development_collector_url"]) if self.configuration.get("development_collector_url") @@ -31,7 +38,7 @@ def initialize(self) -> None: ) session = shared_session(self.context.services, origin) self.delivery = GraphPublicationDelivery( - self.queue, + queue, credential=session.credential, invalidate_credential=session.invalidate, origin=self.configuration.get( @@ -40,10 +47,23 @@ def initialize(self) -> None: ) def handle_message(self, message: JsonObject) -> None: - self.queue.enqueue(Path(message["directory"]), dict(message["inventory"])) + assert self.queue is not None, "The host initializes modules first." + # enqueue() recomputes the inventory from the files and rejects any + # message whose inventory differs, so this boundary is validated. + inventory = cast(PublicationInventory, dict(message["inventory"])) + self.queue.enqueue(Path(message["directory"]), inventory) def tick(self, now: float) -> None: - self.delivery.flush_once() + assert self.delivery is not None, "The host initializes modules first." + try: + self.delivery.flush_once() + except Exception: + # The host stops a module whose tick raises. A database or + # filesystem error leaves the job leased for a later retry; it + # must not end delivery of every other job. + if not self._warned: + print(GRAPH_WORKER_ERROR_MESSAGE, file=sys.stderr, flush=True) + self._warned = True def parent_exited(self) -> None: """Preserved jobs remain eligible when a later host resumes delivery.""" diff --git a/packages/provider-orrery/src/microcosm_provider_orrery/service/queue.py b/packages/provider-orrery/src/microcosm_provider_orrery/service/queue.py index 11facd78..715fc5dd 100644 --- a/packages/provider-orrery/src/microcosm_provider_orrery/service/queue.py +++ b/packages/provider-orrery/src/microcosm_provider_orrery/service/queue.py @@ -39,6 +39,10 @@ GRAPH_PUBLICATION_WAIT_SECONDS = 30.0 GRAPH_PUBLICATION_LEASE_SECONDS = 300.0 GRAPH_PUBLICATION_STATUS_FILENAME = "publication.status.json" +GRAPH_WORKER_ERROR_MESSAGE = ( + "Microcosm graph publication hit a local queue error; pending jobs remain " + "queued for a later attempt." +) DEFAULT_GRAPH_PUBLICATION_ORIGIN = "https://microcosm-runs.vercel.app" GRAPH_PUBLICATION_ROLES = { "graph": r"graph\.orrery\.json", @@ -118,6 +122,16 @@ def publication_inventory( return {"version": 1, "publication_id": id_, "files": files} +def _write_status(directory: Path, receipt: PublicationReceipt) -> None: + path = directory / GRAPH_PUBLICATION_STATUS_FILENAME + temporary = path.with_suffix(f".{uuid.uuid4().hex}.tmp") + try: + temporary.write_text(json.dumps(receipt, sort_keys=True) + "\n") + temporary.replace(path) + except OSError: + temporary.unlink(missing_ok=True) + + class GraphPublicationQueue: """SQLite job records with leases; retained until publication or operator action.""" @@ -195,7 +209,11 @@ def claim( GraphPublicationJob.status == "pending", GraphPublicationJob.next_attempt_at <= now, GraphPublicationJob.lease_until <= now, - ).limit(1) + ) + # Earliest-due first, so a job that keeps failing cannot + # repeatedly displace other pending publications. + .order_by(GraphPublicationJob.next_attempt_at) + .limit(1) ) if id_ is None: return None @@ -258,10 +276,10 @@ def finish( if job is None: return job.next_attempt_at = time.time() + min(300, 2 ** min(job.attempts, 8)) - job.receipt = { + receipt: PublicationReceipt = { "version": 1, "publication_id": id_, - "status": job.status, + "status": "published" if url else "pending", "error_code": error_code, "url": url, "attempts": job.attempts, @@ -269,11 +287,12 @@ def finish( cast(PublicationInventory, job.inventory) ), } - # This is a separate latest-result file, not the build/HF receipt. - path = Path(job.directory) / GRAPH_PUBLICATION_STATUS_FILENAME - temporary = path.with_suffix(f".{uuid.uuid4().hex}.tmp") - temporary.write_text(json.dumps(job.receipt, sort_keys=True) + "\n") - temporary.replace(path) + job.receipt = dict(receipt) + directory = Path(job.directory) + # The queue row is authoritative. This separate latest-result file (not + # the build/HF receipt) is a convenience copy, written only after the + # commit: a deleted preserved directory must not roll back the result. + _write_status(directory, receipt) def retry(self, id_: str | None = None) -> None: """Make preserved pending jobs eligible again, including missing credentials.""" @@ -330,6 +349,13 @@ def flush_once(self, *, publication_id: str | None = None) -> bool: ) error = "publication_unavailable" lease = job.lease_until + if not directory.is_dir(): + # Retained until an operator restores the files to this directory; + # the retry backoff lets other pending jobs proceed meanwhile. + self.queue.finish( + id_, error_code="preserved_files_missing", expected_lease=lease + ) + return False try: status, token = self.credential() if token is None: diff --git a/packages/provider-orrery/tests/test_publication.py b/packages/provider-orrery/tests/test_publication.py index 69dce888..df0fde54 100644 --- a/packages/provider-orrery/tests/test_publication.py +++ b/packages/provider-orrery/tests/test_publication.py @@ -296,3 +296,71 @@ def flush_once(self, *, publication_id): ) assert calls == [inventory["publication_id"]] assert snapshot.read_text() == '{"status":"pending"}' + + +def test_deleted_preserved_directory_is_retained_without_stopping_delivery( + tmp_path, monkeypatch +): + import shutil + + from microcosm_provider_orrery.service.queue import time + + clock = [1000.0] + monkeypatch.setattr(time, "time", lambda: clock[0]) + queue = GraphPublicationQueue(tmp_path / "jobs.sqlite3") + (tmp_path / "removed").mkdir() + (tmp_path / "kept").mkdir() + removed, removed_inventory = source(tmp_path / "removed") + queue.enqueue(removed, removed_inventory) + clock[0] = 1001.0 + kept, kept_inventory = source(tmp_path / "kept") + queue.enqueue(kept, kept_inventory) + shutil.rmtree(removed) + delivery = GraphPublicationDelivery( + queue, credential=lambda: ("missing", None), origin="https://runs.example" + ) + + assert delivery.flush_once() is False + receipt = queue.receipt(removed_inventory["publication_id"]) + assert receipt["status"] == "pending" + assert receipt["error_code"] == "preserved_files_missing" + assert receipt["attempts"] == 1 + # The failed job backs off, so the next claim reaches the other job. + assert queue.claim().publication_id == kept_inventory["publication_id"] + + +def test_status_file_failure_does_not_roll_back_the_queue_result(tmp_path): + directory, inventory = source(tmp_path) + queue = GraphPublicationQueue(tmp_path / "jobs.sqlite3") + queue.enqueue(directory, inventory) + claimed = queue.claim() + (directory / "publication.status.json").mkdir() + queue.finish( + claimed.publication_id, + url="https://runs.example/runs/published/", + expected_lease=claimed.lease_until, + ) + assert queue.receipt(claimed.publication_id)["status"] == "published" + assert list(directory.glob("*.tmp")) == [] + + +def test_orrery_module_survives_a_local_queue_error(capsys): + from microcosm_provider_orrery.service.module import OrreryModule + from sqlalchemy.exc import OperationalError + + attempts = [] + + class Delivery: + def flush_once(self): + attempts.append(None) + if len(attempts) == 1: + raise OperationalError("UPDATE", {}, Exception("database is locked")) + return False + + module = OrreryModule({}, None) + module.delivery = Delivery() + module.tick(0.0) + module.tick(1.0) + module.tick(2.0) + assert len(attempts) == 3 + assert capsys.readouterr().err.count("local queue error") == 1 From 7fef7c91641c9fc487e55d0e6b4717eb62a92376 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Mon, 12 Oct 2026 18:40:22 +0400 Subject: [PATCH 5/5] fix: preserve build outcomes and shared emitter lifetime --- .github/workflows/ci.yml | 2 +- README.md | 8 +- changelog.d/3-build-outcomes.fixed.md | 1 + src/microcosm_emitter/telemetry/client.py | 44 +++++- src/microcosm_emitter/telemetry/constants.py | 6 + src/microcosm_emitter/telemetry/protocol.py | 6 +- src/microcosm_emitter/telemetry/requests.py | 1 + .../telemetry/service/collector.py | 31 +++- .../telemetry/service/lifecycle.py | 72 ++++++++- tests/telemetry/test_client.py | 107 +++++++++++++ tests/telemetry/test_delivery.py | 117 ++++++++++++++ tests/telemetry/test_module.py | 145 ++++++++++++++++++ tests/test_collector_contract.py | 125 +++++++++++++++ tests/test_subprocess.py | 83 ++++++++++ tools/check-collector.sh | 2 +- tools/installed-smoke.py | 57 ++++++- 16 files changed, 787 insertions(+), 20 deletions(-) create mode 100644 changelog.d/3-build-outcomes.fixed.md diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7543d100..dba7400e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -73,7 +73,7 @@ jobs: - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 with: repository: PolicyEngine/calibration-diagnostics - ref: 80a2a4ac7654f586b2b4b8d4c6ec7d2379616722 + ref: 280459c8d782658b3fb6098b27e9acba11af29d8 path: .collector persist-credentials: false - uses: astral-sh/setup-uv@d0d8abe699bfb85fec6de9f7adb5ae17292296ff # v6 diff --git a/README.md b/README.md index 3795e305..9d0544cf 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,9 @@ The service interprets requests such as a stage change or build failure, timesta The adapter bounds socket calls and reports telemetry failures without failing the build. It converts Python arguments into JSON but does not process their telemetry meaning. Network requests run on the service worker, outside the message-handling lock. -The service uses the existing Hugging Face credential from the environment or local login cache. The collector validates PolicyEngine membership before issuing a short-lived write credential. Missing or rejected credentials permanently mark that producer's events as local-only. Transient collector failures retain events for retry. The production destination belongs to the package; development overrides accept loopback addresses only. +`block()` finishes the current stage and records that validation checks refused the build, including the phase, check identifiers, failure count and optional check statuses. `fail(error_code=...)` preserves the caller's failure code alongside the exception type and failure class. Microcosm classifies interruptions, process termination and memory failures; this package forwards those classifications rather than duplicating them. Both outcomes finish telemetry, so later heartbeat or parent-exit handling cannot replace them. + +The service uses the existing Hugging Face credential from the environment or local login cache. The collector validates PolicyEngine membership before issuing a short-lived write credential. Missing or rejected credentials permanently mark that producer's events as local-only. Permanent HTTP client errors when registering a run or uploading events also keep only the affected producer's records local; HTTP 408, 429, server errors and network failures remain retryable. HTTP 401 invalidates the short-lived credential for renewal. The production destination belongs to the package; development overrides accept loopback addresses only. The queue upgrades the existing `20261007_01` Alembic revision to `20261008_02`, adding graph-job records without changing producer sequences, event IDs or upload eligibility. Unversioned databases are rejected without schema modification. There is no raw SQL, automatic schema adoption, or parallel legacy implementation. The hosted telemetry collector and calibration dashboard do not change. @@ -29,7 +31,7 @@ The queue upgrades the existing `20261007_01` Alembic revision to `20261008_02`, Graph requests allow up to 30 seconds for bounded file validation and hashing; telemetry acknowledgements, readiness probes and close requests keep their existing short timeout. Very slow storage can still exceed the graph request limit, so a timeout is not proof that a late service-side commit did or did not occur. Repeating an enqueue with the same publication ID and inventory is idempotent. -The host can also start telemetry and graph modules together with `start_services()`. Each module has an independent worker, so a slow graph upload does not block telemetry requests. Completing telemetry does not stop the graph module, and a module-specific client does not own shared-host shutdown. Both libraries share an existing HF credential and short-lived collector session, but graph jobs remain pending independently of telemetry event retention or local-only upload eligibility. +The host can also start telemetry and graph modules together with `start_services()`. Each module has an independent worker, so a slow graph upload does not block telemetry requests. Construct the telemetry adapter with `close_service_on_finish=False` when a build must publish graphs after recording completion, failure or refusal. Telemetry still finishes, but explicit outer cleanup closes the shared host after graph requests; standalone adapters close automatically by default. A module-specific graph client does not own shared-host shutdown. Both libraries share an existing HF credential and short-lived collector session, but graph jobs remain pending independently of telemetry event retention or local-only upload eligibility. Preserve the selected source files until publication succeeds. `microcosm-emitter-publish-graph` asks the service to retry a publication by ID and can re-enqueue a preserved directory on another machine. It never rewrites the original build receipt; the queue and separate latest-status file record retries. @@ -49,7 +51,7 @@ bash tools/check-reader.sh Tests isolate ambient credentials and use synthetic identities. The artifact check builds one source archive and wheel, installs that wheel into a clean environment away from the source checkout, verifies import boundaries, and exercises telemetry-only, graph-only and combined service startup with packaged migrations. Reader checks cover tests, formatting, types, build and an isolated tarball installation; they do not inspect visual styling. -The collector compatibility job tests the installed wheel against a pinned dashboard collector and disposable PostgreSQL. It covers the HTTP exchange, event ingestion, dashboard run documents, rejected identities, and duplicate delivery after a lost acknowledgement. Local execution requires that pinned collector checkout and a loopback test database; `tools/check-collector.sh` verifies these inputs. Never point this test at a deployed database. +The collector compatibility job tests the installed wheel against a pinned dashboard collector and disposable PostgreSQL. It covers the HTTP exchange, event ingestion, blocked outcomes and supplied failure codes in dashboard run documents, rejected identities, and duplicate delivery after a lost acknowledgement. Local execution requires that pinned collector checkout and a loopback test database; `tools/check-collector.sh` verifies these inputs. Never point this test at a deployed database. The pin verifies source compatibility, not which revision the hosted collector currently runs. ## Publication diff --git a/changelog.d/3-build-outcomes.fixed.md b/changelog.d/3-build-outcomes.fixed.md new file mode 100644 index 00000000..f62d3fe6 --- /dev/null +++ b/changelog.d/3-build-outcomes.fixed.md @@ -0,0 +1 @@ +Preserve validation-refused build outcomes and caller-supplied failure codes in the telemetry service. Allow shared-host cleanup to wait until graph publication finishes. Retain only rejected producers' records locally after permanent collector HTTP client errors, and test these behaviors against the installed package and collector. diff --git a/src/microcosm_emitter/telemetry/client.py b/src/microcosm_emitter/telemetry/client.py index 2448d2e9..4fd70a9b 100644 --- a/src/microcosm_emitter/telemetry/client.py +++ b/src/microcosm_emitter/telemetry/client.py @@ -22,6 +22,7 @@ TelemetryStatus, ) from microcosm_emitter.telemetry.requests import ( + COMMAND_BLOCK, COMMAND_CALIBRATION_PROGRESS, COMMAND_COMPLETE, COMMAND_EMIT, @@ -75,10 +76,12 @@ def __init__( run: TelemetryRun, transport: Transport | None, handle: ServiceHandle | None = None, + close_service_on_finish: bool = True, ) -> None: self.run = run self._transport = transport self._handle = handle + self._close_service_on_finish = close_service_on_finish self._closed = False self._warned = False @@ -92,6 +95,7 @@ def start( development_collector_url: str | None = None, heartbeat_seconds: float = DEFAULT_HEARTBEAT_SECONDS, startup_timeout_seconds: float = 3.0, + close_service_on_finish: bool = True, ) -> LocalTelemetryEmitter: """Start the installed service; return a disabled adapter on failure.""" try: @@ -113,8 +117,15 @@ def start( SERVICE_START_WARNING.format(error_type=type(error).__name__), file=sys.stderr, ) - return cls(run=run, transport=None) - return cls(run=run, transport=handle.client, handle=handle) + return cls( + run=run, transport=None, close_service_on_finish=close_service_on_finish + ) + return cls( + run=run, + transport=handle.client, + handle=handle, + close_service_on_finish=close_service_on_finish, + ) @property def available(self) -> bool: @@ -215,6 +226,7 @@ def fail( *, failed_during: str | None = None, failure_class: str = DEFAULT_FAILURE_CLASS, + error_code: str | None = None, ) -> None: try: try: @@ -228,15 +240,39 @@ def fail( error_type=type(error).__name__, failed_during=failed_during, failure_class=failure_class, + **({"error_code": error_code} if error_code is not None else {}), + ) + finally: + if self._close_service_on_finish: + self.close() + + def block( + self, + *, + phase: str, + blocking_gate_ids: list[str] | tuple[str, ...], + blocking_failure_count: int | None = None, + gate_statuses: Mapping[str, str] | None = None, + ) -> None: + """Ask the service to finish a run refused by its validation checks.""" + try: + self._request( + COMMAND_BLOCK, + phase=phase, + blocking_gate_ids=blocking_gate_ids, + blocking_failure_count=blocking_failure_count, + gate_statuses=gate_statuses, ) finally: - self.close() + if self._close_service_on_finish: + self.close() def complete(self) -> None: try: self._request(COMMAND_COMPLETE) finally: - self.close() + if self._close_service_on_finish: + self.close() def close(self) -> None: """Request a bounded background drain without inventing a build outcome.""" diff --git a/src/microcosm_emitter/telemetry/constants.py b/src/microcosm_emitter/telemetry/constants.py index 5646c9aa..8a3ea561 100644 --- a/src/microcosm_emitter/telemetry/constants.py +++ b/src/microcosm_emitter/telemetry/constants.py @@ -8,3 +8,9 @@ QUEUE_WARNING: Final = ( "Telemetry could not be queued ({error_type}); the build will continue." ) +LOCAL_ONLY_REJECTED_EVENTS: Final = "collector_rejected_events" +REJECTED_EVENTS_MESSAGE: Final = ( + "Microcosm telemetry is local-only for this run: the collector rejected its " + "events (HTTP {status}), so they will not be retried. The dataset build will " + "continue." +) diff --git a/src/microcosm_emitter/telemetry/protocol.py b/src/microcosm_emitter/telemetry/protocol.py index bbe1dce6..41a75324 100644 --- a/src/microcosm_emitter/telemetry/protocol.py +++ b/src/microcosm_emitter/telemetry/protocol.py @@ -8,7 +8,7 @@ type TelemetryEventType = Literal[ "run", "stage", "progress", "calibration", "heartbeat" ] -type TelemetryStatus = Literal["started", "progress", "completed", "failed"] +type TelemetryStatus = Literal["started", "progress", "completed", "failed", "blocked"] TELEMETRY_SCHEMA_VERSION: Final = 1 TELEMETRY_IDENTIFIER_PATTERN: Final = r"^[A-Za-z0-9][A-Za-z0-9._:-]{0,191}$" @@ -23,15 +23,18 @@ STATUS_PROGRESS: Final = "progress" STATUS_COMPLETED: Final = "completed" STATUS_FAILED: Final = "failed" +STATUS_BLOCKED: Final = "blocked" STAGE_CREATED: Final = "created" STAGE_CALIBRATING: Final = "calibrating" STAGE_COMPLETE: Final = "complete" STAGE_FAILED: Final = "failed" +STAGE_BLOCKED: Final = "blocked" CALIBRATION_EVENT_KIND: Final = "calibration_epoch" BUILD_STARTED_MESSAGE: Final = "Microcosm build started." BUILD_COMPLETED_MESSAGE: Final = "Microcosm build completed." +BUILD_BLOCKED_MESSAGE: Final = "Validation checks refused the candidate at {phase}." UNEXPECTED_PROCESS_EXIT_MESSAGE: Final = ( "The build process exited without reporting completion." ) @@ -41,6 +44,7 @@ MAX_TELEMETRY_DETAILS_DEPTH: Final = 6 MAX_TELEMETRY_COLLECTION_ITEMS: Final = 200 MAX_TELEMETRY_DETAILS_BYTES: Final = 8_192 +MAX_TELEMETRY_CHECK_ID_CHARS: Final = 200 SEQUENTIAL_STATUS_MAP: Final = MappingProxyType( { "failed": STATUS_FAILED, diff --git a/src/microcosm_emitter/telemetry/requests.py b/src/microcosm_emitter/telemetry/requests.py index 61abf95c..039c2bd6 100644 --- a/src/microcosm_emitter/telemetry/requests.py +++ b/src/microcosm_emitter/telemetry/requests.py @@ -9,5 +9,6 @@ COMMAND_CALIBRATION_PROGRESS: Final = "calibration_progress" COMMAND_TRANSITION_CALIBRATION_PROGRESS: Final = "transition_calibration_progress" COMMAND_FAIL: Final = "fail" +COMMAND_BLOCK: Final = "block" COMMAND_COMPLETE: Final = "complete" DEFAULT_FAILURE_CLASS: Final = "build_failure" diff --git a/src/microcosm_emitter/telemetry/service/collector.py b/src/microcosm_emitter/telemetry/service/collector.py index 11bf1df1..e9c4ba8c 100644 --- a/src/microcosm_emitter/telemetry/service/collector.py +++ b/src/microcosm_emitter/telemetry/service/collector.py @@ -28,8 +28,20 @@ RUN_EVENTS_PATH_TEMPLATE, RUN_REGISTRATION_PATH, ) +from microcosm_emitter.telemetry.constants import ( + LOCAL_ONLY_REJECTED_EVENTS, + REJECTED_EVENTS_MESSAGE, +) from microcosm_emitter.telemetry.service.spool import EventSpool +_RETRYABLE_CLIENT_ERRORS = frozenset( + {HTTPStatus.REQUEST_TIMEOUT, HTTPStatus.TOO_MANY_REQUESTS} +) + + +def _permanent_client_error(status: int) -> bool: + return 400 <= int(status) < 500 and status not in _RETRYABLE_CLIENT_ERRORS + def _registration_key(registration: Mapping[str, Any]) -> str: return f"{registration['run_id']}:{registration['producer_id']}" @@ -100,6 +112,12 @@ def flush_once(self) -> bool: registration, LOCAL_ONLY_REJECTED_COLLECTOR_AUTHORIZATION, ) + elif _permanent_client_error(status): + self._make_local_only( + registration, + LOCAL_ONLY_REJECTED_EVENTS, + message=REJECTED_EVENTS_MESSAGE.format(status=int(status)), + ) else: self._defer_retry() return made_progress @@ -134,7 +152,10 @@ def _register(self, registration: Mapping[str, Any], token: str) -> bool: return True if status == HTTPStatus.UNAUTHORIZED: self.session.invalidate() - elif status in {HTTPStatus.FORBIDDEN, HTTPStatus.CONFLICT}: + elif status in { + HTTPStatus.FORBIDDEN, + HTTPStatus.CONFLICT, + } or _permanent_client_error(status): self._make_local_only(registration, LOCAL_ONLY_REJECTED_REGISTRATION) else: self._defer_retry() @@ -144,20 +165,22 @@ def _make_local_only( self, registration: Mapping[str, Any], reason: str, + *, + message: str = REJECTED_CREDENTIAL_MESSAGE, ) -> None: run_id = str(registration["run_id"]) producer_id = str(registration["producer_id"]) registration_key = _registration_key(registration) self.spool.make_local_only(run_id, producer_id, reason) if reason != LOCAL_ONLY_MISSING_CREDENTIAL: - self._warn_denied(registration_key) + self._warn_denied(registration_key, message) def _defer_retry(self) -> None: self._next_attempt_at = time.monotonic() + self._retry_seconds self._retry_seconds = min(MAX_RETRY_SECONDS, self._retry_seconds * 2) - def _warn_denied(self, registration_key: str) -> None: + def _warn_denied(self, registration_key: str, message: str) -> None: if registration_key in self._warned_denied: return - print(REJECTED_CREDENTIAL_MESSAGE, file=sys.stderr, flush=True) + print(message, file=sys.stderr, flush=True) self._warned_denied.add(registration_key) diff --git a/src/microcosm_emitter/telemetry/service/lifecycle.py b/src/microcosm_emitter/telemetry/service/lifecycle.py index 61ee2114..70bc9218 100644 --- a/src/microcosm_emitter/telemetry/service/lifecycle.py +++ b/src/microcosm_emitter/telemetry/service/lifecycle.py @@ -6,6 +6,7 @@ from typing import Any from microcosm_emitter.telemetry.protocol import ( + BUILD_BLOCKED_MESSAGE, BUILD_COMPLETED_MESSAGE, BUILD_STARTED_MESSAGE, CALIBRATION_EVENT_KIND, @@ -14,12 +15,15 @@ EVENT_TYPE_PROGRESS, EVENT_TYPE_RUN, EVENT_TYPE_STAGE, + MAX_TELEMETRY_CHECK_ID_CHARS, MAX_TELEMETRY_MESSAGE_CHARS, SEQUENTIAL_STATUS_MAP, + STAGE_BLOCKED, STAGE_CALIBRATING, STAGE_COMPLETE, STAGE_CREATED, STAGE_FAILED, + STATUS_BLOCKED, STATUS_COMPLETED, STATUS_FAILED, STATUS_PROGRESS, @@ -27,6 +31,7 @@ TELEMETRY_IDENTIFIER_PATTERN, ) from microcosm_emitter.telemetry.requests import ( + COMMAND_BLOCK, COMMAND_CALIBRATION_PROGRESS, COMMAND_COMPLETE, COMMAND_EMIT, @@ -53,7 +58,7 @@ } ) _STATUSES = frozenset( - {STATUS_STARTED, STATUS_PROGRESS, STATUS_COMPLETED, STATUS_FAILED} + {STATUS_STARTED, STATUS_PROGRESS, STATUS_COMPLETED, STATUS_FAILED, STATUS_BLOCKED} ) @@ -111,6 +116,28 @@ def _optional_stage_id(value: Any) -> str | None: return value +def _validate_block_details(details: Mapping[str, Any]) -> None: + """Match the collector's required refusal metadata before persistence.""" + phase = details.get("phase") + count = details.get("blocking_failure_count") + check_ids = details.get("blocking_gate_ids") + if ( + not isinstance(phase, str) + or re.fullmatch(TELEMETRY_IDENTIFIER_PATTERN, phase) is None + ): + raise ValueError("blocked phase must be a safe identifier") + if isinstance(count, bool) or not isinstance(count, int) or count < 1: + raise ValueError("blocking_failure_count must be an integer of at least 1") + if not isinstance(check_ids, list) or any( + not isinstance(check_id, str) + or not 1 <= len(check_id) <= MAX_TELEMETRY_CHECK_ID_CHARS + for check_id in check_ids + ): + raise ValueError( + "blocking_gate_ids must contain check identifiers of 1 to 200 characters" + ) + + def _event( event_type: str, status: str, @@ -123,6 +150,13 @@ def _event( if not isinstance(status, str) or status not in _STATUSES: raise ValueError("unsupported telemetry status") stage_id, message = _optional_stage_id(stage_id), _optional_text(message) + original_details = _mapping(details if details is not None else {}) + if status == STATUS_BLOCKED: + _validate_block_details(original_details) + safe_details = sanitize_details(original_details) + if status == STATUS_BLOCKED: + # Redaction/size handling must not make a valid block undeliverable. + _validate_block_details(safe_details) return { "timestamp": utc_now(), "event_type": event_type, @@ -131,7 +165,7 @@ def _event( "message": sanitize_text(message, limit=MAX_TELEMETRY_MESSAGE_CHARS) if message else None, - "details": sanitize_details(_mapping(details if details is not None else {})), + "details": safe_details, } @@ -201,6 +235,8 @@ def process(self, request: Mapping[str, Any]) -> None: ) elif command == COMMAND_FAIL: self.fail(request) + elif command == COMMAND_BLOCK: + self.block(request) elif command == COMMAND_COMPLETE: self.close_stage() self.add( @@ -221,7 +257,11 @@ def add( details: Mapping[str, Any] | None = None, ) -> None: self.events.append(_event(event_type, status, stage_id, message, details)) - if event_type == EVENT_TYPE_RUN and status in {STATUS_COMPLETED, STATUS_FAILED}: + if event_type == EVENT_TYPE_RUN and status in { + STATUS_COMPLETED, + STATUS_FAILED, + STATUS_BLOCKED, + }: self.state = replace(self.state, finished=True, transition_stage=None) elif stage_id is not None: self.state = replace(self.state, last_stage=stage_id) @@ -275,6 +315,32 @@ def fail(self, request: Mapping[str, Any]) -> None: ), "failed_during": failed_stage, } + error_code = _optional_text(request.get("error_code")) + if error_code is not None: + details["error_code"] = error_code message = _optional_text(request.get("message")) self.close_stage(STATUS_FAILED, message, details) self.add(EVENT_TYPE_RUN, STATUS_FAILED, STAGE_FAILED, message, details) + + def block(self, request: Mapping[str, Any]) -> None: + check_ids = request.get("blocking_gate_ids") + if not isinstance(check_ids, list): + raise ValueError("blocking_gate_ids must be a list") + count = request.get("blocking_failure_count") + details = { + "phase": request.get("phase"), + "blocking_gate_ids": check_ids, + "blocking_failure_count": len(check_ids) if count is None else count, + } + statuses = request.get("gate_statuses") + if statuses is not None: + details["gate_statuses"] = _mapping(statuses) + _validate_block_details(details) + self.close_stage() + self.add( + EVENT_TYPE_RUN, + STATUS_BLOCKED, + STAGE_BLOCKED, + BUILD_BLOCKED_MESSAGE.format(phase=details["phase"]), + details, + ) diff --git a/tests/telemetry/test_client.py b/tests/telemetry/test_client.py index 9eb30e90..f0624b53 100644 --- a/tests/telemetry/test_client.py +++ b/tests/telemetry/test_client.py @@ -147,3 +147,110 @@ def test_serialization_preserves_paths_and_handles_nonfinite_numbers(emitter): "path": "/tmp/test", "loss": None, } + + +def test_block_forwards_metadata_without_constructing_outcome(emitter): + emitter.block(phase="preflight", blocking_gate_ids=("uk_local_target_fit",)) + assert emitter._transport.requests == [ + { + "command": "block", + "phase": "preflight", + "blocking_gate_ids": ["uk_local_target_fit"], + "blocking_failure_count": None, + "gate_statuses": None, + } + ] + assert emitter._transport.closed and not emitter.available + + +def test_failure_code_is_forwarded_without_classifying_exception(emitter): + emitter.fail( + KeyboardInterrupt("operator"), + failed_during="compile", + failure_class="interrupted", + error_code="BUILD_INTERRUPTED", + ) + assert emitter._transport.requests[-1]["error_code"] == "BUILD_INTERRUPTED" + assert emitter._transport.requests[-1]["failure_class"] == "interrupted" + assert emitter._transport.requests[-1]["error_type"] == "KeyboardInterrupt" + assert emitter._transport.closed + + +def test_block_transport_failure_still_closes(emitter, monkeypatch): + def failed(request): + raise OSError("synthetic private error") + + monkeypatch.setattr(emitter._transport, "send", failed) + emitter.block(phase="preflight", blocking_gate_ids=["check"]) + assert emitter._transport.closed and not emitter.available + + +@pytest.mark.parametrize("outcome", ["complete", "fail", "block"]) +@pytest.mark.parametrize("deferred", [False, True]) +@pytest.mark.parametrize("send_fails", [False, True]) +def test_outcome_cleanup_can_be_deferred(outcome, deferred, send_fails, monkeypatch): + transport = RecordingTransport() + emitter = LocalTelemetryEmitter( + run=TelemetryRun("run", "UK", "test"), + transport=transport, + close_service_on_finish=not deferred, + ) + if send_fails: + + def failed(request): + raise OSError("synthetic send failure") + + monkeypatch.setattr(transport, "send", failed) + try: + if outcome == "fail": + emitter.fail(RuntimeError("synthetic"), error_code="BUILD_FAILED") + elif outcome == "block": + emitter.block(phase="preflight", blocking_gate_ids=["check"]) + else: + emitter.complete() + assert transport.closed is not deferred + assert emitter.available is deferred + finally: + emitter.close() + assert transport.closed and not emitter.available + + +@pytest.mark.parametrize("outcome", ["complete", "fail", "block"]) +def test_cleanup_failure_is_contained_for_every_outcome(outcome, monkeypatch): + transport = RecordingTransport() + emitter = LocalTelemetryEmitter( + run=TelemetryRun("run", "UK", "test"), + transport=transport, + ) + + def failed(): + raise OSError("synthetic cleanup failure") + + monkeypatch.setattr(transport, "close", failed) + if outcome == "fail": + emitter.fail(RuntimeError("synthetic")) + elif outcome == "block": + emitter.block(phase="preflight", blocking_gate_ids=["check"]) + else: + emitter.complete() + assert not emitter.available + + +def test_start_propagates_deferred_cleanup(monkeypatch, tmp_path): + from unittest.mock import Mock + + transport = RecordingTransport() + handle = Mock(client=transport) + monkeypatch.setattr(client, "start_service", Mock(return_value=handle)) + emitter = LocalTelemetryEmitter.start( + run=TelemetryRun("run", "UK", "test"), + spool_path=tmp_path / "spool", + close_service_on_finish=False, + ) + try: + emitter.complete() + handle.close.assert_not_called() + assert emitter.available + finally: + emitter.close() + handle.close.assert_called_once() diff --git a/tests/telemetry/test_delivery.py b/tests/telemetry/test_delivery.py index 63cbf162..84986aa1 100644 --- a/tests/telemetry/test_delivery.py +++ b/tests/telemetry/test_delivery.py @@ -3,7 +3,9 @@ from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from types import SimpleNamespace +import pytest from alembic import command +from sqlalchemy.orm import Session from microcosm_emitter.core.constants import ( PRODUCTION_COLLECTOR_URL, @@ -17,6 +19,7 @@ migration_head_revision, ) from microcosm_emitter.core.models import ( + TelemetryRunRecord, serialized_json_length, ) from microcosm_emitter.telemetry.client import ( @@ -315,6 +318,120 @@ def test_missing_credential_never_contacts_collector_and_stays_local_only( assert reopened.pending_runs() == [] +@pytest.mark.parametrize("endpoint", ["registration", "events"]) +@pytest.mark.parametrize("status", [400, 404, 413, 422]) +def test_permanent_collector_rejection_retains_only_rejected_producer( + tmp_path, + monkeypatch, + capsys, + endpoint, + status, +): + spool = EventSpool(tmp_path / "rejected.sqlite3") + bad = _registration() + good = {**bad, "producer_id": "producer-good"} + spool.register(bad) + spool.register(good) + rejected = spool.append(bad, _event()) + spool.append(good, _event()) + requests = [] + monkeypatch.setattr( + collector_module, "_huggingface_token", lambda: "hf_test_member" + ) + + def post(url, payload, token): + requests.append((url, payload)) + if url.endswith("/exchange"): + return 200, {"access_token": "synthetic", "expires_in": 3600} + if url.endswith("/v1/runs"): + if ( + endpoint == "registration" + and payload["producer_id"] == bad["producer_id"] + ): + return status, {} + return 201, {} + if ( + endpoint == "events" + and payload["events"][0]["producer_id"] == bad["producer_id"] + ): + return status, {} + return 202, {} + + monkeypatch.setattr(collector_module, "_http_post", post) + delivery = CollectorDelivery(spool, development_collector_url="http://127.0.0.1:1") + try: + assert delivery.flush_once() # The unrelated producer still uploads. + assert spool.pending_runs() == [] + assert spool.batch(bad["run_id"], bad["producer_id"]) == [rejected] + with Session(spool._engine) as session: + record = session.get( + TelemetryRunRecord, (bad["run_id"], bad["producer_id"]) + ) + assert record.upload_state == "local_only" + assert record.local_only_reason == ( + "collector_rejected_events" + if endpoint == "events" + else "run_registration_rejected" + ) + request_count = len(requests) + assert not delivery.flush_once() + assert len(requests) == request_count + warning = capsys.readouterr().err + assert warning.count("local-only for this run") == 1 + if endpoint == "events": + assert f"HTTP {status}" in warning and "will not be retried" in warning + finally: + spool.close() + + +@pytest.mark.parametrize("endpoint", ["registration", "events"]) +@pytest.mark.parametrize("status", [408, 429, 503, None]) +def test_transient_collector_rejection_remains_retryable( + tmp_path, + monkeypatch, + endpoint, + status, +): + spool = EventSpool(tmp_path / "retry.sqlite3") + registration = _registration() + spool.register(registration) + queued = spool.append(registration, _event()) + clock = [100.0] + attempts = [] + monkeypatch.setattr(collector_module.time, "monotonic", lambda: clock[0]) + monkeypatch.setattr( + collector_module, "_huggingface_token", lambda: "hf_test_member" + ) + + def post(url, payload, token): + if url.endswith("/exchange"): + return 200, {"access_token": "synthetic", "expires_in": 3600} + current_endpoint = "registration" if url.endswith("/v1/runs") else "events" + if current_endpoint == endpoint: + attempts.append(current_endpoint) + if len(attempts) == 1: + if status is None: + raise OSError("synthetic transient network failure") + return status, {} + return (201, {}) if current_endpoint == "registration" else (202, {}) + + monkeypatch.setattr(collector_module, "_http_post", post) + delivery = CollectorDelivery(spool, development_collector_url="http://127.0.0.1:1") + try: + assert not delivery.flush_once() + assert spool.pending_runs() == [registration] + assert spool.batch(registration["run_id"], registration["producer_id"]) == [ + queued + ] + assert not delivery.flush_once() # Respect the retry deadline. + assert len(attempts) == 1 + clock[0] += 100.0 + assert delivery.flush_once() + assert len(attempts) == 2 and not spool.has_pending() + finally: + spool.close() + + def test_resource_sampler_keeps_reaped_child_cpu_monotonic(monkeypatch) -> None: state = {"child_alive": True} diff --git a/tests/telemetry/test_module.py b/tests/telemetry/test_module.py index 268882a7..5c16d937 100644 --- a/tests/telemetry/test_module.py +++ b/tests/telemetry/test_module.py @@ -330,3 +330,148 @@ def test_invalid_stage_identifier_preserves_queue_and_lifecycle( module.handle_message({"command": "complete"}) assert [event["sequence"] for event in events(module)] == [1, 2, 3, 4] assert events(module)[-1]["status"] == "completed" + + +@pytest.mark.parametrize("active", [False, True]) +def test_block_closes_active_stage_and_finishes_without_failure(module, active): + if active: + module.handle_message( + {"command": "transition_stage", "stage_id": "hf_download"} + ) + module.handle_message( + { + "command": "block", + "phase": "preflight", + "blocking_gate_ids": ["uk_local_target_fit"], + "gate_statuses": {"uk_local_target_fit": "failed"}, + } + ) + recorded = events(module) + assert ( + recorded[-1]["event_type"], + recorded[-1]["stage_id"], + recorded[-1]["status"], + ) == ("run", "blocked", "blocked") + assert recorded[-1]["details"] == { + "phase": "preflight", + "blocking_gate_ids": ["uk_local_target_fit"], + "blocking_failure_count": 1, + "gate_statuses": {"uk_local_target_fit": "failed"}, + } + if active: + assert (recorded[-2]["stage_id"], recorded[-2]["status"]) == ( + "hf_download", + "completed", + ) + assert module.lifecycle.finished and module.lifecycle.transition_stage is None + module.parent_exited() + module.tick(time.monotonic() + 120) + module.handle_message({"command": "complete"}) + assert events(module) == recorded + + +def test_block_accepts_empty_ids_with_explicit_positive_count(module): + module.handle_message( + { + "command": "block", + "phase": "post_solve", + "blocking_gate_ids": [], + "blocking_failure_count": 3, + } + ) + assert events(module)[-1]["details"] == { + "phase": "post_solve", + "blocking_gate_ids": [], + "blocking_failure_count": 3, + } + + +@pytest.mark.parametrize( + "failure_class,error_code", + [ + ("error", "BUILD_FAILED"), + ("interrupted", "BUILD_INTERRUPTED"), + ("terminated", "BUILD_TERMINATED"), + ("oom", "BUILD_OUT_OF_MEMORY"), + ], +) +def test_failure_code_persists_on_stage_and_run(module, failure_class, error_code): + module.handle_message({"command": "transition_stage", "stage_id": "compile"}) + module.handle_message( + { + "command": "fail", + "error_type": "MemoryError", + "message": "Bearer hf_private_credentials", + "failure_class": failure_class, + "error_code": error_code, + } + ) + assert [event["details"]["error_code"] for event in events(module)[-2:]] == [ + error_code, + error_code, + ] + assert events(module)[-1]["details"]["failure_class"] == failure_class + assert events(module)[-1]["message"] == "[redacted]" + + +def test_block_commit_failure_preserves_stage_and_sequence(module): + module.handle_message({"command": "transition_stage", "stage_id": "compile"}) + recorded = events(module) + + def fail_commit(session): + session.flush() + raise OSError("simulated block commit failure") + + factory = module.spool._session_factory + sqlalchemy_event.listen(factory, "before_commit", fail_commit) + try: + with pytest.raises(OSError, match="block commit"): + module.handle_message( + { + "command": "block", + "phase": "preflight", + "blocking_gate_ids": ["check"], + } + ) + finally: + sqlalchemy_event.remove(factory, "before_commit", fail_commit) + assert events(module) == recorded and not module.lifecycle.finished + assert module.lifecycle.transition_stage == "compile" + module.handle_message( + {"command": "block", "phase": "preflight", "blocking_gate_ids": ["check"]} + ) + assert [event["sequence"] for event in events(module)] == [1, 2, 3, 4] + + +@pytest.mark.parametrize( + "details", + [ + {"phase": "bad phase"}, + {"phase": "hf_private_credentials"}, + {"blocking_failure_count": 0}, + {"blocking_failure_count": True}, + {"blocking_failure_count": "1"}, + {"blocking_gate_ids": [""]}, + {"blocking_gate_ids": ["x" * 201]}, + {"blocking_gate_ids": [1]}, + {"blocking_gate_ids": [], "blocking_failure_count": None}, + {"gate_statuses": {f"check-{number}": "x" * 2000 for number in range(10)}}, + ], +) +def test_invalid_block_cannot_poison_queue_or_lifecycle(module, details): + module.handle_message({"command": "transition_stage", "stage_id": "compile"}) + recorded, state = events(module), module.lifecycle + request = { + "command": "block", + "phase": "preflight", + "blocking_gate_ids": ["check"], + **details, + } + with pytest.raises((ValueError, TypeError)): + module.handle_message(request) + assert events(module) == recorded and module.lifecycle == state + + +def test_uncoded_failure_omits_optional_error_code(module): + module.handle_message({"command": "fail", "message": "failed"}) + assert "error_code" not in events(module)[-1]["details"] diff --git a/tests/test_collector_contract.py b/tests/test_collector_contract.py index b072828b..636f4abb 100644 --- a/tests/test_collector_contract.py +++ b/tests/test_collector_contract.py @@ -190,3 +190,128 @@ def lose_ack(url, *args, **kwargs): assert [entry["event_id"] for entry in response.json()["events"]] == [ event["event_id"] ] + + +@pytest.mark.parametrize("unnamed", [False, True]) +def test_blocked_run_projects_check_metadata_without_failure( + tmp_path, + monkeypatch, + live_collector, + unnamed, +): + address, reader = live_collector + monkeypatch.setenv("HF_TOKEN", "hf_test_member") + run_id = uuid.uuid4().hex + emitter = LocalTelemetryEmitter.start( + run=TelemetryRun(run_id, "UK", "contract"), + spool_path=tmp_path / "blocked.sqlite3", + development_collector_url=address, + ) + assert emitter.available + check_ids = [] if unnamed else ["uk_local_target_fit"] + count = 3 if unnamed else 1 + try: + emitter.transition_stage("hf_download") + emitter.block( + phase="preflight", + blocking_gate_ids=check_ids, + blocking_failure_count=count, + gate_statuses={ + "uk_local_target_fit": "failed", + "HF_TOKEN": "hf_private_check", + }, + ) + assert emitter._handle.process.wait(timeout=10) == 0 + finally: + emitter.close() + response = reader.get(f"/v1/runs/{run_id}") + assert response.status_code == 200, response.text + document = response.json() + assert document["progress"]["status"] == "blocked" + assert document["progress"]["current_stage"] == "blocked" + assert document["progress"]["failure"] is None + expected = { + "phase": "preflight", + "blocking_gate_ids": check_ids, + "blocking_failure_count": count, + } + assert document["progress"]["block"] == expected + assert document["run_manifest"]["block"] == expected + assert document["run_manifest"]["failure"] is None + assert [(event["stage_id"], event["status"]) for event in document["events"]] == [ + ("created", "started"), + ("hf_download", "started"), + ("hf_download", "completed"), + ("blocked", "blocked"), + ] + assert document["events"][-1]["details"] == { + **expected, + "gate_statuses": {"uk_local_target_fit": "failed", "HF_TOKEN": "[redacted]"}, + } + assert "hf_private_check" not in response.text + + +@pytest.mark.parametrize( + "failure_class,error_code,error", + [ + ("error", "BUILD_FAILED", RuntimeError("Bearer hf_private_error")), + ( + "interrupted", + "BUILD_INTERRUPTED", + KeyboardInterrupt("Bearer hf_private_error"), + ), + ("terminated", "BUILD_TERMINATED", RuntimeError("Bearer hf_private_error")), + ("oom", "BUILD_OUT_OF_MEMORY", MemoryError("Bearer hf_private_error")), + ], +) +def test_supplied_failure_codes_survive_collector_storage_and_documents( + tmp_path, + monkeypatch, + live_collector, + failure_class, + error_code, + error, +): + address, reader = live_collector + monkeypatch.setenv("HF_TOKEN", "hf_test_member") + run_id = uuid.uuid4().hex + emitter = LocalTelemetryEmitter.start( + run=TelemetryRun(run_id, "UK", "contract"), + spool_path=tmp_path / "failed.sqlite3", + development_collector_url=address, + ) + assert emitter.available + try: + emitter.transition_stage("hf_download") + emitter.fail( + error, + failed_during="hf_download", + failure_class=failure_class, + error_code=error_code, + ) + assert emitter._handle.process.wait(timeout=10) == 0 + finally: + emitter.close() + response = reader.get(f"/v1/runs/{run_id}") + assert response.status_code == 200, response.text + document = response.json() + # Generic details retain credential redaction; the wire stage identity does not. + expected = { + "error_code": error_code, + "error_type": type(error).__name__, + "failure_class": failure_class, + "failed_during": "[redacted]", + } + assert document["progress"]["status"] == "failed" + assert document["progress"]["current_stage"] == "failed" + assert document["progress"]["block"] is None + assert document["run_manifest"]["failure"] == document["progress"]["failure"] + assert document["progress"]["failure"] == {**expected, "message": "[redacted]"} + assert [ + (event["stage_id"], event["status"]) for event in document["events"][-2:] + ] == [ + ("hf_download", "failed"), + ("failed", "failed"), + ] + assert all(event["details"] == expected for event in document["events"][-2:]) + assert "hf_private_error" not in response.text diff --git a/tests/test_subprocess.py b/tests/test_subprocess.py index cebe8601..028ce38a 100644 --- a/tests/test_subprocess.py +++ b/tests/test_subprocess.py @@ -3,11 +3,14 @@ import json import threading from concurrent.futures import ThreadPoolExecutor +from contextlib import nullcontext from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer import psutil import pytest +from microcosm_emitter.host.launcher import start_services +from microcosm_emitter.orrery.client import GraphPublicationClient from microcosm_emitter.telemetry.client import LocalTelemetryEmitter, TelemetryRun from microcosm_emitter.telemetry.service.spool import EventSpool @@ -122,3 +125,83 @@ def build(number): ] == [1, 2, 3] finally: spool.close() + + +@pytest.mark.parametrize("outcome", ["complete", "fail", "block"]) +@pytest.mark.parametrize("export_fails", [False, True]) +def test_finished_telemetry_keeps_shared_graph_process_until_outer_cleanup( + tmp_path, + outcome, + export_fails, +): + path = tmp_path / "combined.sqlite3" + run = TelemetryRun("combined-outcome", "UK", "test") + handle = start_services( + { + "telemetry": { + "registration": run.as_registration(), + "spool_path": str(path), + "heartbeat_seconds": 0.001, + "development_collector_url": "http://127.0.0.1:1", + }, + "orrery": { + "spool_path": str(path), + "development_collector_url": "http://127.0.0.1:1", + "publication_origin": "http://127.0.0.1:1", + }, + } + ) + emitter = LocalTelemetryEmitter( + run=run, + transport=handle.client.for_module("telemetry"), + handle=handle, + close_service_on_finish=False, + ) + graph = GraphPublicationClient(handle.client.for_module("orrery")) + directory = tmp_path / "graph" + directory.mkdir() + (directory / "graph.orrery.json").write_text('{"nodes":[]}') + expectation = ( + pytest.raises(RuntimeError, match="synthetic export") + if export_fails + else nullcontext() + ) + with expectation: + try: + emitter.transition_stage("hf_download") + if outcome == "block": + emitter.block(phase="preflight", blocking_gate_ids=["check"]) + elif outcome == "fail": + emitter.fail(RuntimeError("synthetic"), error_code="BUILD_FAILED") + else: + emitter.complete() + assert emitter.available and handle.process.poll() is None + handle.client.ping() + inventory = graph.inventory(directory, {"graph.orrery.json": "graph"}) + assert inventory is not None + assert graph.enqueue(directory, inventory)["status"] == "pending" + assert graph.receipt(inventory["publication_id"])["status"] == "pending" + emitter.stage("after-finish") + if export_fails: + raise RuntimeError("synthetic export") + finally: + graph.close() + emitter.close() + emitter.close() + assert handle.process.wait(timeout=20) == 0 + spool = EventSpool(path) + try: + events = spool.batch(run.run_id, run.producer_id) + status = {"complete": "completed", "fail": "failed", "block": "blocked"}[ + outcome + ] + finished_index = next( + index + for index, event in enumerate(events) + if event["event_type"] == "run" and event["status"] == status + ) + assert finished_index == len(events) - 1 + assert events[-1]["status"] == status + assert not any(event["stage_id"] == "after-finish" for event in events) + finally: + spool.close() diff --git a/tools/check-collector.sh b/tools/check-collector.sh index a00b859e..8a47f773 100644 --- a/tools/check-collector.sh +++ b/tools/check-collector.sh @@ -2,7 +2,7 @@ set -euo pipefail : "${TEST_DATABASE_URL:?Set the disposable PostgreSQL test database URL}" -expected_commit=80a2a4ac7654f586b2b4b8d4c6ec7d2379616722 +expected_commit=280459c8d782658b3fb6098b27e9acba11af29d8 test "$(git -C .collector rev-parse HEAD)" = "$expected_commit" uv sync --locked uv pip install -e .collector/telemetry-service diff --git a/tools/installed-smoke.py b/tools/installed-smoke.py index 292162c5..a8863172 100644 --- a/tools/installed-smoke.py +++ b/tools/installed-smoke.py @@ -126,10 +126,13 @@ def combined_smoke(): (directory / "graph.orrery.json").write_text('{"synthetic":true}') try: emitter = LocalTelemetryEmitter( - run=run, transport=handle.client.for_module("telemetry") + run=run, + transport=handle.client.for_module("telemetry"), + handle=handle, + close_service_on_finish=False, ) emitter.stage("hf_download") - emitter.complete() + emitter.block(phase="preflight", blocking_gate_ids=["check"]) handle.client.ping() inventory = client.inventory(directory, {"graph.orrery.json": "graph"}) assert inventory is not None @@ -139,8 +142,55 @@ def combined_smoke(): ) finally: client.close() - handle.close() + emitter.close() assert handle.process.wait(timeout=20) == 0 + from microcosm_emitter.telemetry.service.spool import EventSpool + + spool = EventSpool(path) + try: + assert spool.batch(run.run_id, run.producer_id)[-1]["status"] == "blocked" + finally: + spool.close() + + +def coded_failure_smoke(): + from microcosm_emitter.telemetry.client import LocalTelemetryEmitter, TelemetryRun + + run = TelemetryRun("failure-test", "UK", "installed") + path = Path.cwd() / "failure.sqlite3" + emitter = LocalTelemetryEmitter.start( + run=run, + spool_path=path, + development_collector_url="http://127.0.0.1:1", + startup_timeout_seconds=10, + ) + assert emitter.available + try: + emitter.transition_stage("hf_download") + emitter.fail( + MemoryError("synthetic"), + failed_during="hf_download", + failure_class="oom", + error_code="BUILD_OUT_OF_MEMORY", + ) + assert emitter._handle.process.wait(timeout=20) == 0 + finally: + emitter.close() + from microcosm_emitter.telemetry.service.spool import EventSpool + + spool = EventSpool(path) + try: + events = spool.batch(run.run_id, run.producer_id) + assert [(event["stage_id"], event["status"]) for event in events[-2:]] == [ + ("hf_download", "failed"), + ("failed", "failed"), + ] + assert all( + event["details"]["error_code"] == "BUILD_OUT_OF_MEMORY" + for event in events[-2:] + ) + finally: + spool.close() if __name__ == "__main__": @@ -148,3 +198,4 @@ def combined_smoke(): telemetry_smoke() graph_only_smoke() combined_smoke() + coded_failure_smoke()