diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9e49180b..dba7400e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -73,7 +73,7 @@ jobs: - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 with: repository: PolicyEngine/calibration-diagnostics - ref: 80a2a4ac7654f586b2b4b8d4c6ec7d2379616722 + ref: 280459c8d782658b3fb6098b27e9acba11af29d8 path: .collector persist-credentials: false - uses: astral-sh/setup-uv@d0d8abe699bfb85fec6de9f7adb5ae17292296ff # v6 @@ -85,13 +85,27 @@ jobs: path: dist - run: bash tools/check-collector.sh + reader: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2 + with: + bun-version: "1.4.2" + - run: bash tools/check-reader.sh + - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: orrery-reader + path: reader-dist/*.tgz + if-no-files-found: error + required: name: Required checks if: always() - needs: [quality, tests, build-packages, collector] + needs: [quality, tests, build-packages, collector, reader] runs-on: ubuntu-latest steps: - name: Require all validation jobs to succeed env: RESULTS: ${{ join(needs.*.result, ' ') }} - run: test "$RESULTS" = "success success success success" + run: test "$RESULTS" = "success success success success success" diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 350c0804..1a1d5428 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -40,3 +40,22 @@ jobs: - uses: pypa/gh-action-pypi-publish@dc37677b2e1c63e2034f94d8a5b11f265b73ba33 # release/v1 with: packages-dir: dist/ + + publish-reader: + needs: [verify, publish] + runs-on: ubuntu-latest + environment: pypi + permissions: + contents: read + id-token: write + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 + with: + node-version: "24" + registry-url: "https://registry.npmjs.org" + - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 + with: + name: orrery-reader + path: reader-dist + - run: bash tools/publish-reader.sh diff --git a/.gitignore b/.gitignore index f54261ab..e598fa62 100644 --- a/.gitignore +++ b/.gitignore @@ -7,4 +7,6 @@ dist/ .coverage .collector/ .test-artifacts/ - +node_modules/ +reader-dist/ +typescript/orrery/dist/ diff --git a/AGENTS.md b/AGENTS.md index 93d0fd91..b9c49e47 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -2,6 +2,7 @@ - Use uv, Python 3.13+, pytest, and Ruff. Keep workflows as single-command calls to Bash scripts when shell logic has multiple lines. - The generic runtime must not import telemetry or Microcosm. The telemetry client must not import service implementation, database, or authentication modules. +- Graph adapters must not import service, database, authentication or HTTP code. Inventory hashing, durable enqueue, receipt lookup and retries run in the separate local emitter service; no direct-database fallback. - Publish only the `microcosm-emitter` distribution. Keep the generic host and required telemetry library internally separate under `microcosm_emitter`; do not introduce standalone telemetry packages or optional telemetry installation. - Python databases use SQLAlchemy declarative ORM and Alembic exclusively. No handwritten SQL, direct-driver persistence, create_all(), schema repair, or pre-Alembic adoption. TypeScript database work, if introduced, must use Drizzle ORM and Drizzle Kit migrations. - Tests use temporary databases created through real migrations, synthetic credentials, and loopback network services. Never use ambient credentials or contact the production collector in tests. diff --git a/README.md b/README.md index 9ff89483..9d0544cf 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,13 @@ # Microcosm emitter -`microcosm-emitter` is one Python distribution containing the process host and a required telemetry library: +`microcosm-emitter` is one Python distribution containing the process host, required telemetry library and completed-graph publication library: - `microcosm_emitter.host` owns subprocess startup, a private Unix socket, explicit module loading, parent-process monitoring, and shutdown. Its code does not import telemetry, database, authentication, or Microcosm modules. - `microcosm_emitter.telemetry.client` is the thin build adapter. It starts the local telemetry emitter service and sends build requests, without importing the service implementation, database, or authentication libraries. - `microcosm_emitter.telemetry.service` interprets those requests in the separate process. It owns stage tracking, event construction, credential redaction, resource sampling, SQLAlchemy ORM persistence, Alembic migrations, authentication, and collector delivery. +- `microcosm_emitter.orrery.client` sends graph inventory, enqueue, receipt and retry requests over the same private socket. It does not hash files, open a database, authenticate or upload. +- `microcosm_emitter.orrery.service` validates and hashes the selected graph files, commits publication jobs, and uploads exact bytes with retry and expiring worker reservations. A graph publication requires no telemetry run registration. +- `microcosm_emitter.core` provides service-side authentication and SQLAlchemy/Alembic persistence shared by the telemetry and graph libraries. The generic host does not import it. Installing `microcosm-emitter` always installs the telemetry library and all its dependencies. Telemetry is not a standalone distribution or an optional extra. The package keeps import boundaries between the host, adapter, and service implementation; they do not need separate installation or publication. @@ -16,9 +19,23 @@ The service interprets requests such as a stage change or build failure, timesta The adapter bounds socket calls and reports telemetry failures without failing the build. It converts Python arguments into JSON but does not process their telemetry meaning. Network requests run on the service worker, outside the message-handling lock. -The service uses the existing Hugging Face credential from the environment or local login cache. The collector validates PolicyEngine membership before issuing a short-lived write credential. Missing or rejected credentials permanently mark that producer's events as local-only. Transient collector failures retain events for retry. The production destination belongs to the package; development overrides accept loopback addresses only. +`block()` finishes the current stage and records that validation checks refused the build, including the phase, check identifiers, failure count and optional check statuses. `fail(error_code=...)` preserves the caller's failure code alongside the exception type and failure class. Microcosm classifies interruptions, process termination and memory failures; this package forwards those classifications rather than duplicating them. Both outcomes finish telemetry, so later heartbeat or parent-exit handling cannot replace them. -The queue retains the current Microcosm Alembic revision, producer sequences, event IDs, and upload eligibility. Unversioned databases are rejected without schema modification. There is no raw SQL, automatic schema adoption, or parallel legacy implementation. The hosted collector and dashboard do not change. +The service uses the existing Hugging Face credential from the environment or local login cache. The collector validates PolicyEngine membership before issuing a short-lived write credential. Missing or rejected credentials permanently mark that producer's events as local-only. Permanent HTTP client errors when registering a run or uploading events also keep only the affected producer's records local; HTTP 408, 429, server errors and network failures remain retryable. HTTP 401 invalidates the short-lived credential for renewal. The production destination belongs to the package; development overrides accept loopback addresses only. + +The queue upgrades the existing `20261007_01` Alembic revision to `20261008_02`, adding graph-job records without changing producer sequences, event IDs or upload eligibility. Unversioned databases are rejected without schema modification. There is no raw SQL, automatic schema adoption, or parallel legacy implementation. The hosted telemetry collector and calibration dashboard do not change. + +## Graph publication + +`GraphPublicationClient.start()` starts the graph module in a separate local emitter service process. `inventory()` asks that process to hash an explicit set of allowed graph files; `enqueue()` returns a pending receipt only after the job commits. `receipt()` reads its result and `retry()` makes a pending job eligible again. Startup or socket failures return an unavailable result, not a claim that a job was saved; the build adapter never writes a fallback database. + +Graph requests allow up to 30 seconds for bounded file validation and hashing; telemetry acknowledgements, readiness probes and close requests keep their existing short timeout. Very slow storage can still exceed the graph request limit, so a timeout is not proof that a late service-side commit did or did not occur. Repeating an enqueue with the same publication ID and inventory is idempotent. + +The host can also start telemetry and graph modules together with `start_services()`. Each module has an independent worker, so a slow graph upload does not block telemetry requests. Construct the telemetry adapter with `close_service_on_finish=False` when a build must publish graphs after recording completion, failure or refusal. Telemetry still finishes, but explicit outer cleanup closes the shared host after graph requests; standalone adapters close automatically by default. A module-specific graph client does not own shared-host shutdown. Both libraries share an existing HF credential and short-lived collector session, but graph jobs remain pending independently of telemetry event retention or local-only upload eligibility. + +Preserve the selected source files until publication succeeds. `microcosm-emitter-publish-graph` asks the service to retry a publication by ID and can re-enqueue a preserved directory on another machine. It never rewrites the original build receipt; the queue and separate latest-status file record retries. + +The existing TypeScript package `@policyengine/microcosm-provider-orrery` reads immutable published graph files on a server or in a browser. It remains separate from the Python distribution and does not change the hosted telemetry API. ## Development @@ -29,14 +46,17 @@ uv sync --locked bash tools/check-quality.sh uv run --no-sync pytest -m 'not collector' bash tools/check-artifacts.sh +bash tools/check-reader.sh ``` -Tests isolate ambient credentials and use synthetic identities. The artifact check builds one source archive and wheel, installs that wheel into a clean environment away from the source checkout, verifies that the host does not import telemetry or its dependencies, and exercises the real telemetry service process and packaged migrations. +Tests isolate ambient credentials and use synthetic identities. The artifact check builds one source archive and wheel, installs that wheel into a clean environment away from the source checkout, verifies import boundaries, and exercises telemetry-only, graph-only and combined service startup with packaged migrations. Reader checks cover tests, formatting, types, build and an isolated tarball installation; they do not inspect visual styling. -The collector compatibility job tests the installed wheel against a pinned dashboard collector and disposable PostgreSQL. It covers the HTTP exchange, event ingestion, dashboard run documents, rejected identities, and duplicate delivery after a lost acknowledgement. Local execution requires that pinned collector checkout and a loopback test database; `tools/check-collector.sh` verifies these inputs. Never point this test at a deployed database. +The collector compatibility job tests the installed wheel against a pinned dashboard collector and disposable PostgreSQL. It covers the HTTP exchange, event ingestion, blocked outcomes and supplied failure codes in dashboard run documents, rejected identities, and duplicate delivery after a lost acknowledgement. Local execution requires that pinned collector checkout and a loopback test database; `tools/check-collector.sh` verifies these inputs. Never point this test at a deployed database. The pin verifies source compatibility, not which revision the hosted collector currently runs. ## Publication -The project publishes only `microcosm-emitter`. A published GitHub release triggers version and main-branch ancestry checks, all CI checks, and artifact verification. Only then can the protected `pypi` environment approve publishing the exact tested wheel and source archive. The publish job uses PyPI Trusted Publishing; no long-lived publishing token is supplied through CI. +The project publishes one Python distribution, `microcosm-emitter`. A published GitHub release triggers version and main-branch ancestry checks, all CI checks, and artifact verification. Only then can the protected `pypi` environment approve publishing the exact tested wheel and source archive. The publish job uses PyPI Trusted Publishing; no long-lived publishing token is supplied through CI. The separate TypeScript reader publication waits for Python publication and publishes the verified tarball through npm OIDC. First publication requires one PyPI pending trusted publisher for `microcosm-emitter`, attached to GitHub owner `PolicyEngine`, project `microcosm-emitter`, workflow `publish.yml`, and environment `pypi`. That external setup must be verified before releasing. No release is created or merged automatically by this project. + +The npm reader requires registry-owner first-publication and trusted-publisher setup; this code does not provision or verify those private account settings. No hosted-service deployment, registry publication or consumer activation follows from installing this package. diff --git a/changelog.d/3-build-outcomes.fixed.md b/changelog.d/3-build-outcomes.fixed.md new file mode 100644 index 00000000..f62d3fe6 --- /dev/null +++ b/changelog.d/3-build-outcomes.fixed.md @@ -0,0 +1 @@ +Preserve validation-refused build outcomes and caller-supplied failure codes in the telemetry service. Allow shared-host cleanup to wait until graph publication finishes. Retain only rejected producers' records locally after permanent collector HTTP client errors, and test these behaviors against the installed package and collector. diff --git a/changelog.d/3-publication-failures.fixed.md b/changelog.d/3-publication-failures.fixed.md new file mode 100644 index 00000000..ebbb7780 --- /dev/null +++ b/changelog.d/3-publication-failures.fixed.md @@ -0,0 +1 @@ +Keep graph publication running when a preserved directory is removed or the local queue reports an error, and record the latest status only after the queue commit. diff --git a/changelog.d/3-publication-types.changed.md b/changelog.d/3-publication-types.changed.md new file mode 100644 index 00000000..eb6d1f10 --- /dev/null +++ b/changelog.d/3-publication-types.changed.md @@ -0,0 +1 @@ +Define explicit types for publication inventories and results, and return a frozen record when reserving a graph upload. diff --git a/changelog.d/3-single-emitter.changed.md b/changelog.d/3-single-emitter.changed.md new file mode 100644 index 00000000..8f7e7f12 --- /dev/null +++ b/changelog.d/3-single-emitter.changed.md @@ -0,0 +1 @@ +Integrate completed-graph publication into the single microcosm-emitter package. Keep graph adapters limited to socket requests and preserve shared SQLAlchemy/Alembic persistence, telemetry lifecycle behavior and separate TypeScript readers. diff --git a/changelog.d/3.added.md b/changelog.d/3.added.md new file mode 100644 index 00000000..6df38cd5 --- /dev/null +++ b/changelog.d/3.added.md @@ -0,0 +1 @@ +Add independent completed-graph publication and verified Orrery document readers alongside telemetry in the local provider. diff --git a/conftest.py b/conftest.py index 84f3f43e..0ad48ecc 100644 --- a/conftest.py +++ b/conftest.py @@ -15,6 +15,7 @@ def isolate_credentials_and_network(tmp_path, monkeypatch): from huggingface_hub import constants monkeypatch.setattr(constants, "HF_TOKEN_PATH", str(hf_home / "token")) + from microcosm_emitter.core import auth from microcosm_emitter.telemetry.service import collector real_post = collector._http_post @@ -25,3 +26,4 @@ def loopback_post(url, *args, **kwargs): return real_post(url, *args, **kwargs) monkeypatch.setattr(collector, "_http_post", loopback_post) + monkeypatch.setattr(auth, "_http_post", loopback_post) diff --git a/package.json b/package.json new file mode 100644 index 00000000..711114cc --- /dev/null +++ b/package.json @@ -0,0 +1,15 @@ +{ + "name": "@policyengine/microcosm-provider-orrery", + "version": "0.1.0", + "private": true, + "type": "module", + "description": "Pinned-source development entry points; publish the built package from typescript/orrery", + "exports": { + "./contract": "./typescript/orrery/src/contract.ts", + "./browser": "./typescript/orrery/src/browser.ts", + "./server": "./typescript/orrery/src/server.ts" + }, + "peerDependencies": { + "@axiom-foundation/orrery": "0.6.1" + } +} diff --git a/pyproject.toml b/pyproject.toml index 1b86590b..ae9c8ff5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,6 +10,7 @@ dependencies = [ "sqlalchemy>=2,<3", "huggingface-hub>=0.20,<2", "psutil>=6,<8", + "httpx>=0.27,<1", ] [project.urls] @@ -17,6 +18,10 @@ Source = "https://github.com/PolicyEngine/microcosm-emitter" [project.entry-points."microcosm.emitter.modules"] telemetry = "microcosm_emitter.telemetry.service:create_module" +orrery = "microcosm_emitter.orrery.service:create_module" + +[project.scripts] +microcosm-emitter-publish-graph = "microcosm_emitter.orrery.cli:main" [build-system] requires = ["hatchling"] diff --git a/src/microcosm_emitter/core/__init__.py b/src/microcosm_emitter/core/__init__.py new file mode 100644 index 00000000..b92eeaef --- /dev/null +++ b/src/microcosm_emitter/core/__init__.py @@ -0,0 +1 @@ +"""Shared provider services; importing this package has no runtime side effects.""" diff --git a/src/microcosm_emitter/telemetry/service/alembic/env.py b/src/microcosm_emitter/core/alembic/env.py similarity index 91% rename from src/microcosm_emitter/telemetry/service/alembic/env.py rename to src/microcosm_emitter/core/alembic/env.py index 510b0bc1..35c1cbba 100644 --- a/src/microcosm_emitter/telemetry/service/alembic/env.py +++ b/src/microcosm_emitter/core/alembic/env.py @@ -3,7 +3,7 @@ from alembic import context from sqlalchemy.engine import Connection -from microcosm_emitter.telemetry.service.models import SpoolModel +from microcosm_emitter.core.models import SpoolModel _MISSING_CONNECTION_ERROR = ( "telemetry spool migrations require a programmatically supplied connection" diff --git a/src/microcosm_emitter/telemetry/service/alembic/script.py.mako b/src/microcosm_emitter/core/alembic/script.py.mako similarity index 100% rename from src/microcosm_emitter/telemetry/service/alembic/script.py.mako rename to src/microcosm_emitter/core/alembic/script.py.mako diff --git a/src/microcosm_emitter/telemetry/service/alembic/versions/v20261007_01_initial_spool.py b/src/microcosm_emitter/core/alembic/versions/v20261007_01_initial_spool.py similarity index 100% rename from src/microcosm_emitter/telemetry/service/alembic/versions/v20261007_01_initial_spool.py rename to src/microcosm_emitter/core/alembic/versions/v20261007_01_initial_spool.py diff --git a/src/microcosm_emitter/core/alembic/versions/v20261008_02_graph_publication.py b/src/microcosm_emitter/core/alembic/versions/v20261008_02_graph_publication.py new file mode 100644 index 00000000..14ae0318 --- /dev/null +++ b/src/microcosm_emitter/core/alembic/versions/v20261008_02_graph_publication.py @@ -0,0 +1,27 @@ +"""Add durable graph publication jobs without changing event retention.""" + +import sqlalchemy as sa +from alembic import op + +revision = "20261008_02" +down_revision = "20261007_01" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.create_table( + "graph_publication_jobs", + sa.Column("publication_id", sa.Text(), primary_key=True), + sa.Column("inventory", sa.Text(), nullable=False), + sa.Column("directory", sa.Text(), nullable=False), + sa.Column("status", sa.Text(), nullable=False), + sa.Column("attempts", sa.Integer(), nullable=False), + sa.Column("next_attempt_at", sa.Float(), nullable=False), + sa.Column("lease_until", sa.Float(), nullable=False), + sa.Column("receipt", sa.Text(), nullable=False), + ) + + +def downgrade() -> None: + op.drop_table("graph_publication_jobs") diff --git a/src/microcosm_emitter/core/auth.py b/src/microcosm_emitter/core/auth.py new file mode 100644 index 00000000..527bcf4c --- /dev/null +++ b/src/microcosm_emitter/core/auth.py @@ -0,0 +1,169 @@ +"""HF-to-collector session shared by telemetry and graph publication.""" + +from __future__ import annotations + +import json +import os +import time +import urllib.error +import urllib.request +from collections.abc import Mapping +from http import HTTPStatus +from typing import Any +from urllib.parse import urlsplit + +from huggingface_hub import get_token + +from microcosm_emitter.core.constants import ( + COLLECTOR_RESPONSE_TOO_LARGE_ERROR, + COLLECTOR_URL_HTTPS_ERROR, + COLLECTOR_URL_ORIGIN_ERROR, + DEFAULT_TOKEN_LIFETIME_SECONDS, + DEVELOPMENT_COLLECTOR_LOOPBACK_ERROR, + HTTP_TIMEOUT_SECONDS, + HTTP_USER_AGENT, + LOOPBACK_HOSTS, + MAX_HTTP_RESPONSE_BYTES, + MINIMUM_TOKEN_LIFETIME_SECONDS, + TOKEN_EXCHANGE_PATH, + TOKEN_REFRESH_MARGIN_SECONDS, +) + + +class _NoRedirectHandler(urllib.request.HTTPRedirectHandler): + """Keep bearer credentials on the explicitly configured origin.""" + + def redirect_request(self, req, fp, code, msg, headers, newurl): + return None + + +def _collector_origin(value: str, *, allow_loopback_http: bool = False) -> str: + parsed = urlsplit(value) + loopback = parsed.hostname in LOOPBACK_HOSTS + valid_scheme = parsed.scheme == "https" or ( + allow_loopback_http and loopback and parsed.scheme == "http" + ) + if not parsed.hostname or not valid_scheme: + raise ValueError(COLLECTOR_URL_HTTPS_ERROR) + if ( + parsed.username + or parsed.password + or parsed.query + or parsed.fragment + or parsed.path not in {"", "/"} + ): + raise ValueError(COLLECTOR_URL_ORIGIN_ERROR) + return value.rstrip("/") + + +def _development_collector_url(value: str) -> str: + parsed = urlsplit(value) + if parsed.hostname not in LOOPBACK_HOSTS: + raise ValueError(DEVELOPMENT_COLLECTOR_LOOPBACK_ERROR) + return _collector_origin(value, allow_loopback_http=True) + + +def _decode_response(body: bytes) -> dict[str, Any]: + try: + response = json.loads(body) if body else {} + except (json.JSONDecodeError, UnicodeDecodeError): + return {} + return response if isinstance(response, dict) else {} + + +def _http_post( + url: str, + payload: Mapping[str, Any], + bearer_token: str, + *, + timeout: float = HTTP_TIMEOUT_SECONDS, +) -> tuple[int, dict[str, Any]]: + request = urllib.request.Request( + url, + data=json.dumps(payload, separators=(",", ":")).encode(), + headers={ + "Authorization": f"Bearer {bearer_token}", + "Content-Type": "application/json", + "User-Agent": HTTP_USER_AGENT, + }, + method="POST", + ) + try: + opener = urllib.request.build_opener(_NoRedirectHandler) + with opener.open(request, timeout=timeout) as response: + body = response.read(MAX_HTTP_RESPONSE_BYTES + 1) + if len(body) > MAX_HTTP_RESPONSE_BYTES: + raise OSError(COLLECTOR_RESPONSE_TOO_LARGE_ERROR) + return response.status, _decode_response(body) + except urllib.error.HTTPError as error: + body = error.read(MAX_HTTP_RESPONSE_BYTES + 1) + if len(body) > MAX_HTTP_RESPONSE_BYTES: + return error.code, {} + return error.code, _decode_response(body) + + +def _huggingface_token() -> str | None: + return ( + os.environ.get("HF_TOKEN", "").strip() + or os.environ.get("HUGGINGFACE_TOKEN", "").strip() + or get_token() + ) + + +class CollectorSession: + """One thread-safe short-lived credential shared by independent publishers.""" + + def __init__(self, origin: str, *, post=None, credential=None): + import threading + + self.origin = origin + self._post = post or _http_post + self._credential = credential or _huggingface_token + self._session_token: tuple[str, float] | None = None + self._lock = threading.Lock() + + def invalidate(self) -> None: + with self._lock: + self._session_token = None + + def credential(self) -> tuple[str, str | None]: + with self._lock: + if ( + self._session_token is not None + and self._session_token[1] + > time.monotonic() + TOKEN_REFRESH_MARGIN_SECONDS + ): + return "ok", self._session_token[0] + hf_token = self._credential() + if not hf_token: + return "missing", None + try: + status, response = self._post( + self.origin + TOKEN_EXCHANGE_PATH, {}, hf_token + ) + except (OSError, TimeoutError): + return "unavailable", None + if status == HTTPStatus.OK and isinstance( + response.get("access_token"), str + ): + try: + expires_in = max( + MINIMUM_TOKEN_LIFETIME_SECONDS, + int(response.get("expires_in", DEFAULT_TOKEN_LIFETIME_SECONDS)), + ) + except (ValueError, TypeError): + return "unavailable", None + token = response["access_token"] + self._session_token = (token, time.monotonic() + expires_in) + return "ok", token + if status in {HTTPStatus.UNAUTHORIZED, HTTPStatus.FORBIDDEN}: + return "rejected", None + return "unavailable", None + + +def shared_session(services: dict, origin: str) -> CollectorSession: + """Share a session among explicit modules without adding auth to the host.""" + session = services.setdefault("collector_session", CollectorSession(origin)) + if not isinstance(session, CollectorSession) or session.origin != origin: + raise ValueError("Provider modules must use the same collector origin.") + return session diff --git a/src/microcosm_emitter/telemetry/service/constants.py b/src/microcosm_emitter/core/constants.py similarity index 100% rename from src/microcosm_emitter/telemetry/service/constants.py rename to src/microcosm_emitter/core/constants.py diff --git a/src/microcosm_emitter/telemetry/service/database.py b/src/microcosm_emitter/core/database.py similarity index 94% rename from src/microcosm_emitter/telemetry/service/database.py rename to src/microcosm_emitter/core/database.py index 02ed6f2c..2f434330 100644 --- a/src/microcosm_emitter/telemetry/service/database.py +++ b/src/microcosm_emitter/core/database.py @@ -8,7 +8,7 @@ from sqlalchemy.engine import Engine from sqlalchemy.orm import Session, sessionmaker -from microcosm_emitter.telemetry.service.constants import ( +from microcosm_emitter.core.constants import ( DATABASE_TIMEOUT_SECONDS, ) diff --git a/src/microcosm_emitter/telemetry/service/migrations.py b/src/microcosm_emitter/core/migrations.py similarity index 97% rename from src/microcosm_emitter/telemetry/service/migrations.py rename to src/microcosm_emitter/core/migrations.py index 3e0c4b76..9815d9d3 100644 --- a/src/microcosm_emitter/telemetry/service/migrations.py +++ b/src/microcosm_emitter/core/migrations.py @@ -15,7 +15,7 @@ from sqlalchemy import inspect from sqlalchemy.engine import Connection, Engine -from microcosm_emitter.telemetry.service.database import create_spool_engine +from microcosm_emitter.core.database import create_spool_engine _MIGRATION_TARGET = "head" diff --git a/src/microcosm_emitter/telemetry/service/models.py b/src/microcosm_emitter/core/models.py similarity index 82% rename from src/microcosm_emitter/telemetry/service/models.py rename to src/microcosm_emitter/core/models.py index 6b83b0e8..0bca2a97 100644 --- a/src/microcosm_emitter/telemetry/service/models.py +++ b/src/microcosm_emitter/core/models.py @@ -6,6 +6,7 @@ from typing import Any from sqlalchemy import ( + Float, ForeignKeyConstraint, Index, Integer, @@ -15,7 +16,7 @@ from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column, relationship from sqlalchemy.types import TypeDecorator -from microcosm_emitter.telemetry.service.constants import ( +from microcosm_emitter.core.constants import ( UPLOAD_STATE_PENDING, ) @@ -128,3 +129,17 @@ class TelemetryEventRecord(SpoolModel): ) created_at: Mapped[str] = mapped_column(Text, nullable=False) run: Mapped[TelemetryRunRecord] = relationship(back_populates="events") + + +class GraphPublicationJob(SpoolModel): + """Durable artifact delivery, independent of event retention and run IDs.""" + + __tablename__ = "graph_publication_jobs" + publication_id: Mapped[str] = mapped_column(Text, primary_key=True) + inventory: Mapped[dict[str, Any]] = mapped_column(JsonObjectText, nullable=False) + directory: Mapped[str] = mapped_column(Text, nullable=False) + status: Mapped[str] = mapped_column(Text, nullable=False) + attempts: Mapped[int] = mapped_column(Integer, nullable=False) + next_attempt_at: Mapped[float] = mapped_column(Float, nullable=False) + lease_until: Mapped[float] = mapped_column(Float, nullable=False) + receipt: Mapped[dict[str, Any]] = mapped_column(JsonObjectText, nullable=False) diff --git a/src/microcosm_emitter/host/__main__.py b/src/microcosm_emitter/host/__main__.py index 5e47ca15..1af8a6c8 100644 --- a/src/microcosm_emitter/host/__main__.py +++ b/src/microcosm_emitter/host/__main__.py @@ -14,7 +14,6 @@ def main() -> int: parser = argparse.ArgumentParser() - parser.add_argument("--module", required=True) parser.add_argument("--socket", type=Path, required=True) parser.add_argument("--parent-pid", type=int, required=True) parser.add_argument("--parent-created-at", type=float, required=True) @@ -23,12 +22,17 @@ def main() -> int: payload = sys.stdin.buffer.read(MAX_MESSAGE_BYTES + 1) if len(payload) > MAX_MESSAGE_BYTES: raise ValueError("configuration is too large") - configuration = json.loads(payload) - if not isinstance(configuration, dict): - raise ValueError("configuration must be an object") + configurations = json.loads(payload) + if not isinstance(configurations, dict) or not configurations: + raise ValueError("module configurations must be a nonempty object") parent = ParentProcess(args.parent_pid, args.parent_created_at) - module = load_module(args.module, configuration, ModuleContext(args.parent_pid)) - ServiceRuntime(args.socket, module, is_parent_alive=parent.alive).run() + context = ModuleContext(args.parent_pid) + modules = {} + for name, settings in configurations.items(): + if not isinstance(name, str) or not isinstance(settings, dict): + raise ValueError("invalid module configuration") + modules[name] = load_module(name, settings, context) + ServiceRuntime(args.socket, modules=modules, is_parent_alive=parent.alive).run() return 0 except Exception as error: # Never include configuration or module exception payloads in stderr. diff --git a/src/microcosm_emitter/host/client.py b/src/microcosm_emitter/host/client.py index 91bb64a8..c0b22042 100644 --- a/src/microcosm_emitter/host/client.py +++ b/src/microcosm_emitter/host/client.py @@ -1,5 +1,7 @@ """Bounded, private Unix-socket transport without domain-specific behavior.""" +from __future__ import annotations + import json import socket from pathlib import Path @@ -20,7 +22,9 @@ def __init__(self, socket_path: Path, *, timeout: float = SEND_TIMEOUT): self.timeout = timeout self.closed = False - def _request(self, payload: JsonObject) -> None: + def _request( + self, payload: JsonObject, *, timeout: float | None = None + ) -> JsonObject | None: try: data = ( json.dumps(payload, separators=(",", ":"), allow_nan=False).encode() @@ -29,23 +33,48 @@ def _request(self, payload: JsonObject) -> None: if len(data) > MAX_MESSAGE_BYTES: raise ValueError("local message exceeds size limit") with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as peer: - peer.settimeout(self.timeout) + peer.settimeout(self.timeout if timeout is None else timeout) peer.connect(str(self.socket_path)) peer.sendall(data) response = bytearray() - while DELIMITER not in response and len(response) < 16: - part = peer.recv(16 - len(response)) + while DELIMITER not in response: + part = peer.recv(min(4096, MAX_MESSAGE_BYTES + 1 - len(response))) if not part: break response.extend(part) - if bytes(response) != ACK_OK: + if len(response) > MAX_MESSAGE_BYTES: + raise ValueError("local response exceeds size limit") + if bytes(response) == ACK_OK: + return None + if not response.endswith(DELIMITER): + raise ValueError("incomplete response") + result = json.loads(bytes(response).removesuffix(DELIMITER)) + if not isinstance(result, dict): raise ValueError("service rejected the request") + return result except (OSError, ValueError, TypeError) as error: raise LocalServiceError(type(error).__name__) from error def ping(self) -> None: self._request({"action": "ping"}) + def for_module(self, name: str) -> ModuleClient: + """Route messages without giving a component ownership of host shutdown.""" + return ModuleClient(self, name) + + def request( + self, message: JsonObject, *, module: str, timeout: float | None = None + ) -> JsonObject: + """Return one bounded service result after its request has completed.""" + if self.closed: + raise LocalServiceError("client is closed") + response = self._request( + {"action": "query", "module": module, "message": message}, timeout=timeout + ) + if response is None: + raise LocalServiceError("service did not return a result") + return response + def send(self, message: JsonObject) -> None: if self.closed: raise LocalServiceError("client is closed") @@ -58,3 +87,25 @@ def close(self) -> None: self._request({"action": "close"}) finally: self.closed = True + + +class ModuleClient: + """One selected module's transport; the enclosing ServiceHandle owns shutdown.""" + + def __init__(self, client: SocketClient, name: str): + self.client, self.name = client, name + + def send(self, message: JsonObject) -> None: + if self.client.closed: + raise LocalServiceError("client is closed") + self.client._request( + {"action": "message", "module": self.name, "message": message} + ) + + def request( + self, message: JsonObject, *, timeout: float | None = None + ) -> JsonObject: + return self.client.request(message, module=self.name, timeout=timeout) + + def close(self) -> None: + """A domain component cannot close the shared host.""" diff --git a/src/microcosm_emitter/host/contracts.py b/src/microcosm_emitter/host/contracts.py index e7f2aa7c..a12f9ff7 100644 --- a/src/microcosm_emitter/host/contracts.py +++ b/src/microcosm_emitter/host/contracts.py @@ -1,7 +1,7 @@ """Interfaces shared by a local service host and its selected module.""" from collections.abc import Mapping -from dataclasses import dataclass +from dataclasses import dataclass, field from typing import Any, Protocol type JsonObject = Mapping[str, Any] @@ -12,6 +12,7 @@ class ModuleContext: """Identity of the process whose lifetime owns this service.""" parent_pid: int + services: dict[str, Any] = field(default_factory=dict) class ServiceModule(Protocol): diff --git a/src/microcosm_emitter/host/launcher.py b/src/microcosm_emitter/host/launcher.py index b823a27a..732d518f 100644 --- a/src/microcosm_emitter/host/launcher.py +++ b/src/microcosm_emitter/host/launcher.py @@ -52,12 +52,24 @@ def _reap(self) -> None: def start_service( module: str, configuration: JsonObject, *, startup_timeout: float = STARTUP_TIMEOUT +) -> ServiceHandle: + """Start one explicitly selected module.""" + return start_services({module: configuration}, startup_timeout=startup_timeout) + + +def start_services( + modules: JsonObject, *, startup_timeout: float = STARTUP_TIMEOUT ) -> ServiceHandle: """Return after readiness, or reap the child and remove temporary files.""" directory = None process = None try: - payload = json.dumps(configuration, allow_nan=False).encode() + if not modules or any( + not isinstance(name, str) or not isinstance(settings, dict) + for name, settings in modules.items() + ): + raise LocalServiceError("Module configuration must be a nonempty mapping.") + payload = json.dumps(modules, allow_nan=False).encode() if len(payload) > MAX_MESSAGE_BYTES: raise LocalServiceError("Service configuration is too large.") deadline = time.monotonic() + startup_timeout @@ -73,8 +85,6 @@ def start_service( sys.executable, "-m", MODULE_HOST, - "--module", - module, "--socket", str(path), "--parent-pid", diff --git a/src/microcosm_emitter/host/runtime.py b/src/microcosm_emitter/host/runtime.py index 26b5b9d7..a26c161b 100644 --- a/src/microcosm_emitter/host/runtime.py +++ b/src/microcosm_emitter/host/runtime.py @@ -1,4 +1,4 @@ -"""Host one explicitly selected service module for one parent process.""" +"""Host explicitly selected modules with independent periodic workers.""" import json import os @@ -26,28 +26,38 @@ class ServiceRuntime: def __init__( self, socket_path: Path, - module: ServiceModule, + module: ServiceModule | None = None, *, + modules: Mapping[str, ServiceModule] | None = None, is_parent_alive: Callable[[], bool], tick_seconds: float = TICK_SECONDS, drain_seconds: float = DRAIN_SECONDS, ): self.socket_path = socket_path + if (module is None) == (modules is None): + raise ValueError("Select one module or an explicit module mapping.") + self.modules = dict(modules) if modules is not None else {"default": module} + if not self.modules: + raise ValueError("At least one module is required.") self.module = module self.is_parent_alive = is_parent_alive self.tick_seconds = tick_seconds self.drain_seconds = drain_seconds self._stop = threading.Event() self._worker_error: Exception | None = None + self._failed: set[str] = set() def stop(self) -> None: self._stop.set() def run(self) -> None: - worker = None + workers = {} + initialized = [] bound = False try: - self.module.initialize() + for module in self.modules.values(): + initialized.append(module) + module.initialize() with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as server: # The launcher owns a fresh private directory. Never unlink an # existing socket which might belong to another running host. @@ -56,11 +66,17 @@ def run(self) -> None: os.chmod(self.socket_path, 0o600) server.listen(16) server.settimeout(SOCKET_TIMEOUT) - worker = threading.Thread(target=self._work, daemon=True) - worker.start() + for name, module in self.modules.items(): + worker = threading.Thread( + target=self._work, args=(name, module), daemon=True + ) + workers[name] = worker + worker.start() while not self._stop.is_set(): if not self.is_parent_alive(): - self.module.parent_exited() + for name, module in self.modules.items(): + if name not in self._failed: + module.parent_exited() break try: connection, _ = server.accept() @@ -75,12 +91,14 @@ def run(self) -> None: pass finally: self.stop() - if worker is not None: - worker.join(timeout=self.drain_seconds + SOCKET_TIMEOUT) + deadline = time.monotonic() + self.drain_seconds + SOCKET_TIMEOUT + for worker in workers.values(): + worker.join(timeout=max(0, deadline - time.monotonic())) if worker.is_alive(): self._worker_error = TimeoutError("Service shutdown timed out.") - else: - self.module.close(time.monotonic() + self.drain_seconds) + if not workers: + for module in initialized: + module.close(deadline) if bound: self.socket_path.unlink(missing_ok=True) if self._worker_error is not None: @@ -104,11 +122,28 @@ def _respond(self, connection: socket.socket) -> bytes: if not isinstance(request, Mapping): raise ValueError("request must be an object") action = request.get("action") - if action == "message": + if action in {"message", "query"}: message = request.get("message") if not isinstance(message, Mapping): raise ValueError("message must be an object") - self.module.handle_message(message) + name = request.get("module") + if name is None and len(self.modules) == 1: + name = next(iter(self.modules)) + if name not in self.modules or name in self._failed: + raise ValueError("module is unavailable") + module = self.modules[name] + if action == "query": + handler = getattr(module, "query", None) + if not callable(handler): + raise ValueError("module does not support queries") + response = ( + json.dumps(handler(message), allow_nan=False).encode() + + DELIMITER + ) + if len(response) > MAX_MESSAGE_BYTES: + raise ValueError("response too large") + return response + module.handle_message(message) elif action == "close": self.stop() elif action != "ping": @@ -118,16 +153,20 @@ def _respond(self, connection: socket.socket) -> bytes: return ACK_ERROR return ACK_OK - def _work(self) -> None: + def _work(self, name: str, module: ServiceModule) -> None: try: while not self._stop.wait(self.tick_seconds): - self.module.tick(time.monotonic()) + module.tick(time.monotonic()) except Exception as error: self._worker_error = error - self.stop() + self._failed.add(name) + if len(self._failed) == len(self.modules): + self.stop() finally: try: - self.module.close(time.monotonic() + self.drain_seconds) + module.close(time.monotonic() + self.drain_seconds) except Exception as error: self._worker_error = error - self.stop() + self._failed.add(name) + if len(self._failed) == len(self.modules): + self.stop() diff --git a/src/microcosm_emitter/orrery/__init__.py b/src/microcosm_emitter/orrery/__init__.py new file mode 100644 index 00000000..7e5fc1e6 --- /dev/null +++ b/src/microcosm_emitter/orrery/__init__.py @@ -0,0 +1 @@ +"""Immutable graph publication; no Microcosm dependency or import side effects.""" diff --git a/src/microcosm_emitter/orrery/cli.py b/src/microcosm_emitter/orrery/cli.py new file mode 100644 index 00000000..9a181538 --- /dev/null +++ b/src/microcosm_emitter/orrery/cli.py @@ -0,0 +1,39 @@ +"""Request graph retries from the local service without registering a telemetry run.""" + +import argparse +import json +from pathlib import Path + +from microcosm_emitter.orrery.client import GraphPublicationClient + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--spool", type=Path) + parser.add_argument("--publication-id", required=True) + parser.add_argument( + "--directory", + type=Path, + help="Preserved graph directory for re-enqueueing on another machine.", + ) + parser.add_argument("--origin", help="Graph publication origin.") + args = parser.parse_args(argv) + client = GraphPublicationClient.start( + spool_path=args.spool, publication_origin=args.origin + ) + try: + if args.directory is not None: + inventory = json.loads((args.directory / "orrery.upload.json").read_text()) + if inventory["publication_id"] != args.publication_id: + parser.error("The preserved inventory has a different publication ID.") + receipt = client.enqueue(args.directory, inventory) + if receipt["status"] == "failed": + print(json.dumps(receipt, sort_keys=True)) + return 1 + receipt = client.retry(args.publication_id, wait_seconds=30) + else: + receipt = client.retry(args.publication_id, wait_seconds=30) + print(json.dumps(receipt, sort_keys=True)) + return 0 if receipt["status"] == "published" else 1 + finally: + client.close() diff --git a/src/microcosm_emitter/orrery/client.py b/src/microcosm_emitter/orrery/client.py new file mode 100644 index 00000000..a31c9dcb --- /dev/null +++ b/src/microcosm_emitter/orrery/client.py @@ -0,0 +1,223 @@ +"""Thin graph requests; hashing, persistence, auth and uploads stay in the service.""" + +from __future__ import annotations + +import os +import time +from collections.abc import Mapping +from pathlib import Path +from typing import Protocol, cast + +from microcosm_emitter.host.contracts import JsonObject +from microcosm_emitter.host.errors import LocalServiceError +from microcosm_emitter.host.launcher import ServiceHandle, start_service +from microcosm_emitter.orrery.constants import GRAPH_REQUEST_TIMEOUT_SECONDS +from microcosm_emitter.orrery.contracts import ( + PublicationFileRole, + PublicationInventory, + PublicationReceipt, +) + + +class GraphTransport(Protocol): + """Only a request adapter, never a database or HTTP implementation.""" + + def request( + self, message: JsonObject, *, timeout: float | None = None + ) -> JsonObject: ... + + def close(self) -> None: ... + + +def default_spool_path() -> Path: + """Keep the existing queue location so versioned jobs survive extraction.""" + configured = os.environ.get("XDG_CACHE_HOME", "").strip() + root = Path(configured).expanduser() if configured else Path.home() / ".cache" + return root / "microcosm" / "telemetry" / "events.sqlite3" + + +def _unavailable(publication_id: str | None) -> PublicationReceipt: + return { + "version": 1, + "publication_id": publication_id, + "status": "failed", + "error_code": "local_service_unavailable", + } + + +class GraphPublicationClient: + """Send metadata to a separately running graph publisher over its private socket.""" + + def __init__(self, transport: GraphTransport | None = None) -> None: + self.transport = transport + self._handle: ServiceHandle | None = None + + @property + def available(self) -> bool: + return self.transport is not None + + @classmethod + def start( + cls, + *, + spool_path: Path | None = None, + development_collector_url: str | None = None, + publication_origin: str | None = None, + startup_timeout: float = 3.0, + ) -> GraphPublicationClient: + client = cls() + configuration = {"spool_path": str(spool_path or default_spool_path())} + if development_collector_url is not None: + configuration["development_collector_url"] = development_collector_url + if publication_origin is not None: + configuration["publication_origin"] = publication_origin + try: + client._handle = start_service( + "orrery", configuration, startup_timeout=startup_timeout + ) + client.transport = client._handle.client.for_module("orrery") + except (LocalServiceError, OSError, ValueError, TypeError): + # No acknowledgement means no claim that a durable job exists. + pass + return client + + def _request(self, message: JsonObject) -> JsonObject | None: + if self.transport is None: + return None + try: + return self.transport.request( + message, timeout=GRAPH_REQUEST_TIMEOUT_SECONDS + ) + except (LocalServiceError, OSError, ValueError, TypeError): + return None + + def inventory( + self, + directory: Path, + roles: Mapping[str, PublicationFileRole], + *, + publication_id: str | None = None, + ) -> PublicationInventory | None: + response = self._request( + { + "command": "inventory", + "directory": str(directory.resolve()), + "roles": dict(roles), + "publication_id": publication_id, + } + ) + value = response.get("inventory") if response is not None else None + return cast(PublicationInventory, value) if isinstance(value, dict) else None + + def enqueue( + self, directory: Path, inventory: PublicationInventory + ) -> PublicationReceipt: + response = self._request( + { + "command": "enqueue", + "directory": str(directory.resolve()), + "inventory": inventory, + } + ) + value = response.get("receipt") if response is not None else None + return ( + cast(PublicationReceipt, value) + if isinstance(value, dict) + else _unavailable(inventory["publication_id"]) + ) + + def receipt(self, publication_id: str) -> PublicationReceipt: + response = self._request( + {"command": "receipt", "publication_id": publication_id} + ) + value = response.get("receipt") if response is not None else None + return ( + cast(PublicationReceipt, value) + if isinstance(value, dict) + else _unavailable(publication_id) + ) + + def retry( + self, publication_id: str, *, wait_seconds: float = 0 + ) -> PublicationReceipt: + response = self._request({"command": "retry", "publication_id": publication_id}) + value = response.get("receipt") if response is not None else None + receipt = ( + cast(PublicationReceipt, value) + if isinstance(value, dict) + else _unavailable(publication_id) + ) + previous_attempts = receipt.get("attempts", 0) + deadline = time.monotonic() + max(0, wait_seconds) + while ( + self.available + and receipt["status"] == "pending" + and receipt.get("attempts", 0) == previous_attempts + and time.monotonic() < deadline + ): + time.sleep(0.1) + receipt = self.receipt(publication_id) + return receipt + + def close(self) -> None: + try: + if self._handle is not None: + self._handle.close() + elif self.transport is not None: + self.transport.close() + except (LocalServiceError, OSError): + pass + finally: + self.transport = None + + +def publication_inventory( + directory: Path, + roles: Mapping[str, PublicationFileRole], + *, + publication_id: str | None = None, + client: GraphPublicationClient | None = None, + spool_path: Path | None = None, +) -> PublicationInventory | None: + """Ask the service to hash allowed files without importing its implementation.""" + publisher = ( + client + if client is not None + else GraphPublicationClient.start(spool_path=spool_path) + ) + try: + return publisher.inventory(directory, roles, publication_id=publication_id) + finally: + if client is None: + publisher.close() + + +def publish_graph( + directory: Path, + inventory: PublicationInventory, + *, + spool_path: Path | None = None, + client: GraphPublicationClient | None = None, + wait_seconds: float = 30.0, +) -> PublicationReceipt: + """Wait only after service-side durable enqueue; never write a fallback queue.""" + publisher = ( + client + if client is not None + else GraphPublicationClient.start(spool_path=spool_path) + ) + try: + receipt = publisher.enqueue(directory, inventory) + deadline = time.monotonic() + max(0, wait_seconds) + while ( + publisher.available + and receipt["status"] == "pending" + and not receipt.get("attempts", 0) + and time.monotonic() < deadline + ): + time.sleep(0.1) + receipt = publisher.receipt(inventory["publication_id"]) + return receipt + finally: + if client is None: + publisher.close() diff --git a/src/microcosm_emitter/orrery/constants.py b/src/microcosm_emitter/orrery/constants.py new file mode 100644 index 00000000..8fe737db --- /dev/null +++ b/src/microcosm_emitter/orrery/constants.py @@ -0,0 +1,5 @@ +"""Bounded graph request limits, independent of short telemetry acknowledgements.""" + +from typing import Final + +GRAPH_REQUEST_TIMEOUT_SECONDS: Final = 30.0 diff --git a/src/microcosm_emitter/orrery/contracts.py b/src/microcosm_emitter/orrery/contracts.py new file mode 100644 index 00000000..94f4452f --- /dev/null +++ b/src/microcosm_emitter/orrery/contracts.py @@ -0,0 +1,57 @@ +"""Explicit data contracts for immutable graph publication.""" + +from dataclasses import dataclass +from pathlib import Path +from typing import Literal, NotRequired, TypedDict + +type PublicationFileRole = Literal[ + "graph", + "schema", + "index", + "declaration", + "manifest", + "binding", + "summaries", + "upstream", +] + + +class PublicationFile(TypedDict): + """One public file, identified by its exact bytes and allowed role.""" + + name: str + role: PublicationFileRole + bytes: int + sha256: str + + +class PublicationInventory(TypedDict): + """The versioned file inventory submitted to the publication API.""" + + version: Literal[1] + publication_id: str + files: list[PublicationFile] + + +class PublicationReceipt(TypedDict): + """Local publication result, including retry and build-snapshot fields.""" + + version: Literal[1] + publication_id: str | None + status: Literal["pending", "published", "skipped", "failed", "no_evidence"] + error_code: str | None + url: NotRequired[str | None] + attempts: NotRequired[int] + inventory_sha256: NotRequired[str] + preserved_directory: NotRequired[str] + recorded_execution: NotRequired[bool] + + +@dataclass(frozen=True) +class ClaimedGraphPublication: + """A job reserved until lease_until; the timestamp also identifies its lease.""" + + publication_id: str + directory: Path + inventory: PublicationInventory + lease_until: float diff --git a/src/microcosm_emitter/orrery/py.typed b/src/microcosm_emitter/orrery/py.typed new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/src/microcosm_emitter/orrery/py.typed @@ -0,0 +1 @@ + diff --git a/src/microcosm_emitter/orrery/service/__init__.py b/src/microcosm_emitter/orrery/service/__init__.py new file mode 100644 index 00000000..6f66f7e2 --- /dev/null +++ b/src/microcosm_emitter/orrery/service/__init__.py @@ -0,0 +1,8 @@ +"""Explicit graph-publication service module registration.""" + +from microcosm_emitter.orrery.service.module import OrreryModule + + +def create_module(configuration, context): + """Construct a module without starting workers or performing network I/O.""" + return OrreryModule(configuration, context) diff --git a/src/microcosm_emitter/orrery/service/module.py b/src/microcosm_emitter/orrery/service/module.py new file mode 100644 index 00000000..3f97adc7 --- /dev/null +++ b/src/microcosm_emitter/orrery/service/module.py @@ -0,0 +1,98 @@ +"""Completed graph delivery, independent of telemetry registration and retention.""" + +import sys +from pathlib import Path +from typing import cast + +from microcosm_emitter.core.auth import ( + _collector_origin, + _development_collector_url, + shared_session, +) +from microcosm_emitter.core.constants import PRODUCTION_COLLECTOR_URL +from microcosm_emitter.host.contracts import JsonObject, ModuleContext +from microcosm_emitter.orrery.contracts import PublicationInventory +from microcosm_emitter.orrery.service.queue import ( + DEFAULT_GRAPH_PUBLICATION_ORIGIN, + GRAPH_WORKER_ERROR_MESSAGE, + GraphPublicationDelivery, + GraphPublicationQueue, + publication_inventory, +) + + +class OrreryModule: + def __init__(self, configuration: JsonObject, context: ModuleContext): + self.configuration, self.context = configuration, context + self.queue: GraphPublicationQueue | None = None + self.delivery: GraphPublicationDelivery | None = None + self._warned = False + + def initialize(self) -> None: + queue = GraphPublicationQueue(Path(self.configuration["spool_path"])) + self.queue = queue + origin = ( + _development_collector_url(self.configuration["development_collector_url"]) + if self.configuration.get("development_collector_url") + else _collector_origin(PRODUCTION_COLLECTOR_URL) + ) + session = shared_session(self.context.services, origin) + self.delivery = GraphPublicationDelivery( + queue, + credential=session.credential, + invalidate_credential=session.invalidate, + origin=self.configuration.get( + "publication_origin", DEFAULT_GRAPH_PUBLICATION_ORIGIN + ), + ) + + def handle_message(self, message: JsonObject) -> None: + assert self.queue is not None, "The host initializes modules first." + # enqueue() recomputes the inventory from the files and rejects any + # message whose inventory differs, so this boundary is validated. + inventory = cast(PublicationInventory, dict(message["inventory"])) + self.queue.enqueue(Path(message["directory"]), inventory) + + def query(self, message: JsonObject) -> dict: + """Construct/persist results before the host returns its JSON response.""" + assert self.queue is not None, "The host initializes modules first." + command = message.get("command") + if command == "inventory": + return { + "inventory": publication_inventory( + Path(message["directory"]), + message["roles"], + publication_id=message.get("publication_id"), + ) + } + if command == "enqueue": + self.handle_message(message) + return { + "receipt": self.queue.receipt(message["inventory"]["publication_id"]) + } + if command == "retry": + self.queue.retry(message["publication_id"]) + elif command != "receipt": + raise ValueError("unsupported graph publication command") + return {"receipt": self.queue.receipt(message["publication_id"])} + + def tick(self, now: float) -> None: + assert self.delivery is not None, "The host initializes modules first." + try: + self.delivery.flush_once() + except Exception: + # The host stops a module whose tick raises. A database or + # filesystem error leaves the job leased for a later retry; it + # must not end delivery of every other job. + if not self._warned: + print(GRAPH_WORKER_ERROR_MESSAGE, file=sys.stderr, flush=True) + self._warned = True + + def parent_exited(self) -> None: + """Preserved jobs remain eligible when a later host resumes delivery.""" + + def close(self, deadline: float) -> None: + if self.delivery is not None: + self.delivery.client.close() + if self.queue is not None: + self.queue.close() diff --git a/src/microcosm_emitter/orrery/service/queue.py b/src/microcosm_emitter/orrery/service/queue.py new file mode 100644 index 00000000..9e7e592b --- /dev/null +++ b/src/microcosm_emitter/orrery/service/queue.py @@ -0,0 +1,433 @@ +"""Durable, retryable immutable graph uploads owned by the emitter service. + +Graph bytes stay in preserved files, never in event details. Pending graph jobs +are not subject to the telemetry event queue's time or size pruning. +""" + +from __future__ import annotations + +import hashlib +import json +import re +import time +import uuid +from collections.abc import Callable, Mapping +from pathlib import Path +from typing import cast +from urllib.parse import urlsplit + +import httpx +from sqlalchemy import select, update + +from microcosm_emitter.core.database import ( + create_spool_engine, + create_spool_session_factory, +) +from microcosm_emitter.core.migrations import upgrade_spool_database +from microcosm_emitter.core.models import GraphPublicationJob +from microcosm_emitter.orrery.contracts import ( + ClaimedGraphPublication, + PublicationFile, + PublicationFileRole, + PublicationInventory, + PublicationReceipt, +) + +MAX_GRAPH_FILE_BYTES = 64 * 1024 * 1024 +MAX_GRAPH_TOTAL_BYTES = 512 * 1024 * 1024 +GRAPH_PUBLICATION_ACTION = "graph_publication" +GRAPH_PUBLICATION_WAIT_SECONDS = 30.0 +GRAPH_PUBLICATION_LEASE_SECONDS = 300.0 +GRAPH_PUBLICATION_STATUS_FILENAME = "publication.status.json" +GRAPH_WORKER_ERROR_MESSAGE = ( + "Microcosm graph publication hit a local queue error; pending jobs remain " + "queued for a later attempt." +) +DEFAULT_GRAPH_PUBLICATION_ORIGIN = "https://microcosm-runs.vercel.app" +GRAPH_PUBLICATION_ROLES = { + "graph": r"graph\.orrery\.json", + "schema": r"graph\.schema\.json", + "index": r"execution\.evidence\.json", + "declaration": r"evidence-[A-Za-z0-9_-]+-graph\.json", + "manifest": r"evidence-[A-Za-z0-9_-]+-manifest\.json", + "binding": r"evidence-[A-Za-z0-9_-]+-binding\.json", + "summaries": r"evidence-[A-Za-z0-9_-]+-summaries\.json", + "upstream": r"upstream-[a-f0-9]{64}\.json", +} + + +def _digest(payload: bytes) -> str: + return hashlib.sha256(payload).hexdigest() + + +def inventory_digest(inventory: PublicationInventory) -> str: + """Digest the versioned field order shared with the publication API.""" + ordered: PublicationInventory = { + "version": inventory["version"], + "publication_id": inventory["publication_id"], + "files": [ + { + "name": item["name"], + "role": item["role"], + "bytes": item["bytes"], + "sha256": item["sha256"], + } + for item in sorted(inventory["files"], key=lambda file: file["name"]) + ], + } + return _digest( + json.dumps(ordered, separators=(",", ":"), ensure_ascii=True).encode() + ) + + +def publication_inventory( + directory: Path, + roles: Mapping[str, PublicationFileRole], + *, + publication_id: str | None = None, +) -> PublicationInventory: + """Hash an explicit allowlist of files; publication identity is not a run ID.""" + files: list[PublicationFile] = [] + for name, role in sorted(roles.items()): + if role not in GRAPH_PUBLICATION_ROLES or not re.fullmatch( + GRAPH_PUBLICATION_ROLES[role], name + ): + raise ValueError("Ineligible graph publication filename or role.") + path = Path(directory) / name + if path.is_symlink() or not path.is_file(): + raise ValueError( + "Graph publication files must be regular, preserved files." + ) + size = path.stat().st_size + if not 0 < size <= MAX_GRAPH_FILE_BYTES: + raise ValueError("Graph publication file exceeds its size limit.") + files.append( + { + "name": name, + "role": role, + "bytes": size, + "sha256": _digest(path.read_bytes()), + } + ) + if ( + not files + or len(files) > 256 + or sum(f["bytes"] for f in files) > MAX_GRAPH_TOTAL_BYTES + or sum(f["role"] == "graph" for f in files) != 1 + ): + raise ValueError("Invalid graph publication inventory.") + id_ = publication_id or uuid.uuid4().hex + if not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_-]{0,127}", id_): + raise ValueError("Invalid graph publication ID.") + return {"version": 1, "publication_id": id_, "files": files} + + +def _write_status(directory: Path, receipt: PublicationReceipt) -> None: + path = directory / GRAPH_PUBLICATION_STATUS_FILENAME + temporary = path.with_suffix(f".{uuid.uuid4().hex}.tmp") + try: + temporary.write_text(json.dumps(receipt, sort_keys=True) + "\n") + temporary.replace(path) + except OSError: + temporary.unlink(missing_ok=True) + + +class GraphPublicationQueue: + """SQLite job records with leases; retained until publication or operator action.""" + + def __init__(self, path: Path) -> None: + self.path = Path(path) + self.path.parent.mkdir(parents=True, exist_ok=True) + self._engine = create_spool_engine(self.path) + upgrade_spool_database(self._engine) + self._sessions = create_spool_session_factory(self._engine) + + def enqueue(self, directory: Path, inventory: PublicationInventory) -> None: + """Durably register exact bytes without a telemetry run or credential.""" + directory = Path(directory).resolve() + expected = publication_inventory( + directory, + {f["name"]: f["role"] for f in inventory["files"]}, + publication_id=inventory["publication_id"], + ) + if expected != inventory: + raise ValueError("Graph publication digest mismatch.") + id_ = inventory["publication_id"] + with self._sessions.begin() as session: + existing = session.get(GraphPublicationJob, id_) + if existing is not None: + if existing.inventory != inventory or existing.directory != str( + directory + ): + raise ValueError( + "Graph publication identity conflicts with a queued job." + ) + return + session.add( + GraphPublicationJob( + publication_id=id_, + inventory=inventory, + directory=str(directory), + status="pending", + attempts=0, + next_attempt_at=0, + lease_until=0, + receipt={ + "version": 1, + "publication_id": id_, + "status": "pending", + "error_code": None, + }, + ) + ) + + def close(self) -> None: + """Release this handle's database connections without deleting jobs.""" + self._engine.dispose() + + def claim( + self, + *, + lease_seconds: float = GRAPH_PUBLICATION_LEASE_SECONDS, + publication_id: str | None = None, + ) -> ClaimedGraphPublication | None: + """Reserve one pending job whose retry delay and previous lease expired. + + A conditional database update allows only one current lease. Workers + that exit leave their jobs eligible again after lease expiry; completion + uses the returned lease timestamp to reject outdated workers. + """ + now = time.time() + with self._sessions.begin() as session: + statement = select(GraphPublicationJob.publication_id) + if publication_id is not None: + statement = statement.where( + GraphPublicationJob.publication_id == publication_id + ) + id_ = session.scalar( + statement.where( + GraphPublicationJob.status == "pending", + GraphPublicationJob.next_attempt_at <= now, + GraphPublicationJob.lease_until <= now, + ) + # Earliest-due first, so a job that keeps failing cannot + # repeatedly displace other pending publications. + .order_by(GraphPublicationJob.next_attempt_at) + .limit(1) + ) + if id_ is None: + return None + claimed_id = session.scalar( + update(GraphPublicationJob) + .where( + GraphPublicationJob.publication_id == id_, + GraphPublicationJob.status == "pending", + GraphPublicationJob.lease_until <= now, + ) + .values(lease_until=now + lease_seconds) + .returning(GraphPublicationJob.publication_id) + ) + if claimed_id is None: + return None + job = session.get(GraphPublicationJob, id_) + assert job is not None, ( + "A claimed publication must exist in its transaction." + ) + return ClaimedGraphPublication( + publication_id=id_, + directory=Path(job.directory), + inventory=cast(PublicationInventory, job.inventory), + lease_until=job.lease_until, + ) + + def receipt(self, id_: str) -> PublicationReceipt | None: + """Read the latest local result; old HF receipt snapshots remain unchanged.""" + with self._sessions() as session: + job = session.get(GraphPublicationJob, id_) + # ORM JSON columns are generic at the persistence boundary. + return None if job is None else cast(PublicationReceipt, dict(job.receipt)) + + def finish( + self, + id_: str, + *, + error_code: str | None = None, + url: str | None = None, + expected_lease: float | None = None, + ) -> None: + with self._sessions.begin() as session: + statement = update(GraphPublicationJob).where( + GraphPublicationJob.publication_id == id_, + GraphPublicationJob.status == "pending", + ) + if expected_lease is not None: + statement = statement.where( + GraphPublicationJob.lease_until == expected_lease + ) + job = session.scalar( + statement.values( + attempts=GraphPublicationJob.attempts + 1, + status="published" if url else "pending", + lease_until=0, + ).returning(GraphPublicationJob) + ) + # Completion is conditional in SQL, not a read-then-write check: + # expired workers cannot release another lease or replace success. + if job is None: + return + job.next_attempt_at = time.time() + min(300, 2 ** min(job.attempts, 8)) + receipt: PublicationReceipt = { + "version": 1, + "publication_id": id_, + "status": "published" if url else "pending", + "error_code": error_code, + "url": url, + "attempts": job.attempts, + "inventory_sha256": inventory_digest( + cast(PublicationInventory, job.inventory) + ), + } + job.receipt = dict(receipt) + directory = Path(job.directory) + # The queue row is authoritative. This separate latest-result file (not + # the build/HF receipt) is a convenience copy, written only after the + # commit: a deleted preserved directory must not roll back the result. + _write_status(directory, receipt) + + def retry(self, id_: str | None = None) -> None: + """Make preserved pending jobs eligible again, including missing credentials.""" + statement = update(GraphPublicationJob).where( + GraphPublicationJob.status == "pending" + ) + if id_ is not None: + statement = statement.where(GraphPublicationJob.publication_id == id_) + with self._sessions.begin() as session: + session.execute(statement.values(next_attempt_at=0)) + + +class GraphPublicationDelivery: + """Upload exact files with scoped signed URLs, then seal the publication.""" + + def __init__( + self, + queue: GraphPublicationQueue, + *, + credential: Callable[[], tuple[str, str | None]], + origin: str, + client: httpx.Client | None = None, + invalidate_credential: Callable[[], None] | None = None, + ) -> None: + parsed = urlsplit(origin) + if parsed.scheme != "https" and not ( + parsed.scheme == "http" + and parsed.hostname in {"localhost", "127.0.0.1", "::1"} + ): + raise ValueError( + "Graph publication origin must be HTTPS or local development." + ) + if ( + parsed.username + or parsed.password + or parsed.query + or parsed.fragment + or parsed.path not in {"", "/"} + ): + raise ValueError("Graph publication destination must be an origin.") + self.queue, self.credential, self.origin = queue, credential, origin.rstrip("/") + self.client = client or httpx.Client(timeout=60, follow_redirects=False) + self.invalidate_credential = invalidate_credential + + def flush_once(self, *, publication_id: str | None = None) -> bool: + """Try one job; failures retain it without changing telemetry eligibility.""" + job = self.queue.claim(publication_id=publication_id) + if job is None: + return False + id_, inventory, directory = ( + job.publication_id, + job.inventory, + job.directory, + ) + error = "publication_unavailable" + lease = job.lease_until + if not directory.is_dir(): + # Retained until an operator restores the files to this directory; + # the retry backoff lets other pending jobs proceed meanwhile. + self.queue.finish( + id_, error_code="preserved_files_missing", expected_lease=lease + ) + return False + try: + status, token = self.credential() + if token is None: + self.queue.finish( + id_, error_code=f"credential_{status}", expected_lease=lease + ) + return False + current = publication_inventory( + directory, + {f["name"]: f["role"] for f in inventory["files"]}, + publication_id=id_, + ) + if current != inventory: + raise ValueError("preserved_bytes_changed") + headers = {"Authorization": f"Bearer {token}"} + response = self.client.post( + self.origin + "/api/uploads/", json=inventory, headers=headers + ) + response.raise_for_status() + body = response.json() + uploads = body.get("uploads") + if not isinstance(uploads, list): + raise ValueError("invalid_upload_response") + files = {f["name"]: f for f in inventory["files"]} + for upload in uploads: + name = upload.get("name") + if name not in files: + raise ValueError("invalid_upload_response") + if upload.get("already_uploaded"): + continue + url = urlsplit(upload["url"]) + if ( + url.scheme != "https" + or not ( + (url.hostname == "vercel.com" and url.path == "/api/blob/") + or url.hostname == "blob.vercel-storage.com" + or (url.hostname or "").endswith(".blob.vercel-storage.com") + ) + or url.username + or url.password + ): + raise ValueError("invalid_upload_url") + with (directory / name).open("rb") as stream: + result = self.client.put( + upload["url"], + content=stream, + headers={ + "Content-Type": "application/json", + "Content-Length": str(files[name]["bytes"]), + }, + ) + result.raise_for_status() + response = self.client.post( + self.origin + "/api/publications/", json=inventory, headers=headers + ) + response.raise_for_status() + result = response.json() + if result.get("publication_id") != id_ or result.get( + "inventory_sha256" + ) != inventory_digest(inventory): + raise ValueError("invalid_publication_response") + self.queue.finish( + id_, url=f"{self.origin}/runs/{id_}/", expected_lease=lease + ) + return True + except httpx.HTTPStatusError as exception: + error = f"publication_http_{exception.response.status_code}" + if exception.response.status_code == 401 and self.invalidate_credential: + self.invalidate_credential() + except httpx.HTTPError: + error = "publication_network_error" + except (OSError, ValueError, KeyError, TypeError): + error = "publication_invalid_evidence_or_response" + self.queue.finish(id_, error_code=error, expected_lease=lease) + return False diff --git a/src/microcosm_emitter/py.typed b/src/microcosm_emitter/py.typed new file mode 100644 index 00000000..5edeae50 --- /dev/null +++ b/src/microcosm_emitter/py.typed @@ -0,0 +1 @@ +# Package includes inline Python typing information. diff --git a/src/microcosm_emitter/telemetry/client.py b/src/microcosm_emitter/telemetry/client.py index 2448d2e9..4fd70a9b 100644 --- a/src/microcosm_emitter/telemetry/client.py +++ b/src/microcosm_emitter/telemetry/client.py @@ -22,6 +22,7 @@ TelemetryStatus, ) from microcosm_emitter.telemetry.requests import ( + COMMAND_BLOCK, COMMAND_CALIBRATION_PROGRESS, COMMAND_COMPLETE, COMMAND_EMIT, @@ -75,10 +76,12 @@ def __init__( run: TelemetryRun, transport: Transport | None, handle: ServiceHandle | None = None, + close_service_on_finish: bool = True, ) -> None: self.run = run self._transport = transport self._handle = handle + self._close_service_on_finish = close_service_on_finish self._closed = False self._warned = False @@ -92,6 +95,7 @@ def start( development_collector_url: str | None = None, heartbeat_seconds: float = DEFAULT_HEARTBEAT_SECONDS, startup_timeout_seconds: float = 3.0, + close_service_on_finish: bool = True, ) -> LocalTelemetryEmitter: """Start the installed service; return a disabled adapter on failure.""" try: @@ -113,8 +117,15 @@ def start( SERVICE_START_WARNING.format(error_type=type(error).__name__), file=sys.stderr, ) - return cls(run=run, transport=None) - return cls(run=run, transport=handle.client, handle=handle) + return cls( + run=run, transport=None, close_service_on_finish=close_service_on_finish + ) + return cls( + run=run, + transport=handle.client, + handle=handle, + close_service_on_finish=close_service_on_finish, + ) @property def available(self) -> bool: @@ -215,6 +226,7 @@ def fail( *, failed_during: str | None = None, failure_class: str = DEFAULT_FAILURE_CLASS, + error_code: str | None = None, ) -> None: try: try: @@ -228,15 +240,39 @@ def fail( error_type=type(error).__name__, failed_during=failed_during, failure_class=failure_class, + **({"error_code": error_code} if error_code is not None else {}), + ) + finally: + if self._close_service_on_finish: + self.close() + + def block( + self, + *, + phase: str, + blocking_gate_ids: list[str] | tuple[str, ...], + blocking_failure_count: int | None = None, + gate_statuses: Mapping[str, str] | None = None, + ) -> None: + """Ask the service to finish a run refused by its validation checks.""" + try: + self._request( + COMMAND_BLOCK, + phase=phase, + blocking_gate_ids=blocking_gate_ids, + blocking_failure_count=blocking_failure_count, + gate_statuses=gate_statuses, ) finally: - self.close() + if self._close_service_on_finish: + self.close() def complete(self) -> None: try: self._request(COMMAND_COMPLETE) finally: - self.close() + if self._close_service_on_finish: + self.close() def close(self) -> None: """Request a bounded background drain without inventing a build outcome.""" diff --git a/src/microcosm_emitter/telemetry/constants.py b/src/microcosm_emitter/telemetry/constants.py index 5646c9aa..8a3ea561 100644 --- a/src/microcosm_emitter/telemetry/constants.py +++ b/src/microcosm_emitter/telemetry/constants.py @@ -8,3 +8,9 @@ QUEUE_WARNING: Final = ( "Telemetry could not be queued ({error_type}); the build will continue." ) +LOCAL_ONLY_REJECTED_EVENTS: Final = "collector_rejected_events" +REJECTED_EVENTS_MESSAGE: Final = ( + "Microcosm telemetry is local-only for this run: the collector rejected its " + "events (HTTP {status}), so they will not be retried. The dataset build will " + "continue." +) diff --git a/src/microcosm_emitter/telemetry/protocol.py b/src/microcosm_emitter/telemetry/protocol.py index bbe1dce6..41a75324 100644 --- a/src/microcosm_emitter/telemetry/protocol.py +++ b/src/microcosm_emitter/telemetry/protocol.py @@ -8,7 +8,7 @@ type TelemetryEventType = Literal[ "run", "stage", "progress", "calibration", "heartbeat" ] -type TelemetryStatus = Literal["started", "progress", "completed", "failed"] +type TelemetryStatus = Literal["started", "progress", "completed", "failed", "blocked"] TELEMETRY_SCHEMA_VERSION: Final = 1 TELEMETRY_IDENTIFIER_PATTERN: Final = r"^[A-Za-z0-9][A-Za-z0-9._:-]{0,191}$" @@ -23,15 +23,18 @@ STATUS_PROGRESS: Final = "progress" STATUS_COMPLETED: Final = "completed" STATUS_FAILED: Final = "failed" +STATUS_BLOCKED: Final = "blocked" STAGE_CREATED: Final = "created" STAGE_CALIBRATING: Final = "calibrating" STAGE_COMPLETE: Final = "complete" STAGE_FAILED: Final = "failed" +STAGE_BLOCKED: Final = "blocked" CALIBRATION_EVENT_KIND: Final = "calibration_epoch" BUILD_STARTED_MESSAGE: Final = "Microcosm build started." BUILD_COMPLETED_MESSAGE: Final = "Microcosm build completed." +BUILD_BLOCKED_MESSAGE: Final = "Validation checks refused the candidate at {phase}." UNEXPECTED_PROCESS_EXIT_MESSAGE: Final = ( "The build process exited without reporting completion." ) @@ -41,6 +44,7 @@ MAX_TELEMETRY_DETAILS_DEPTH: Final = 6 MAX_TELEMETRY_COLLECTION_ITEMS: Final = 200 MAX_TELEMETRY_DETAILS_BYTES: Final = 8_192 +MAX_TELEMETRY_CHECK_ID_CHARS: Final = 200 SEQUENTIAL_STATUS_MAP: Final = MappingProxyType( { "failed": STATUS_FAILED, diff --git a/src/microcosm_emitter/telemetry/requests.py b/src/microcosm_emitter/telemetry/requests.py index 61abf95c..039c2bd6 100644 --- a/src/microcosm_emitter/telemetry/requests.py +++ b/src/microcosm_emitter/telemetry/requests.py @@ -9,5 +9,6 @@ COMMAND_CALIBRATION_PROGRESS: Final = "calibration_progress" COMMAND_TRANSITION_CALIBRATION_PROGRESS: Final = "transition_calibration_progress" COMMAND_FAIL: Final = "fail" +COMMAND_BLOCK: Final = "block" COMMAND_COMPLETE: Final = "complete" DEFAULT_FAILURE_CLASS: Final = "build_failure" diff --git a/src/microcosm_emitter/telemetry/service/collector.py b/src/microcosm_emitter/telemetry/service/collector.py index eac2ab5f..e9c4ba8c 100644 --- a/src/microcosm_emitter/telemetry/service/collector.py +++ b/src/microcosm_emitter/telemetry/service/collector.py @@ -1,126 +1,46 @@ -"""Collector authentication and best-effort event delivery.""" +"""Best-effort telemetry delivery; shared authentication lives outside this component.""" from __future__ import annotations -import json -import os import sys import time -import urllib.error -import urllib.request from collections.abc import Mapping from http import HTTPStatus from typing import Any -from urllib.parse import urlsplit -from huggingface_hub import get_token - -from microcosm_emitter.telemetry.service.constants import ( - COLLECTOR_RESPONSE_TOO_LARGE_ERROR, - COLLECTOR_URL_HTTPS_ERROR, - COLLECTOR_URL_ORIGIN_ERROR, - DEFAULT_TOKEN_LIFETIME_SECONDS, - DEVELOPMENT_COLLECTOR_LOOPBACK_ERROR, - HTTP_TIMEOUT_SECONDS, - HTTP_USER_AGENT, +from microcosm_emitter.core.auth import ( + CollectorSession, + _collector_origin, + _development_collector_url, + _http_post, + _huggingface_token, +) +from microcosm_emitter.core.constants import ( INITIAL_RETRY_SECONDS, LOCAL_ONLY_MISSING_CREDENTIAL, LOCAL_ONLY_REJECTED_COLLECTOR_AUTHORIZATION, LOCAL_ONLY_REJECTED_CREDENTIAL, LOCAL_ONLY_REJECTED_REGISTRATION, - LOOPBACK_HOSTS, - MAX_HTTP_RESPONSE_BYTES, MAX_RETRY_SECONDS, - MINIMUM_TOKEN_LIFETIME_SECONDS, NO_CREDENTIAL_MESSAGE, PRODUCTION_COLLECTOR_URL, REJECTED_CREDENTIAL_MESSAGE, RUN_EVENTS_PATH_TEMPLATE, RUN_REGISTRATION_PATH, - TOKEN_EXCHANGE_PATH, - TOKEN_REFRESH_MARGIN_SECONDS, +) +from microcosm_emitter.telemetry.constants import ( + LOCAL_ONLY_REJECTED_EVENTS, + REJECTED_EVENTS_MESSAGE, ) from microcosm_emitter.telemetry.service.spool import EventSpool - -class _NoRedirectHandler(urllib.request.HTTPRedirectHandler): - """Keep bearer credentials on the explicitly configured origin.""" - - def redirect_request(self, req, fp, code, msg, headers, newurl): - return None - - -def _collector_origin(value: str, *, allow_loopback_http: bool = False) -> str: - parsed = urlsplit(value) - loopback = parsed.hostname in LOOPBACK_HOSTS - valid_scheme = parsed.scheme == "https" or ( - allow_loopback_http and loopback and parsed.scheme == "http" - ) - if not parsed.hostname or not valid_scheme: - raise ValueError(COLLECTOR_URL_HTTPS_ERROR) - if ( - parsed.username - or parsed.password - or parsed.query - or parsed.fragment - or parsed.path not in {"", "/"} - ): - raise ValueError(COLLECTOR_URL_ORIGIN_ERROR) - return value.rstrip("/") - - -def _development_collector_url(value: str) -> str: - parsed = urlsplit(value) - if parsed.hostname not in LOOPBACK_HOSTS: - raise ValueError(DEVELOPMENT_COLLECTOR_LOOPBACK_ERROR) - return _collector_origin(value, allow_loopback_http=True) - - -def _decode_response(body: bytes) -> dict[str, Any]: - try: - response = json.loads(body) if body else {} - except (json.JSONDecodeError, UnicodeDecodeError): - return {} - return response if isinstance(response, dict) else {} - - -def _http_post( - url: str, - payload: Mapping[str, Any], - bearer_token: str, - *, - timeout: float = HTTP_TIMEOUT_SECONDS, -) -> tuple[int, dict[str, Any]]: - request = urllib.request.Request( - url, - data=json.dumps(payload, separators=(",", ":")).encode(), - headers={ - "Authorization": f"Bearer {bearer_token}", - "Content-Type": "application/json", - "User-Agent": HTTP_USER_AGENT, - }, - method="POST", - ) - try: - opener = urllib.request.build_opener(_NoRedirectHandler) - with opener.open(request, timeout=timeout) as response: - body = response.read(MAX_HTTP_RESPONSE_BYTES + 1) - if len(body) > MAX_HTTP_RESPONSE_BYTES: - raise OSError(COLLECTOR_RESPONSE_TOO_LARGE_ERROR) - return response.status, _decode_response(body) - except urllib.error.HTTPError as error: - body = error.read(MAX_HTTP_RESPONSE_BYTES + 1) - if len(body) > MAX_HTTP_RESPONSE_BYTES: - return error.code, {} - return error.code, _decode_response(body) +_RETRYABLE_CLIENT_ERRORS = frozenset( + {HTTPStatus.REQUEST_TIMEOUT, HTTPStatus.TOO_MANY_REQUESTS} +) -def _huggingface_token() -> str | None: - return ( - os.environ.get("HF_TOKEN", "").strip() - or os.environ.get("HUGGINGFACE_TOKEN", "").strip() - or get_token() - ) +def _permanent_client_error(status: int) -> bool: + return 400 <= int(status) < 500 and status not in _RETRYABLE_CLIENT_ERRORS def _registration_key(registration: Mapping[str, Any]) -> str: @@ -135,6 +55,7 @@ def __init__( spool: EventSpool, *, development_collector_url: str | None = None, + session: CollectorSession | None = None, ) -> None: self.collector_url = ( _development_collector_url(development_collector_url) @@ -142,7 +63,11 @@ def __init__( else _collector_origin(PRODUCTION_COLLECTOR_URL) ) self.spool = spool - self._session_token: tuple[str, float] | None = None + self.session = session or CollectorSession( + self.collector_url, + post=lambda *args: _http_post(*args), + credential=lambda: _huggingface_token(), + ) self._registered: set[str] = set() self._warned_no_token = False self._warned_denied: set[str] = set() @@ -181,51 +106,34 @@ def flush_once(self) -> bool: made_progress = True self._retry_seconds = INITIAL_RETRY_SECONDS elif status == HTTPStatus.UNAUTHORIZED: - self._session_token = None + self.session.invalidate() elif status == HTTPStatus.FORBIDDEN: self._make_local_only( registration, LOCAL_ONLY_REJECTED_COLLECTOR_AUTHORIZATION, ) + elif _permanent_client_error(status): + self._make_local_only( + registration, + LOCAL_ONLY_REJECTED_EVENTS, + message=REJECTED_EVENTS_MESSAGE.format(status=int(status)), + ) else: self._defer_retry() return made_progress def _collector_token(self, registration: Mapping[str, Any]) -> str | None: - if ( - self._session_token is not None - and self._session_token[1] > time.monotonic() + TOKEN_REFRESH_MARGIN_SECONDS - ): - return self._session_token[0] - hf_token = _huggingface_token() - if not hf_token: + result, token = self.session.credential() + if result == "missing": if not self._warned_no_token: print(NO_CREDENTIAL_MESSAGE, file=sys.stderr, flush=True) self._warned_no_token = True self._make_local_only(registration, LOCAL_ONLY_MISSING_CREDENTIAL) - return None - try: - status, response = _http_post( - self.collector_url + TOKEN_EXCHANGE_PATH, - {}, - hf_token, - ) - except (OSError, TimeoutError): - self._defer_retry() - return None - if status == HTTPStatus.OK and isinstance(response.get("access_token"), str): - expires_in = max( - MINIMUM_TOKEN_LIFETIME_SECONDS, - int(response.get("expires_in", DEFAULT_TOKEN_LIFETIME_SECONDS)), - ) - token = response["access_token"] - self._session_token = (token, time.monotonic() + expires_in) - return token - if status in {HTTPStatus.UNAUTHORIZED, HTTPStatus.FORBIDDEN}: + elif result == "rejected": self._make_local_only(registration, LOCAL_ONLY_REJECTED_CREDENTIAL) - else: + elif result != "ok": self._defer_retry() - return None + return token def _register(self, registration: Mapping[str, Any], token: str) -> bool: registration_key = _registration_key(registration) @@ -243,8 +151,11 @@ def _register(self, registration: Mapping[str, Any], token: str) -> bool: self._retry_seconds = INITIAL_RETRY_SECONDS return True if status == HTTPStatus.UNAUTHORIZED: - self._session_token = None - elif status in {HTTPStatus.FORBIDDEN, HTTPStatus.CONFLICT}: + self.session.invalidate() + elif status in { + HTTPStatus.FORBIDDEN, + HTTPStatus.CONFLICT, + } or _permanent_client_error(status): self._make_local_only(registration, LOCAL_ONLY_REJECTED_REGISTRATION) else: self._defer_retry() @@ -254,20 +165,22 @@ def _make_local_only( self, registration: Mapping[str, Any], reason: str, + *, + message: str = REJECTED_CREDENTIAL_MESSAGE, ) -> None: run_id = str(registration["run_id"]) producer_id = str(registration["producer_id"]) registration_key = _registration_key(registration) self.spool.make_local_only(run_id, producer_id, reason) if reason != LOCAL_ONLY_MISSING_CREDENTIAL: - self._warn_denied(registration_key) + self._warn_denied(registration_key, message) def _defer_retry(self) -> None: self._next_attempt_at = time.monotonic() + self._retry_seconds self._retry_seconds = min(MAX_RETRY_SECONDS, self._retry_seconds * 2) - def _warn_denied(self, registration_key: str) -> None: + def _warn_denied(self, registration_key: str, message: str) -> None: if registration_key in self._warned_denied: return - print(REJECTED_CREDENTIAL_MESSAGE, file=sys.stderr, flush=True) + print(message, file=sys.stderr, flush=True) self._warned_denied.add(registration_key) diff --git a/src/microcosm_emitter/telemetry/service/lifecycle.py b/src/microcosm_emitter/telemetry/service/lifecycle.py index 61ee2114..70bc9218 100644 --- a/src/microcosm_emitter/telemetry/service/lifecycle.py +++ b/src/microcosm_emitter/telemetry/service/lifecycle.py @@ -6,6 +6,7 @@ from typing import Any from microcosm_emitter.telemetry.protocol import ( + BUILD_BLOCKED_MESSAGE, BUILD_COMPLETED_MESSAGE, BUILD_STARTED_MESSAGE, CALIBRATION_EVENT_KIND, @@ -14,12 +15,15 @@ EVENT_TYPE_PROGRESS, EVENT_TYPE_RUN, EVENT_TYPE_STAGE, + MAX_TELEMETRY_CHECK_ID_CHARS, MAX_TELEMETRY_MESSAGE_CHARS, SEQUENTIAL_STATUS_MAP, + STAGE_BLOCKED, STAGE_CALIBRATING, STAGE_COMPLETE, STAGE_CREATED, STAGE_FAILED, + STATUS_BLOCKED, STATUS_COMPLETED, STATUS_FAILED, STATUS_PROGRESS, @@ -27,6 +31,7 @@ TELEMETRY_IDENTIFIER_PATTERN, ) from microcosm_emitter.telemetry.requests import ( + COMMAND_BLOCK, COMMAND_CALIBRATION_PROGRESS, COMMAND_COMPLETE, COMMAND_EMIT, @@ -53,7 +58,7 @@ } ) _STATUSES = frozenset( - {STATUS_STARTED, STATUS_PROGRESS, STATUS_COMPLETED, STATUS_FAILED} + {STATUS_STARTED, STATUS_PROGRESS, STATUS_COMPLETED, STATUS_FAILED, STATUS_BLOCKED} ) @@ -111,6 +116,28 @@ def _optional_stage_id(value: Any) -> str | None: return value +def _validate_block_details(details: Mapping[str, Any]) -> None: + """Match the collector's required refusal metadata before persistence.""" + phase = details.get("phase") + count = details.get("blocking_failure_count") + check_ids = details.get("blocking_gate_ids") + if ( + not isinstance(phase, str) + or re.fullmatch(TELEMETRY_IDENTIFIER_PATTERN, phase) is None + ): + raise ValueError("blocked phase must be a safe identifier") + if isinstance(count, bool) or not isinstance(count, int) or count < 1: + raise ValueError("blocking_failure_count must be an integer of at least 1") + if not isinstance(check_ids, list) or any( + not isinstance(check_id, str) + or not 1 <= len(check_id) <= MAX_TELEMETRY_CHECK_ID_CHARS + for check_id in check_ids + ): + raise ValueError( + "blocking_gate_ids must contain check identifiers of 1 to 200 characters" + ) + + def _event( event_type: str, status: str, @@ -123,6 +150,13 @@ def _event( if not isinstance(status, str) or status not in _STATUSES: raise ValueError("unsupported telemetry status") stage_id, message = _optional_stage_id(stage_id), _optional_text(message) + original_details = _mapping(details if details is not None else {}) + if status == STATUS_BLOCKED: + _validate_block_details(original_details) + safe_details = sanitize_details(original_details) + if status == STATUS_BLOCKED: + # Redaction/size handling must not make a valid block undeliverable. + _validate_block_details(safe_details) return { "timestamp": utc_now(), "event_type": event_type, @@ -131,7 +165,7 @@ def _event( "message": sanitize_text(message, limit=MAX_TELEMETRY_MESSAGE_CHARS) if message else None, - "details": sanitize_details(_mapping(details if details is not None else {})), + "details": safe_details, } @@ -201,6 +235,8 @@ def process(self, request: Mapping[str, Any]) -> None: ) elif command == COMMAND_FAIL: self.fail(request) + elif command == COMMAND_BLOCK: + self.block(request) elif command == COMMAND_COMPLETE: self.close_stage() self.add( @@ -221,7 +257,11 @@ def add( details: Mapping[str, Any] | None = None, ) -> None: self.events.append(_event(event_type, status, stage_id, message, details)) - if event_type == EVENT_TYPE_RUN and status in {STATUS_COMPLETED, STATUS_FAILED}: + if event_type == EVENT_TYPE_RUN and status in { + STATUS_COMPLETED, + STATUS_FAILED, + STATUS_BLOCKED, + }: self.state = replace(self.state, finished=True, transition_stage=None) elif stage_id is not None: self.state = replace(self.state, last_stage=stage_id) @@ -275,6 +315,32 @@ def fail(self, request: Mapping[str, Any]) -> None: ), "failed_during": failed_stage, } + error_code = _optional_text(request.get("error_code")) + if error_code is not None: + details["error_code"] = error_code message = _optional_text(request.get("message")) self.close_stage(STATUS_FAILED, message, details) self.add(EVENT_TYPE_RUN, STATUS_FAILED, STAGE_FAILED, message, details) + + def block(self, request: Mapping[str, Any]) -> None: + check_ids = request.get("blocking_gate_ids") + if not isinstance(check_ids, list): + raise ValueError("blocking_gate_ids must be a list") + count = request.get("blocking_failure_count") + details = { + "phase": request.get("phase"), + "blocking_gate_ids": check_ids, + "blocking_failure_count": len(check_ids) if count is None else count, + } + statuses = request.get("gate_statuses") + if statuses is not None: + details["gate_statuses"] = _mapping(statuses) + _validate_block_details(details) + self.close_stage() + self.add( + EVENT_TYPE_RUN, + STATUS_BLOCKED, + STAGE_BLOCKED, + BUILD_BLOCKED_MESSAGE.format(phase=details["phase"]), + details, + ) diff --git a/src/microcosm_emitter/telemetry/service/module.py b/src/microcosm_emitter/telemetry/service/module.py index d3b9998f..7a787353 100644 --- a/src/microcosm_emitter/telemetry/service/module.py +++ b/src/microcosm_emitter/telemetry/service/module.py @@ -4,16 +4,22 @@ import threading import time -from microcosm_emitter.host.contracts import JsonObject, ModuleContext -from microcosm_emitter.telemetry.protocol import UNEXPECTED_PROCESS_EXIT_MESSAGE -from microcosm_emitter.telemetry.requests import COMMAND_FAIL -from microcosm_emitter.telemetry.service.collector import CollectorDelivery -from microcosm_emitter.telemetry.service.constants import ( +from microcosm_emitter.core.auth import ( + _collector_origin, + _development_collector_url, + shared_session, +) +from microcosm_emitter.core.constants import ( DEFAULT_HEARTBEAT_SECONDS, DRAIN_RETRY_SECONDS, FAILURE_CLASS_UNEXPECTED_PROCESS_EXIT, MINIMUM_HEARTBEAT_SECONDS, + PRODUCTION_COLLECTOR_URL, ) +from microcosm_emitter.host.contracts import JsonObject, ModuleContext +from microcosm_emitter.telemetry.protocol import UNEXPECTED_PROCESS_EXIT_MESSAGE +from microcosm_emitter.telemetry.requests import COMMAND_FAIL +from microcosm_emitter.telemetry.service.collector import CollectorDelivery from microcosm_emitter.telemetry.service.lifecycle import ( LifecycleState, heartbeat_event, @@ -49,6 +55,14 @@ def initialize(self) -> None: try: self.delivery = CollectorDelivery( self.spool, + session=shared_session( + self.context.services, + _development_collector_url( + self.configuration["development_collector_url"] + ) + if self.configuration.get("development_collector_url") + else _collector_origin(PRODUCTION_COLLECTOR_URL), + ), development_collector_url=self.configuration.get( "development_collector_url" ), diff --git a/src/microcosm_emitter/telemetry/service/spool.py b/src/microcosm_emitter/telemetry/service/spool.py index a11ae681..fa47b000 100644 --- a/src/microcosm_emitter/telemetry/service/spool.py +++ b/src/microcosm_emitter/telemetry/service/spool.py @@ -13,8 +13,7 @@ from sqlalchemy import func, select from sqlalchemy.orm import Session -from microcosm_emitter.telemetry.protocol import TELEMETRY_SCHEMA_VERSION -from microcosm_emitter.telemetry.service.constants import ( +from microcosm_emitter.core.constants import ( BATCH_SIZE, MAX_QUEUED_BYTES, PRUNE_INTERVAL_SECONDS, @@ -22,18 +21,19 @@ UPLOAD_STATE_LOCAL_ONLY, UPLOAD_STATE_PENDING, ) -from microcosm_emitter.telemetry.service.database import ( +from microcosm_emitter.core.database import ( create_spool_engine, create_spool_session_factory, ) -from microcosm_emitter.telemetry.service.migrations import ( +from microcosm_emitter.core.migrations import ( upgrade_spool_database, ) -from microcosm_emitter.telemetry.service.models import ( +from microcosm_emitter.core.models import ( TelemetryEventRecord, TelemetryRunRecord, serialized_json_length, ) +from microcosm_emitter.telemetry.protocol import TELEMETRY_SCHEMA_VERSION from microcosm_emitter.telemetry.service.timestamps import utc_now diff --git a/tests/host/test_multiple_modules.py b/tests/host/test_multiple_modules.py new file mode 100644 index 00000000..5486c76e --- /dev/null +++ b/tests/host/test_multiple_modules.py @@ -0,0 +1,109 @@ +"""One local host routes named modules and gives each its own worker.""" + +import threading +import time + +from microcosm_emitter.host.client import SocketClient +from microcosm_emitter.host.runtime import ServiceRuntime + + +class Module: + def __init__(self): + self.messages = [] + self.entered = threading.Event() + self.release = threading.Event() + self.fail = False + + def initialize(self): + pass + + def handle_message(self, message): + self.messages.append(dict(message)) + + def query(self, message): + return {"result": message["value"]} + + def tick(self, now): + self.entered.set() + if self.fail: + raise ValueError("synthetic failure") + self.release.wait(2) + + def parent_exited(self): + pass + + def close(self, deadline): + pass + + +def test_slow_module_does_not_block_another_worker_or_socket(tmp_path): + # macOS needs short socket paths; reuse the runtime suite's fixture. + import tempfile + from pathlib import Path + + with tempfile.TemporaryDirectory(prefix="provider-test-", dir="/tmp") as directory: + path = Path(directory) / "e.sock" + slow, other = Module(), Module() + other.release.set() + runtime = ServiceRuntime( + path, + modules={"publication": slow, "telemetry": other}, + is_parent_alive=lambda: True, + tick_seconds=0.01, + ) + thread = threading.Thread(target=runtime.run) + thread.start() + try: + assert slow.entered.wait(2) + assert other.entered.wait(2) + client = SocketClient(path) + client.for_module("telemetry").send({"number": 1}) + assert other.messages == [{"number": 1}] + assert slow.messages == [] + client.ping() + assert client.for_module("publication").request({"value": "x" * 5000}) == { + "result": "x" * 5000 + } + finally: + slow.release.set() + runtime.stop() + thread.join(3) + assert not thread.is_alive() + + +def test_one_worker_failure_does_not_stop_another_module(tmp_path): + import tempfile + from pathlib import Path + + with tempfile.TemporaryDirectory(prefix="provider-test-", dir="/tmp") as directory: + path = Path(directory) / "e.sock" + broken, healthy = Module(), Module() + broken.fail = True + healthy.release.set() + runtime = ServiceRuntime( + path, + modules={"broken": broken, "healthy": healthy}, + is_parent_alive=lambda: True, + tick_seconds=0.01, + ) + errors = [] + + def run(): + try: + runtime.run() + except Exception as error: + errors.append(error) + + thread = threading.Thread(target=run) + thread.start() + try: + assert broken.entered.wait(2) + assert healthy.entered.wait(2) + time.sleep(0.05) + SocketClient(path).for_module("healthy").send({"accepted": True}) + assert healthy.messages == [{"accepted": True}] + finally: + runtime.stop() + thread.join(3) + assert not thread.is_alive() + assert errors diff --git a/tests/host/test_runtime.py b/tests/host/test_runtime.py index f02a52df..d849426e 100644 --- a/tests/host/test_runtime.py +++ b/tests/host/test_runtime.py @@ -228,3 +228,35 @@ def initialize(module): request.getfixturevalue("running") assert len(modules) == 1 assert modules[0].closed.is_set() + + +def test_query_timeout_override_preserves_short_acknowledgement_timeout(running): + module, client, errors = running + original_timeout = client.timeout + + def delayed_query(message): + time.sleep(original_timeout + 0.05) + return {"value": message["value"]} + + module.query = delayed_query + assert client.request({"value": "graph"}, module="default", timeout=1) == { + "value": "graph" + } + assert client.timeout == original_timeout + client.send({"ready": True}) + assert not errors + + +def test_query_timeout_rejects_unacknowledged_response(running): + module, client, errors = running + + def delayed_query(message): + time.sleep(0.1) + return {"receipt": {"status": "pending"}} + + module.query = delayed_query + with pytest.raises(LocalServiceError, match="TimeoutError"): + client.request({}, module="default", timeout=0.01) + time.sleep(0.15) + client.ping() + assert not errors diff --git a/tests/orrery/test_client.py b/tests/orrery/test_client.py new file mode 100644 index 00000000..62e1c769 --- /dev/null +++ b/tests/orrery/test_client.py @@ -0,0 +1,185 @@ +"""Graph adapters only request service-side work and never claim lost data is queued.""" + +import ast +from pathlib import Path +from unittest.mock import Mock + +import pytest +from sqlalchemy import func, select +from sqlalchemy.orm import Session + +from microcosm_emitter.core.auth import shared_session +from microcosm_emitter.core.database import create_spool_engine +from microcosm_emitter.core.models import TelemetryRunRecord +from microcosm_emitter.host.errors import LocalServiceError +from microcosm_emitter.host.launcher import start_services +from microcosm_emitter.orrery import client as client_module +from microcosm_emitter.orrery.client import GraphPublicationClient, publish_graph +from microcosm_emitter.orrery.service.queue import GraphPublicationQueue +from microcosm_emitter.telemetry.client import LocalTelemetryEmitter, TelemetryRun + + +def graph_files(tmp_path): + directory = tmp_path / "graph" + directory.mkdir() + (directory / "graph.orrery.json").write_text('{"nodes":[]}') + return directory + + +def test_public_client_enqueues_through_graph_only_service_without_run(tmp_path): + directory = graph_files(tmp_path) + path = tmp_path / "jobs.sqlite3" + client = GraphPublicationClient.start( + spool_path=path, + development_collector_url="http://127.0.0.1:1", + publication_origin="http://127.0.0.1:1", + ) + assert client.available + try: + inventory = client.inventory( + directory, + {"graph.orrery.json": "graph"}, + publication_id="independent-graph", + ) + assert inventory is not None and "run_id" not in inventory + receipt = publish_graph(directory, inventory, client=client, wait_seconds=0) + assert receipt["status"] == "pending" + assert receipt["publication_id"] == "independent-graph" + assert client.receipt("independent-graph")["status"] == "pending" + assert client.retry("independent-graph")["status"] == "pending" + finally: + handle = client._handle + client.close() + assert handle.process.wait(timeout=20) == 0 + queue = GraphPublicationQueue(path) + try: + assert queue.receipt("independent-graph")["status"] == "pending" + finally: + queue.close() + engine = create_spool_engine(path) + try: + with Session(engine) as session: + assert ( + session.scalar(select(func.count()).select_from(TelemetryRunRecord)) + == 0 + ) + finally: + engine.dispose() + + +def test_combined_host_keeps_graph_service_after_telemetry_completion(tmp_path): + path = tmp_path / "jobs.sqlite3" + run = TelemetryRun("combined", "UK", "test") + handle = start_services( + { + "telemetry": { + "registration": run.as_registration(), + "spool_path": str(path), + "development_collector_url": "http://127.0.0.1:1", + }, + "orrery": { + "spool_path": str(path), + "development_collector_url": "http://127.0.0.1:1", + "publication_origin": "http://127.0.0.1:1", + }, + } + ) + graph = GraphPublicationClient(handle.client.for_module("orrery")) + emitter = LocalTelemetryEmitter( + run=run, transport=handle.client.for_module("telemetry") + ) + try: + emitter.transition_stage("hf_download") + emitter.complete() + handle.client.ping() + directory = graph_files(tmp_path) + inventory = graph.inventory(directory, {"graph.orrery.json": "graph"}) + assert inventory is not None + assert graph.enqueue(directory, inventory)["status"] == "pending" + assert graph.receipt(inventory["publication_id"])["status"] == "pending" + finally: + graph.close() + handle.client.ping() # A module adapter cannot close the shared host. + handle.close() + assert handle.process.wait(timeout=20) == 0 + + +def test_graph_adapter_never_imports_persistence_or_auth(): + tree = ast.parse(Path(client_module.__file__).read_text()) + imports = [ + node.module or "" for node in ast.walk(tree) if isinstance(node, ast.ImportFrom) + ] + assert not any(".service" in name or ".core" in name for name in imports) + + +def test_startup_failure_returns_unavailable_without_direct_queue( + tmp_path, monkeypatch +): + path = tmp_path / "jobs.sqlite3" + monkeypatch.setattr( + client_module, "start_service", Mock(side_effect=LocalServiceError("synthetic")) + ) + client = GraphPublicationClient.start(spool_path=path) + assert not client.available + inventory = {"version": 1, "publication_id": "not-persisted", "files": []} + assert client.enqueue(tmp_path, inventory) == { + "version": 1, + "publication_id": "not-persisted", + "status": "failed", + "error_code": "local_service_unavailable", + } + assert client.inventory(tmp_path, {}) is None + assert not path.exists() + + +def test_request_failure_is_contained_without_claiming_pending(tmp_path): + transport = Mock() + transport.request.side_effect = OSError("synthetic private error") + client = GraphPublicationClient(transport) + inventory = {"version": 1, "publication_id": "not-acknowledged", "files": []} + receipt = publish_graph(tmp_path, inventory, client=client, wait_seconds=0) + assert receipt["status"] == "failed" + assert "synthetic" not in receipt["error_code"] + + +def test_graph_timeout_failure_does_not_claim_durable_enqueue(tmp_path): + from microcosm_emitter.orrery.constants import GRAPH_REQUEST_TIMEOUT_SECONDS + + transport = Mock() + transport.request.side_effect = LocalServiceError("TimeoutError") + client = GraphPublicationClient(transport) + inventory = {"version": 1, "publication_id": "timed-out", "files": []} + receipt = client.enqueue(tmp_path, inventory) + assert receipt["status"] == "failed" + assert receipt["error_code"] == "local_service_unavailable" + assert transport.request.call_args.kwargs == { + "timeout": GRAPH_REQUEST_TIMEOUT_SECONDS + } + + +def test_modules_share_one_session_and_reject_conflicting_origins(): + services = {} + first = shared_session(services, "https://collector.example") + assert shared_session(services, "https://collector.example") is first + with pytest.raises(ValueError, match="same collector origin"): + shared_session(services, "https://different.example") + + +def test_retry_waits_for_a_new_attempt_via_receipt_requests(monkeypatch): + pending = { + "version": 1, + "publication_id": "retry", + "status": "pending", + "error_code": "credential_missing", + "attempts": 1, + } + tried = {**pending, "attempts": 2} + transport = Mock() + transport.request.side_effect = [{"receipt": pending}, {"receipt": tried}] + monkeypatch.setattr(client_module.time, "sleep", lambda _: None) + client = GraphPublicationClient(transport) + assert client.retry("retry", wait_seconds=1) == tried + assert [call.args[0]["command"] for call in transport.request.call_args_list] == [ + "retry", + "receipt", + ] diff --git a/tests/orrery/test_publication.py b/tests/orrery/test_publication.py new file mode 100644 index 00000000..3ad74ef1 --- /dev/null +++ b/tests/orrery/test_publication.py @@ -0,0 +1,386 @@ +"""Durable graph jobs use fake HTTP, independently of the visualization app.""" + +import json +from dataclasses import FrozenInstanceError + +import httpx +import pytest + +from microcosm_emitter.orrery.contracts import ClaimedGraphPublication +from microcosm_emitter.orrery.service.queue import ( + GraphPublicationDelivery, + GraphPublicationQueue, + inventory_digest, + publication_inventory, +) + + +def source(tmp_path): + directory = tmp_path / "source" + directory.mkdir() + (directory / "graph.orrery.json").write_text('{"fixture":true}') + return directory, publication_inventory(directory, {"graph.orrery.json": "graph"}) + + +def test_jobs_survive_restart_without_run_registration_or_event_retention(tmp_path): + directory, inventory = source(tmp_path) + queue = GraphPublicationQueue(tmp_path / "jobs.sqlite3") + queue.enqueue(directory, inventory) + restarted = GraphPublicationQueue(queue.path) + job = restarted.claim() + assert isinstance(job, ClaimedGraphPublication) + assert job.inventory == inventory + assert "run_id" not in inventory + assert job.publication_id == inventory["publication_id"] + assert job.directory == directory.resolve() + assert restarted.claim() is None + + +def test_claim_is_a_frozen_record_and_expired_leases_can_be_reclaimed( + tmp_path, monkeypatch +): + from microcosm_emitter.orrery.service.queue import time + + directory, inventory = source(tmp_path) + queue = GraphPublicationQueue(tmp_path / "jobs.sqlite3") + queue.enqueue(directory, inventory) + clock = [1000.0] + monkeypatch.setattr(time, "time", lambda: clock[0]) + claimed = queue.claim(lease_seconds=5) + assert isinstance(claimed, ClaimedGraphPublication) + assert claimed.lease_until == 1005.0 + with pytest.raises(FrozenInstanceError): + claimed.lease_until = 0 + assert GraphPublicationQueue(queue.path).claim() is None + clock[0] = 1005.0 + replacement = GraphPublicationQueue(queue.path).claim() + assert replacement.publication_id == claimed.publication_id + assert replacement.lease_until > claimed.lease_until + + +def test_claim_respects_retry_delay_and_published_jobs_are_not_reclaimed( + tmp_path, monkeypatch +): + from microcosm_emitter.orrery.service.queue import time + + directory, inventory = source(tmp_path) + queue = GraphPublicationQueue(tmp_path / "jobs.sqlite3") + queue.enqueue(directory, inventory) + clock = [1000.0] + monkeypatch.setattr(time, "time", lambda: clock[0]) + claimed = queue.claim() + queue.finish( + claimed.publication_id, + error_code="publication_network_error", + expected_lease=claimed.lease_until, + ) + assert queue.claim() is None + clock[0] = 1002.0 + retried = queue.claim() + assert retried.publication_id == claimed.publication_id + queue.finish( + retried.publication_id, + url="https://runs.example/runs/published/", + expected_lease=retried.lease_until, + ) + assert queue.claim() is None + + +def test_queue_rejects_changed_bytes_and_conflicting_inventory(tmp_path): + directory, inventory = source(tmp_path) + queue = GraphPublicationQueue(tmp_path / "jobs.sqlite3") + (directory / "graph.orrery.json").write_text("changed") + with pytest.raises(ValueError, match="digest"): + queue.enqueue(directory, inventory) + with pytest.raises(ValueError): + publication_inventory(directory, {"private.h5": "graph"}) + + +def test_delivery_retries_missing_credentials_and_network_failure(tmp_path): + directory, inventory = source(tmp_path) + queue = GraphPublicationQueue(tmp_path / "jobs.sqlite3") + queue.enqueue(directory, inventory) + delivery = GraphPublicationDelivery( + queue, credential=lambda: ("missing", None), origin="https://runs.example" + ) + delivery.flush_once() + assert queue.receipt(inventory["publication_id"])["status"] == "pending" + queue.retry(inventory["publication_id"]) + client = httpx.Client( + transport=httpx.MockTransport( + lambda request: (_ for _ in ()).throw( + httpx.ConnectError("secret", request=request) + ) + ) + ) + GraphPublicationDelivery( + queue, + credential=lambda: ("ok", "session"), + origin="https://runs.example", + client=client, + ).flush_once() + receipt = queue.receipt(inventory["publication_id"]) + assert receipt["status"] == "pending" + assert "secret" not in json.dumps(receipt) + + +@pytest.mark.parametrize( + "upload_url", + [ + "https://store.blob.vercel-storage.com/upload?signed=credential", + "https://vercel.com/api/blob/?pathname=object&signed=credential", + ], +) +def test_delivery_uses_exact_bytes_and_does_not_forward_session_to_signed_upload( + tmp_path, + upload_url, +): + directory, inventory = source(tmp_path) + queue = GraphPublicationQueue(tmp_path / "jobs.sqlite3") + queue.enqueue(directory, inventory) + requests = [] + + def respond(request): + requests.append(request) + if request.url.path == "/api/uploads/": + return httpx.Response( + 200, + json={ + "uploads": [ + { + "name": "graph.orrery.json", + "url": upload_url, + "already_uploaded": False, + } + ] + }, + ) + if request.method == "PUT": + assert "authorization" not in request.headers + assert request.content == (directory / "graph.orrery.json").read_bytes() + return httpx.Response(200, json={}) + return httpx.Response( + 200, + json={ + "publication_id": inventory["publication_id"], + "inventory_sha256": inventory_digest(inventory), + }, + ) + + delivery = GraphPublicationDelivery( + queue, + credential=lambda: ("ok", "session"), + origin="https://runs.example", + client=httpx.Client(transport=httpx.MockTransport(respond)), + ) + delivery.flush_once() + receipt = queue.receipt(inventory["publication_id"]) + assert receipt["status"] == "published" + assert len(requests) == 3 + assert "signed=" not in json.dumps(receipt) + assert "session" not in json.dumps(receipt) + + +def test_queue_resumes_after_worker_exits_and_keeps_pending_jobs(tmp_path): + directory, inventory = source(tmp_path) + queue = GraphPublicationQueue(tmp_path / "jobs.sqlite3") + queue.enqueue(directory, inventory) + queue.claim(lease_seconds=0) + assert GraphPublicationQueue(queue.path).claim() is not None + + +def test_late_failed_attempt_cannot_replace_a_published_receipt(tmp_path): + directory, inventory = source(tmp_path) + queue = GraphPublicationQueue(tmp_path / "jobs.sqlite3") + queue.enqueue(directory, inventory) + queue.claim(lease_seconds=0) + replacement = GraphPublicationQueue(queue.path) + replacement.claim() + id_ = inventory["publication_id"] + replacement.finish(id_, url=f"https://runs.example/runs/{id_}/") + published = queue.receipt(id_) + queue.finish(id_, error_code="publication_network_error") + assert queue.receipt(id_) == published + assert json.loads((directory / "publication.status.json").read_bytes()) == published + + +def test_expired_worker_cannot_release_a_replacement_workers_lease(tmp_path): + directory, inventory = source(tmp_path) + queue = GraphPublicationQueue(tmp_path / "jobs.sqlite3") + queue.enqueue(directory, inventory) + expired = queue.claim(lease_seconds=0) + replacement = GraphPublicationQueue(queue.path) + current = replacement.claim() + id_ = inventory["publication_id"] + initial = queue.receipt(id_) + queue.finish( + id_, + error_code="publication_network_error", + expected_lease=expired.lease_until, + ) + assert queue.receipt(id_) == initial + assert queue.claim() is None + replacement.finish( + id_, + url=f"https://runs.example/runs/{id_}/", + expected_lease=current.lease_until, + ) + assert queue.receipt(id_)["status"] == "published" + + +def test_telemetry_pruning_cannot_remove_graph_jobs(tmp_path): + from microcosm_emitter.telemetry.service.spool import EventSpool + + directory, inventory = source(tmp_path) + queue = GraphPublicationQueue(tmp_path / "jobs.sqlite3") + queue.enqueue(directory, inventory) + EventSpool(queue.path).prune() + assert ( + GraphPublicationQueue(queue.path).receipt(inventory["publication_id"])["status"] + == "pending" + ) + + +def test_rejected_graph_session_is_refreshed_without_discarding_the_job(tmp_path): + directory, inventory = source(tmp_path) + queue = GraphPublicationQueue(tmp_path / "jobs.sqlite3") + queue.enqueue(directory, inventory) + invalidations = [] + delivery = GraphPublicationDelivery( + queue, + credential=lambda: ("ok", "expired"), + invalidate_credential=lambda: invalidations.append(True), + origin="https://runs.example", + client=httpx.Client( + transport=httpx.MockTransport(lambda request: httpx.Response(401)) + ), + ) + assert not delivery.flush_once() + assert invalidations == [True] + assert queue.receipt(inventory["publication_id"])["status"] == "pending" + + +def test_retry_cli_reuses_identity_and_leaves_hf_snapshot_unchanged( + tmp_path, monkeypatch +): + from microcosm_emitter.orrery import cli + + directory, inventory = source(tmp_path) + (directory / "orrery.upload.json").write_text(json.dumps(inventory)) + snapshot = directory / "orrery.publication.json" + snapshot.write_text('{"status":"pending"}') + calls = [] + + class RequestClient: + def enqueue(self, preserved_directory, supplied_inventory): + assert preserved_directory == directory + assert supplied_inventory == inventory + return { + "version": 1, + "publication_id": inventory["publication_id"], + "status": "pending", + "error_code": None, + } + + def retry(self, publication_id, *, wait_seconds): + calls.append(publication_id) + assert wait_seconds == 30 + return { + "version": 1, + "publication_id": publication_id, + "status": "published", + "error_code": None, + "url": "https://runs.example/runs/published/", + } + + def close(self): + pass + + monkeypatch.setattr( + cli.GraphPublicationClient, "start", lambda **kwargs: RequestClient() + ) + assert ( + cli.main( + [ + "--spool", + str(tmp_path / "jobs.sqlite3"), + "--publication-id", + inventory["publication_id"], + "--directory", + str(directory), + ] + ) + == 0 + ) + assert calls == [inventory["publication_id"]] + assert snapshot.read_text() == '{"status":"pending"}' + assert not (tmp_path / "jobs.sqlite3").exists() # No CLI-side database fallback. + + +def test_deleted_preserved_directory_is_retained_without_stopping_delivery( + tmp_path, monkeypatch +): + import shutil + + from microcosm_emitter.orrery.service.queue import time + + clock = [1000.0] + monkeypatch.setattr(time, "time", lambda: clock[0]) + queue = GraphPublicationQueue(tmp_path / "jobs.sqlite3") + (tmp_path / "removed").mkdir() + (tmp_path / "kept").mkdir() + removed, removed_inventory = source(tmp_path / "removed") + queue.enqueue(removed, removed_inventory) + clock[0] = 1001.0 + kept, kept_inventory = source(tmp_path / "kept") + queue.enqueue(kept, kept_inventory) + shutil.rmtree(removed) + delivery = GraphPublicationDelivery( + queue, credential=lambda: ("missing", None), origin="https://runs.example" + ) + + assert delivery.flush_once() is False + receipt = queue.receipt(removed_inventory["publication_id"]) + assert receipt["status"] == "pending" + assert receipt["error_code"] == "preserved_files_missing" + assert receipt["attempts"] == 1 + # The failed job backs off, so the next claim reaches the other job. + assert queue.claim().publication_id == kept_inventory["publication_id"] + + +def test_status_file_failure_does_not_roll_back_the_queue_result(tmp_path): + directory, inventory = source(tmp_path) + queue = GraphPublicationQueue(tmp_path / "jobs.sqlite3") + queue.enqueue(directory, inventory) + claimed = queue.claim() + (directory / "publication.status.json").mkdir() + queue.finish( + claimed.publication_id, + url="https://runs.example/runs/published/", + expected_lease=claimed.lease_until, + ) + assert queue.receipt(claimed.publication_id)["status"] == "published" + assert list(directory.glob("*.tmp")) == [] + + +def test_orrery_module_survives_a_local_queue_error(capsys): + from sqlalchemy.exc import OperationalError + + from microcosm_emitter.orrery.service.module import OrreryModule + + attempts = [] + + class Delivery: + def flush_once(self): + attempts.append(None) + if len(attempts) == 1: + raise OperationalError("UPDATE", {}, Exception("database is locked")) + return False + + module = OrreryModule({}, None) + module.delivery = Delivery() + module.tick(0.0) + module.tick(1.0) + module.tick(2.0) + assert len(attempts) == 3 + assert capsys.readouterr().err.count("local queue error") == 1 diff --git a/tests/telemetry/test_client.py b/tests/telemetry/test_client.py index 9eb30e90..f0624b53 100644 --- a/tests/telemetry/test_client.py +++ b/tests/telemetry/test_client.py @@ -147,3 +147,110 @@ def test_serialization_preserves_paths_and_handles_nonfinite_numbers(emitter): "path": "/tmp/test", "loss": None, } + + +def test_block_forwards_metadata_without_constructing_outcome(emitter): + emitter.block(phase="preflight", blocking_gate_ids=("uk_local_target_fit",)) + assert emitter._transport.requests == [ + { + "command": "block", + "phase": "preflight", + "blocking_gate_ids": ["uk_local_target_fit"], + "blocking_failure_count": None, + "gate_statuses": None, + } + ] + assert emitter._transport.closed and not emitter.available + + +def test_failure_code_is_forwarded_without_classifying_exception(emitter): + emitter.fail( + KeyboardInterrupt("operator"), + failed_during="compile", + failure_class="interrupted", + error_code="BUILD_INTERRUPTED", + ) + assert emitter._transport.requests[-1]["error_code"] == "BUILD_INTERRUPTED" + assert emitter._transport.requests[-1]["failure_class"] == "interrupted" + assert emitter._transport.requests[-1]["error_type"] == "KeyboardInterrupt" + assert emitter._transport.closed + + +def test_block_transport_failure_still_closes(emitter, monkeypatch): + def failed(request): + raise OSError("synthetic private error") + + monkeypatch.setattr(emitter._transport, "send", failed) + emitter.block(phase="preflight", blocking_gate_ids=["check"]) + assert emitter._transport.closed and not emitter.available + + +@pytest.mark.parametrize("outcome", ["complete", "fail", "block"]) +@pytest.mark.parametrize("deferred", [False, True]) +@pytest.mark.parametrize("send_fails", [False, True]) +def test_outcome_cleanup_can_be_deferred(outcome, deferred, send_fails, monkeypatch): + transport = RecordingTransport() + emitter = LocalTelemetryEmitter( + run=TelemetryRun("run", "UK", "test"), + transport=transport, + close_service_on_finish=not deferred, + ) + if send_fails: + + def failed(request): + raise OSError("synthetic send failure") + + monkeypatch.setattr(transport, "send", failed) + try: + if outcome == "fail": + emitter.fail(RuntimeError("synthetic"), error_code="BUILD_FAILED") + elif outcome == "block": + emitter.block(phase="preflight", blocking_gate_ids=["check"]) + else: + emitter.complete() + assert transport.closed is not deferred + assert emitter.available is deferred + finally: + emitter.close() + assert transport.closed and not emitter.available + + +@pytest.mark.parametrize("outcome", ["complete", "fail", "block"]) +def test_cleanup_failure_is_contained_for_every_outcome(outcome, monkeypatch): + transport = RecordingTransport() + emitter = LocalTelemetryEmitter( + run=TelemetryRun("run", "UK", "test"), + transport=transport, + ) + + def failed(): + raise OSError("synthetic cleanup failure") + + monkeypatch.setattr(transport, "close", failed) + if outcome == "fail": + emitter.fail(RuntimeError("synthetic")) + elif outcome == "block": + emitter.block(phase="preflight", blocking_gate_ids=["check"]) + else: + emitter.complete() + assert not emitter.available + + +def test_start_propagates_deferred_cleanup(monkeypatch, tmp_path): + from unittest.mock import Mock + + transport = RecordingTransport() + handle = Mock(client=transport) + monkeypatch.setattr(client, "start_service", Mock(return_value=handle)) + emitter = LocalTelemetryEmitter.start( + run=TelemetryRun("run", "UK", "test"), + spool_path=tmp_path / "spool", + close_service_on_finish=False, + ) + try: + emitter.complete() + handle.close.assert_not_called() + assert emitter.available + finally: + emitter.close() + handle.close.assert_called_once() diff --git a/tests/telemetry/test_delivery.py b/tests/telemetry/test_delivery.py index dd058a20..84986aa1 100644 --- a/tests/telemetry/test_delivery.py +++ b/tests/telemetry/test_delivery.py @@ -3,32 +3,35 @@ from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from types import SimpleNamespace +import pytest from alembic import command +from sqlalchemy.orm import Session -from microcosm_emitter.telemetry.client import ( - TelemetryRun, -) -from microcosm_emitter.telemetry.protocol import ( - MAX_TELEMETRY_DETAILS_BYTES, -) -from microcosm_emitter.telemetry.service import collector as collector_module -from microcosm_emitter.telemetry.service import resources as resources_module -from microcosm_emitter.telemetry.service import spool as spool_module -from microcosm_emitter.telemetry.service.collector import CollectorDelivery -from microcosm_emitter.telemetry.service.constants import ( +from microcosm_emitter.core.constants import ( PRODUCTION_COLLECTOR_URL, ) -from microcosm_emitter.telemetry.service.database import ( +from microcosm_emitter.core.database import ( create_spool_engine, ) -from microcosm_emitter.telemetry.service.migrations import ( +from microcosm_emitter.core.migrations import ( alembic_config, current_database_revision, migration_head_revision, ) -from microcosm_emitter.telemetry.service.models import ( +from microcosm_emitter.core.models import ( + TelemetryRunRecord, serialized_json_length, ) +from microcosm_emitter.telemetry.client import ( + TelemetryRun, +) +from microcosm_emitter.telemetry.protocol import ( + MAX_TELEMETRY_DETAILS_BYTES, +) +from microcosm_emitter.telemetry.service import collector as collector_module +from microcosm_emitter.telemetry.service import resources as resources_module +from microcosm_emitter.telemetry.service import spool as spool_module +from microcosm_emitter.telemetry.service.collector import CollectorDelivery from microcosm_emitter.telemetry.service.sanitization import ( sanitize_details, sanitize_json, @@ -315,6 +318,120 @@ def test_missing_credential_never_contacts_collector_and_stays_local_only( assert reopened.pending_runs() == [] +@pytest.mark.parametrize("endpoint", ["registration", "events"]) +@pytest.mark.parametrize("status", [400, 404, 413, 422]) +def test_permanent_collector_rejection_retains_only_rejected_producer( + tmp_path, + monkeypatch, + capsys, + endpoint, + status, +): + spool = EventSpool(tmp_path / "rejected.sqlite3") + bad = _registration() + good = {**bad, "producer_id": "producer-good"} + spool.register(bad) + spool.register(good) + rejected = spool.append(bad, _event()) + spool.append(good, _event()) + requests = [] + monkeypatch.setattr( + collector_module, "_huggingface_token", lambda: "hf_test_member" + ) + + def post(url, payload, token): + requests.append((url, payload)) + if url.endswith("/exchange"): + return 200, {"access_token": "synthetic", "expires_in": 3600} + if url.endswith("/v1/runs"): + if ( + endpoint == "registration" + and payload["producer_id"] == bad["producer_id"] + ): + return status, {} + return 201, {} + if ( + endpoint == "events" + and payload["events"][0]["producer_id"] == bad["producer_id"] + ): + return status, {} + return 202, {} + + monkeypatch.setattr(collector_module, "_http_post", post) + delivery = CollectorDelivery(spool, development_collector_url="http://127.0.0.1:1") + try: + assert delivery.flush_once() # The unrelated producer still uploads. + assert spool.pending_runs() == [] + assert spool.batch(bad["run_id"], bad["producer_id"]) == [rejected] + with Session(spool._engine) as session: + record = session.get( + TelemetryRunRecord, (bad["run_id"], bad["producer_id"]) + ) + assert record.upload_state == "local_only" + assert record.local_only_reason == ( + "collector_rejected_events" + if endpoint == "events" + else "run_registration_rejected" + ) + request_count = len(requests) + assert not delivery.flush_once() + assert len(requests) == request_count + warning = capsys.readouterr().err + assert warning.count("local-only for this run") == 1 + if endpoint == "events": + assert f"HTTP {status}" in warning and "will not be retried" in warning + finally: + spool.close() + + +@pytest.mark.parametrize("endpoint", ["registration", "events"]) +@pytest.mark.parametrize("status", [408, 429, 503, None]) +def test_transient_collector_rejection_remains_retryable( + tmp_path, + monkeypatch, + endpoint, + status, +): + spool = EventSpool(tmp_path / "retry.sqlite3") + registration = _registration() + spool.register(registration) + queued = spool.append(registration, _event()) + clock = [100.0] + attempts = [] + monkeypatch.setattr(collector_module.time, "monotonic", lambda: clock[0]) + monkeypatch.setattr( + collector_module, "_huggingface_token", lambda: "hf_test_member" + ) + + def post(url, payload, token): + if url.endswith("/exchange"): + return 200, {"access_token": "synthetic", "expires_in": 3600} + current_endpoint = "registration" if url.endswith("/v1/runs") else "events" + if current_endpoint == endpoint: + attempts.append(current_endpoint) + if len(attempts) == 1: + if status is None: + raise OSError("synthetic transient network failure") + return status, {} + return (201, {}) if current_endpoint == "registration" else (202, {}) + + monkeypatch.setattr(collector_module, "_http_post", post) + delivery = CollectorDelivery(spool, development_collector_url="http://127.0.0.1:1") + try: + assert not delivery.flush_once() + assert spool.pending_runs() == [registration] + assert spool.batch(registration["run_id"], registration["producer_id"]) == [ + queued + ] + assert not delivery.flush_once() # Respect the retry deadline. + assert len(attempts) == 1 + clock[0] += 100.0 + assert delivery.flush_once() + assert len(attempts) == 2 and not spool.has_pending() + finally: + spool.close() + + def test_resource_sampler_keeps_reaped_child_cpu_monotonic(monkeypatch) -> None: state = {"child_alive": True} diff --git a/tests/telemetry/test_module.py b/tests/telemetry/test_module.py index 268882a7..5c16d937 100644 --- a/tests/telemetry/test_module.py +++ b/tests/telemetry/test_module.py @@ -330,3 +330,148 @@ def test_invalid_stage_identifier_preserves_queue_and_lifecycle( module.handle_message({"command": "complete"}) assert [event["sequence"] for event in events(module)] == [1, 2, 3, 4] assert events(module)[-1]["status"] == "completed" + + +@pytest.mark.parametrize("active", [False, True]) +def test_block_closes_active_stage_and_finishes_without_failure(module, active): + if active: + module.handle_message( + {"command": "transition_stage", "stage_id": "hf_download"} + ) + module.handle_message( + { + "command": "block", + "phase": "preflight", + "blocking_gate_ids": ["uk_local_target_fit"], + "gate_statuses": {"uk_local_target_fit": "failed"}, + } + ) + recorded = events(module) + assert ( + recorded[-1]["event_type"], + recorded[-1]["stage_id"], + recorded[-1]["status"], + ) == ("run", "blocked", "blocked") + assert recorded[-1]["details"] == { + "phase": "preflight", + "blocking_gate_ids": ["uk_local_target_fit"], + "blocking_failure_count": 1, + "gate_statuses": {"uk_local_target_fit": "failed"}, + } + if active: + assert (recorded[-2]["stage_id"], recorded[-2]["status"]) == ( + "hf_download", + "completed", + ) + assert module.lifecycle.finished and module.lifecycle.transition_stage is None + module.parent_exited() + module.tick(time.monotonic() + 120) + module.handle_message({"command": "complete"}) + assert events(module) == recorded + + +def test_block_accepts_empty_ids_with_explicit_positive_count(module): + module.handle_message( + { + "command": "block", + "phase": "post_solve", + "blocking_gate_ids": [], + "blocking_failure_count": 3, + } + ) + assert events(module)[-1]["details"] == { + "phase": "post_solve", + "blocking_gate_ids": [], + "blocking_failure_count": 3, + } + + +@pytest.mark.parametrize( + "failure_class,error_code", + [ + ("error", "BUILD_FAILED"), + ("interrupted", "BUILD_INTERRUPTED"), + ("terminated", "BUILD_TERMINATED"), + ("oom", "BUILD_OUT_OF_MEMORY"), + ], +) +def test_failure_code_persists_on_stage_and_run(module, failure_class, error_code): + module.handle_message({"command": "transition_stage", "stage_id": "compile"}) + module.handle_message( + { + "command": "fail", + "error_type": "MemoryError", + "message": "Bearer hf_private_credentials", + "failure_class": failure_class, + "error_code": error_code, + } + ) + assert [event["details"]["error_code"] for event in events(module)[-2:]] == [ + error_code, + error_code, + ] + assert events(module)[-1]["details"]["failure_class"] == failure_class + assert events(module)[-1]["message"] == "[redacted]" + + +def test_block_commit_failure_preserves_stage_and_sequence(module): + module.handle_message({"command": "transition_stage", "stage_id": "compile"}) + recorded = events(module) + + def fail_commit(session): + session.flush() + raise OSError("simulated block commit failure") + + factory = module.spool._session_factory + sqlalchemy_event.listen(factory, "before_commit", fail_commit) + try: + with pytest.raises(OSError, match="block commit"): + module.handle_message( + { + "command": "block", + "phase": "preflight", + "blocking_gate_ids": ["check"], + } + ) + finally: + sqlalchemy_event.remove(factory, "before_commit", fail_commit) + assert events(module) == recorded and not module.lifecycle.finished + assert module.lifecycle.transition_stage == "compile" + module.handle_message( + {"command": "block", "phase": "preflight", "blocking_gate_ids": ["check"]} + ) + assert [event["sequence"] for event in events(module)] == [1, 2, 3, 4] + + +@pytest.mark.parametrize( + "details", + [ + {"phase": "bad phase"}, + {"phase": "hf_private_credentials"}, + {"blocking_failure_count": 0}, + {"blocking_failure_count": True}, + {"blocking_failure_count": "1"}, + {"blocking_gate_ids": [""]}, + {"blocking_gate_ids": ["x" * 201]}, + {"blocking_gate_ids": [1]}, + {"blocking_gate_ids": [], "blocking_failure_count": None}, + {"gate_statuses": {f"check-{number}": "x" * 2000 for number in range(10)}}, + ], +) +def test_invalid_block_cannot_poison_queue_or_lifecycle(module, details): + module.handle_message({"command": "transition_stage", "stage_id": "compile"}) + recorded, state = events(module), module.lifecycle + request = { + "command": "block", + "phase": "preflight", + "blocking_gate_ids": ["check"], + **details, + } + with pytest.raises((ValueError, TypeError)): + module.handle_message(request) + assert events(module) == recorded and module.lifecycle == state + + +def test_uncoded_failure_omits_optional_error_code(module): + module.handle_message({"command": "fail", "message": "failed"}) + assert "error_code" not in events(module)[-1]["details"] diff --git a/tests/telemetry/test_persistence.py b/tests/telemetry/test_persistence.py index ef8d3cf7..c8fdbe35 100644 --- a/tests/telemetry/test_persistence.py +++ b/tests/telemetry/test_persistence.py @@ -6,8 +6,8 @@ from alembic import command from sqlalchemy import inspect -from microcosm_emitter.telemetry.service.database import create_spool_engine -from microcosm_emitter.telemetry.service.migrations import ( +from microcosm_emitter.core.database import create_spool_engine +from microcosm_emitter.core.migrations import ( alembic_config, current_database_revision, ) @@ -37,7 +37,7 @@ def test_migrations_match_models_and_reopening_preserves_sequences(tmp_path): ] == [1, 2] assert spool.batch("fixture-run", "producer-a")[0] == first assert first["event_id"] != second["event_id"] - assert current_database_revision(path) == "20261007_01" + assert current_database_revision(path) == "20261008_02" with ( spool._engine.begin() as connection, alembic_config(connection=connection) as config, @@ -82,5 +82,6 @@ def test_multiple_producers_and_local_only_survive_reopening(tmp_path): "alembic_version", "telemetry_runs", "telemetry_events", + "graph_publication_jobs", } spool.close() diff --git a/tests/test_boundaries.py b/tests/test_boundaries.py index 9bada02c..cc575a9e 100644 --- a/tests/test_boundaries.py +++ b/tests/test_boundaries.py @@ -18,6 +18,8 @@ def imports(path): def test_generic_runtime_has_no_domain_or_database_imports(): forbidden = ( + "microcosm_emitter.core", + "microcosm_emitter.orrery", "microcosm_emitter.telemetry", "microcosm", "sqlalchemy", @@ -66,6 +68,7 @@ def audit(event, args): import microcosm_emitter.host import microcosm_emitter.telemetry import microcosm_emitter.telemetry.client +import microcosm_emitter.orrery.client assert not any(name in sys.modules for name in ( "sqlalchemy", "alembic", "huggingface_hub", "microcosm_emitter.telemetry.service", @@ -76,8 +79,27 @@ def audit(event, args): subprocess.run([sys.executable, "-c", script], check=True) +def test_graph_client_has_no_service_database_auth_or_http_imports(): + root = ROOT / "src/microcosm_emitter/orrery" + forbidden = ( + "microcosm_emitter.core", + "microcosm_emitter.orrery.service", + "sqlalchemy", + "alembic", + "huggingface_hub", + "httpx", + "microcosm", + ) + for path in (root / "client.py", root / "contracts.py"): + assert not any( + name == prefix or name.startswith(prefix + ".") + for name in imports(path) + for prefix in forbidden + ), path + + def test_database_code_has_no_raw_sql_or_manual_schema_creation(): - paths = list((ROOT / "src/microcosm_emitter/telemetry").rglob("*.py")) + paths = list((ROOT / "src/microcosm_emitter").rglob("*.py")) assert paths for path in paths: tree = ast.parse(path.read_text()) diff --git a/tests/test_collector_contract.py b/tests/test_collector_contract.py index b072828b..636f4abb 100644 --- a/tests/test_collector_contract.py +++ b/tests/test_collector_contract.py @@ -190,3 +190,128 @@ def lose_ack(url, *args, **kwargs): assert [entry["event_id"] for entry in response.json()["events"]] == [ event["event_id"] ] + + +@pytest.mark.parametrize("unnamed", [False, True]) +def test_blocked_run_projects_check_metadata_without_failure( + tmp_path, + monkeypatch, + live_collector, + unnamed, +): + address, reader = live_collector + monkeypatch.setenv("HF_TOKEN", "hf_test_member") + run_id = uuid.uuid4().hex + emitter = LocalTelemetryEmitter.start( + run=TelemetryRun(run_id, "UK", "contract"), + spool_path=tmp_path / "blocked.sqlite3", + development_collector_url=address, + ) + assert emitter.available + check_ids = [] if unnamed else ["uk_local_target_fit"] + count = 3 if unnamed else 1 + try: + emitter.transition_stage("hf_download") + emitter.block( + phase="preflight", + blocking_gate_ids=check_ids, + blocking_failure_count=count, + gate_statuses={ + "uk_local_target_fit": "failed", + "HF_TOKEN": "hf_private_check", + }, + ) + assert emitter._handle.process.wait(timeout=10) == 0 + finally: + emitter.close() + response = reader.get(f"/v1/runs/{run_id}") + assert response.status_code == 200, response.text + document = response.json() + assert document["progress"]["status"] == "blocked" + assert document["progress"]["current_stage"] == "blocked" + assert document["progress"]["failure"] is None + expected = { + "phase": "preflight", + "blocking_gate_ids": check_ids, + "blocking_failure_count": count, + } + assert document["progress"]["block"] == expected + assert document["run_manifest"]["block"] == expected + assert document["run_manifest"]["failure"] is None + assert [(event["stage_id"], event["status"]) for event in document["events"]] == [ + ("created", "started"), + ("hf_download", "started"), + ("hf_download", "completed"), + ("blocked", "blocked"), + ] + assert document["events"][-1]["details"] == { + **expected, + "gate_statuses": {"uk_local_target_fit": "failed", "HF_TOKEN": "[redacted]"}, + } + assert "hf_private_check" not in response.text + + +@pytest.mark.parametrize( + "failure_class,error_code,error", + [ + ("error", "BUILD_FAILED", RuntimeError("Bearer hf_private_error")), + ( + "interrupted", + "BUILD_INTERRUPTED", + KeyboardInterrupt("Bearer hf_private_error"), + ), + ("terminated", "BUILD_TERMINATED", RuntimeError("Bearer hf_private_error")), + ("oom", "BUILD_OUT_OF_MEMORY", MemoryError("Bearer hf_private_error")), + ], +) +def test_supplied_failure_codes_survive_collector_storage_and_documents( + tmp_path, + monkeypatch, + live_collector, + failure_class, + error_code, + error, +): + address, reader = live_collector + monkeypatch.setenv("HF_TOKEN", "hf_test_member") + run_id = uuid.uuid4().hex + emitter = LocalTelemetryEmitter.start( + run=TelemetryRun(run_id, "UK", "contract"), + spool_path=tmp_path / "failed.sqlite3", + development_collector_url=address, + ) + assert emitter.available + try: + emitter.transition_stage("hf_download") + emitter.fail( + error, + failed_during="hf_download", + failure_class=failure_class, + error_code=error_code, + ) + assert emitter._handle.process.wait(timeout=10) == 0 + finally: + emitter.close() + response = reader.get(f"/v1/runs/{run_id}") + assert response.status_code == 200, response.text + document = response.json() + # Generic details retain credential redaction; the wire stage identity does not. + expected = { + "error_code": error_code, + "error_type": type(error).__name__, + "failure_class": failure_class, + "failed_during": "[redacted]", + } + assert document["progress"]["status"] == "failed" + assert document["progress"]["current_stage"] == "failed" + assert document["progress"]["block"] is None + assert document["run_manifest"]["failure"] == document["progress"]["failure"] + assert document["progress"]["failure"] == {**expected, "message": "[redacted]"} + assert [ + (event["stage_id"], event["status"]) for event in document["events"][-2:] + ] == [ + ("hf_download", "failed"), + ("failed", "failed"), + ] + assert all(event["details"] == expected for event in document["events"][-2:]) + assert "hf_private_error" not in response.text diff --git a/tests/test_package_names.py b/tests/test_package_names.py index 3ceaaf26..14e41432 100644 --- a/tests/test_package_names.py +++ b/tests/test_package_names.py @@ -24,6 +24,7 @@ def test_single_distribution_with_required_telemetry_dependencies(): "sqlalchemy>=2,<3", "huggingface-hub>=0.20,<2", "psutil>=6,<8", + "httpx>=0.27,<1", } @@ -39,11 +40,13 @@ def test_installed_distribution_includes_host_telemetry_and_migrations(): modules = list(entry_points(group="microcosm.emitter.modules", name="telemetry")) assert len(modules) == 1 assert modules[0].value == "microcosm_emitter.telemetry.service:create_module" + graphs = list(entry_points(group="microcosm.emitter.modules", name="orrery")) + assert len(graphs) == 1 + assert graphs[0].value == "microcosm_emitter.orrery.service:create_module" + assert find_spec("microcosm_emitter.orrery.client") is not None assert find_spec("microcosm_emitter.host.__main__") is not None assert find_spec("microcosm_emitter.telemetry.client") is not None assert ( - find_spec( - "microcosm_emitter.telemetry.service.alembic.versions.v20261007_01_initial_spool" - ) + find_spec("microcosm_emitter.core.alembic.versions.v20261007_01_initial_spool") is not None ) diff --git a/tests/test_queue_compatibility.py b/tests/test_queue_compatibility.py index 413b7aa4..4bff66a1 100644 --- a/tests/test_queue_compatibility.py +++ b/tests/test_queue_compatibility.py @@ -8,9 +8,9 @@ from sqlalchemy import select from sqlalchemy.orm import Session +from microcosm_emitter.core.database import create_spool_engine +from microcosm_emitter.core.models import TelemetryEventRecord from microcosm_emitter.telemetry.service import spool as spool_module -from microcosm_emitter.telemetry.service.database import create_spool_engine -from microcosm_emitter.telemetry.service.models import TelemetryEventRecord from microcosm_emitter.telemetry.service.spool import EventSpool FIXTURES = Path(__file__).parent / "fixtures" diff --git a/tests/test_subprocess.py b/tests/test_subprocess.py index cebe8601..028ce38a 100644 --- a/tests/test_subprocess.py +++ b/tests/test_subprocess.py @@ -3,11 +3,14 @@ import json import threading from concurrent.futures import ThreadPoolExecutor +from contextlib import nullcontext from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer import psutil import pytest +from microcosm_emitter.host.launcher import start_services +from microcosm_emitter.orrery.client import GraphPublicationClient from microcosm_emitter.telemetry.client import LocalTelemetryEmitter, TelemetryRun from microcosm_emitter.telemetry.service.spool import EventSpool @@ -122,3 +125,83 @@ def build(number): ] == [1, 2, 3] finally: spool.close() + + +@pytest.mark.parametrize("outcome", ["complete", "fail", "block"]) +@pytest.mark.parametrize("export_fails", [False, True]) +def test_finished_telemetry_keeps_shared_graph_process_until_outer_cleanup( + tmp_path, + outcome, + export_fails, +): + path = tmp_path / "combined.sqlite3" + run = TelemetryRun("combined-outcome", "UK", "test") + handle = start_services( + { + "telemetry": { + "registration": run.as_registration(), + "spool_path": str(path), + "heartbeat_seconds": 0.001, + "development_collector_url": "http://127.0.0.1:1", + }, + "orrery": { + "spool_path": str(path), + "development_collector_url": "http://127.0.0.1:1", + "publication_origin": "http://127.0.0.1:1", + }, + } + ) + emitter = LocalTelemetryEmitter( + run=run, + transport=handle.client.for_module("telemetry"), + handle=handle, + close_service_on_finish=False, + ) + graph = GraphPublicationClient(handle.client.for_module("orrery")) + directory = tmp_path / "graph" + directory.mkdir() + (directory / "graph.orrery.json").write_text('{"nodes":[]}') + expectation = ( + pytest.raises(RuntimeError, match="synthetic export") + if export_fails + else nullcontext() + ) + with expectation: + try: + emitter.transition_stage("hf_download") + if outcome == "block": + emitter.block(phase="preflight", blocking_gate_ids=["check"]) + elif outcome == "fail": + emitter.fail(RuntimeError("synthetic"), error_code="BUILD_FAILED") + else: + emitter.complete() + assert emitter.available and handle.process.poll() is None + handle.client.ping() + inventory = graph.inventory(directory, {"graph.orrery.json": "graph"}) + assert inventory is not None + assert graph.enqueue(directory, inventory)["status"] == "pending" + assert graph.receipt(inventory["publication_id"])["status"] == "pending" + emitter.stage("after-finish") + if export_fails: + raise RuntimeError("synthetic export") + finally: + graph.close() + emitter.close() + emitter.close() + assert handle.process.wait(timeout=20) == 0 + spool = EventSpool(path) + try: + events = spool.batch(run.run_id, run.producer_id) + status = {"complete": "completed", "fail": "failed", "block": "blocked"}[ + outcome + ] + finished_index = next( + index + for index, event in enumerate(events) + if event["event_type"] == "run" and event["status"] == status + ) + assert finished_index == len(events) - 1 + assert events[-1]["status"] == status + assert not any(event["stage_id"] == "after-finish" for event in events) + finally: + spool.close() diff --git a/tests/test_workflows.py b/tests/test_workflows.py index 66becdc8..7e7adb73 100644 --- a/tests/test_workflows.py +++ b/tests/test_workflows.py @@ -50,6 +50,7 @@ def test_all_required_checks_include_real_collector_and_matrix(): "tests", "build-packages", "collector", + "reader", } assert jobs["tests"]["strategy"]["matrix"] == { "os": ["ubuntu-latest", "macos-latest"], @@ -66,6 +67,18 @@ def test_release_tag_matches_emitter_version(): validate("v9.9.9") +def test_reader_publishes_only_verified_tarball_after_python(): + reader = workflow("publish.yml")["jobs"]["publish-reader"] + assert reader["needs"] == ["verify", "publish"] + assert reader["environment"] == "pypi" + assert reader["permissions"]["id-token"] == "write" + download = next( + step for step in reader["steps"] if "download-artifact@" in step.get("uses", "") + ) + assert download["with"]["name"] == "orrery-reader" + assert reader["steps"][-1]["run"] == "bash tools/publish-reader.sh" + + def test_build_verifies_one_distribution_without_a_workspace(): script = (ROOT / "tools/check-artifacts.sh").read_text() assert script.count("uv build ") == 1 diff --git a/tools/check-collector.sh b/tools/check-collector.sh index a00b859e..8a47f773 100644 --- a/tools/check-collector.sh +++ b/tools/check-collector.sh @@ -2,7 +2,7 @@ set -euo pipefail : "${TEST_DATABASE_URL:?Set the disposable PostgreSQL test database URL}" -expected_commit=80a2a4ac7654f586b2b4b8d4c6ec7d2379616722 +expected_commit=280459c8d782658b3fb6098b27e9acba11af29d8 test "$(git -C .collector rev-parse HEAD)" = "$expected_commit" uv sync --locked uv pip install -e .collector/telemetry-service diff --git a/tools/check-reader-installed.ts b/tools/check-reader-installed.ts new file mode 100644 index 00000000..65296083 --- /dev/null +++ b/tools/check-reader-installed.ts @@ -0,0 +1,65 @@ +import { mkdtempSync, writeFileSync, readdirSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; + +const files = readdirSync("reader-dist").filter((name) => + name.endsWith(".tgz"), +); +if (files.length !== 1) throw new Error("Expected exactly one reader tarball."); +const workspace = mkdtempSync(join(tmpdir(), "provider-reader-")); +function run(args: string[]) { + const result = Bun.spawnSync(args, { + cwd: workspace, + stdout: "inherit", + stderr: "inherit", + }); + if (result.exitCode !== 0) throw new Error("Installed reader check failed."); +} +try { + writeFileSync( + join(workspace, "package.json"), + JSON.stringify({ + private: true, + type: "module", + dependencies: { + "@policyengine/microcosm-provider-orrery": resolve( + "reader-dist", + files[0], + ), + "@axiom-foundation/orrery": "0.6.1", + }, + }), + ); + run(["bun", "install", "--ignore-scripts"]); + writeFileSync( + join(workspace, "check.ts"), + ` +import { loadDocument } from "@policyengine/microcosm-provider-orrery/browser"; +import { createPublicationReader } from "@policyengine/microcosm-provider-orrery/server"; +const reader = createPublicationReader({read: async () => null}); +if (await reader.read("missing") !== null) throw new Error("Expected missing publication."); +try { + await loadDocument("missing", new AbortController().signal, async () => new Response("", {status: 404})); + throw new Error("Missing publication was accepted."); +} catch (error) { + if (!(error instanceof Error) || !error.message.includes("not found")) throw error; +} +`, + ); + run(["bun", "check.ts"]); + writeFileSync( + join(workspace, "browser.ts"), + 'export { loadDocument } from "@policyengine/microcosm-provider-orrery/browser";', + ); + run([ + "bun", + "build", + "browser.ts", + "--target", + "browser", + "--outfile", + "browser.js", + ]); +} finally { + rmSync(workspace, { recursive: true, force: true }); +} diff --git a/tools/check-reader.sh b/tools/check-reader.sh new file mode 100644 index 00000000..c438509b --- /dev/null +++ b/tools/check-reader.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash +set -euo pipefail +cd typescript/orrery +bun install --frozen-lockfile +bun run format:check +bun run typecheck +bun test +bun run build +bun pm pack --destination ../../reader-dist +cd ../.. +bun tools/check-reader-installed.ts diff --git a/tools/check-release.py b/tools/check-release.py index e709fd49..b915ee6d 100644 --- a/tools/check-release.py +++ b/tools/check-release.py @@ -1,5 +1,6 @@ """Refuse tags that do not describe the emitter distribution version.""" +import json import sys import tomllib from pathlib import Path @@ -11,6 +12,8 @@ def validate(tag: str) -> None: project = tomllib.loads((ROOT / "pyproject.toml").read_text())["project"] assert project["name"] == "microcosm-emitter" assert tag == f"v{project['version']}", "Release tag must match the emitter version" + reader = json.loads((ROOT / "typescript/orrery/package.json").read_text()) + assert tag == f"v{reader['version']}", "Release tag must match the reader version" if __name__ == "__main__": diff --git a/tools/installed-smoke.py b/tools/installed-smoke.py index 9442cdd9..a8863172 100644 --- a/tools/installed-smoke.py +++ b/tools/installed-smoke.py @@ -21,6 +21,11 @@ def host_smoke(): assert find_spec(name) is not None, name assert name not in sys.modules, name assert "microcosm_emitter.telemetry" not in sys.modules + from microcosm_emitter.orrery.client import GraphPublicationClient + + assert GraphPublicationClient + assert "microcosm_emitter.core" not in sys.modules + assert "microcosm_emitter.orrery.service" not in sys.modules def telemetry_smoke(): @@ -46,12 +51,12 @@ def telemetry_smoke(): assert emitter._handle.process.wait(timeout=10) == 0 finally: emitter.close() - from microcosm_emitter.telemetry.service.migrations import ( + from microcosm_emitter.core.migrations import ( current_database_revision, ) from microcosm_emitter.telemetry.service.spool import EventSpool - assert current_database_revision(path) == "20261007_01" + assert current_database_revision(path) == "20261008_02" spool = EventSpool(path) try: assert not spool.has_deliverable() @@ -62,6 +67,135 @@ def telemetry_smoke(): spool.close() +def graph_only_smoke(): + from microcosm_emitter.orrery.client import GraphPublicationClient, publish_graph + from microcosm_emitter.orrery.service.queue import GraphPublicationQueue + + path = Path.cwd() / "graph-only.sqlite3" + directory = Path.cwd() / "graph-only" + directory.mkdir() + (directory / "graph.orrery.json").write_text('{"synthetic":true}') + client = GraphPublicationClient.start( + spool_path=path, + development_collector_url="http://127.0.0.1:1", + publication_origin="http://127.0.0.1:1", + startup_timeout=10, + ) + assert client.available + try: + inventory = client.inventory(directory, {"graph.orrery.json": "graph"}) + assert inventory is not None and "run_id" not in inventory + receipt = publish_graph(directory, inventory, client=client, wait_seconds=0) + assert receipt["status"] == "pending" + finally: + handle = client._handle + client.close() + assert handle.process.wait(timeout=20) == 0 + queue = GraphPublicationQueue(path) + try: + assert queue.receipt(inventory["publication_id"])["status"] == "pending" + finally: + queue.close() + + +def combined_smoke(): + from microcosm_emitter.host.launcher import start_services + from microcosm_emitter.orrery.client import GraphPublicationClient, publish_graph + from microcosm_emitter.telemetry.client import LocalTelemetryEmitter, TelemetryRun + + path = Path.cwd() / "combined.sqlite3" + run = TelemetryRun("combined-test", "UK", "installed") + handle = start_services( + { + "telemetry": { + "registration": run.as_registration(), + "spool_path": str(path), + "development_collector_url": "http://127.0.0.1:1", + }, + "orrery": { + "spool_path": str(path), + "development_collector_url": "http://127.0.0.1:1", + "publication_origin": "http://127.0.0.1:1", + }, + }, + startup_timeout=10, + ) + client = GraphPublicationClient(handle.client.for_module("orrery")) + directory = Path.cwd() / "combined-graph" + directory.mkdir() + (directory / "graph.orrery.json").write_text('{"synthetic":true}') + try: + emitter = LocalTelemetryEmitter( + run=run, + transport=handle.client.for_module("telemetry"), + handle=handle, + close_service_on_finish=False, + ) + emitter.stage("hf_download") + emitter.block(phase="preflight", blocking_gate_ids=["check"]) + handle.client.ping() + inventory = client.inventory(directory, {"graph.orrery.json": "graph"}) + assert inventory is not None + assert ( + publish_graph(directory, inventory, client=client, wait_seconds=0)["status"] + == "pending" + ) + finally: + client.close() + emitter.close() + assert handle.process.wait(timeout=20) == 0 + from microcosm_emitter.telemetry.service.spool import EventSpool + + spool = EventSpool(path) + try: + assert spool.batch(run.run_id, run.producer_id)[-1]["status"] == "blocked" + finally: + spool.close() + + +def coded_failure_smoke(): + from microcosm_emitter.telemetry.client import LocalTelemetryEmitter, TelemetryRun + + run = TelemetryRun("failure-test", "UK", "installed") + path = Path.cwd() / "failure.sqlite3" + emitter = LocalTelemetryEmitter.start( + run=run, + spool_path=path, + development_collector_url="http://127.0.0.1:1", + startup_timeout_seconds=10, + ) + assert emitter.available + try: + emitter.transition_stage("hf_download") + emitter.fail( + MemoryError("synthetic"), + failed_during="hf_download", + failure_class="oom", + error_code="BUILD_OUT_OF_MEMORY", + ) + assert emitter._handle.process.wait(timeout=20) == 0 + finally: + emitter.close() + from microcosm_emitter.telemetry.service.spool import EventSpool + + spool = EventSpool(path) + try: + events = spool.batch(run.run_id, run.producer_id) + assert [(event["stage_id"], event["status"]) for event in events[-2:]] == [ + ("hf_download", "failed"), + ("failed", "failed"), + ] + assert all( + event["details"]["error_code"] == "BUILD_OUT_OF_MEMORY" + for event in events[-2:] + ) + finally: + spool.close() + + if __name__ == "__main__": host_smoke() telemetry_smoke() + graph_only_smoke() + combined_smoke() + coded_failure_smoke() diff --git a/tools/publish-reader.sh b/tools/publish-reader.sh new file mode 100644 index 00000000..a8978e46 --- /dev/null +++ b/tools/publish-reader.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +set -euo pipefail +node -e 'const [major, minor, patch] = require("child_process").execFileSync("npm", ["--version"], {encoding:"utf8"}).trim().split(".").map(Number); if (major < 11 || (major === 11 && (minor < 5 || (minor === 5 && patch < 1)))) throw new Error("npm OIDC requires npm >=11.5.1");' +shopt -s nullglob +archives=(reader-dist/*.tgz) +if (("${#archives[@]}" != 1)); then + echo "Expected exactly one verified reader tarball." >&2 + exit 1 +fi +npm publish --access public --provenance "${archives[0]}" diff --git a/typescript/orrery/README.md b/typescript/orrery/README.md new file mode 100644 index 00000000..e7830bc6 --- /dev/null +++ b/typescript/orrery/README.md @@ -0,0 +1,13 @@ +# Orrery publication reads + +The package owns publication validation, server-side metadata/evidence reads and +browser-side bounded downloads, SHA-256 verification and Orrery parsing. +It does not contain React components, routes, storage credentials or a hosted API. + +Import `createPublicationReader` from the `/server` entry point with a storage +adapter. Import `loadDocument` from `/browser` in the runs app. Browser exports +never import Node cryptography or server credentials. Existing publication URLs +and the Orrery React component remain unchanged. + +Build and test with Bun. Package publication and consumer activation are separate +operations; do not activate consumers before verifying the published tarball. diff --git a/typescript/orrery/bun.lock b/typescript/orrery/bun.lock new file mode 100644 index 00000000..58fb6c7a --- /dev/null +++ b/typescript/orrery/bun.lock @@ -0,0 +1,88 @@ +{ + "lockfileVersion": 2, + "configVersion": 1, + "workspaces": { + "": { + "name": "@policyengine/microcosm-provider-orrery", + "devDependencies": { + "@axiom-foundation/orrery": "0.6.1", + "@types/bun": "^1.3.0", + "@types/node": "^24.0.0", + "prettier": "^3.6.0", + "typescript": "^5.9.0", + }, + "peerDependencies": { + "@axiom-foundation/orrery": "0.6.1", + }, + }, + }, + "packages": { + "@axiom-foundation/orrery": ["@axiom-foundation/orrery@0.6.1", "", { "dependencies": { "@dagrejs/dagre": "3.1.1", "@xyflow/react": "12.11.6" }, "peerDependencies": { "react": ">=18 <20", "react-dom": ">=18 <20" }, "bin": { "orrery": "dist/cli.js", "graph-explorer": "dist/cli.js" } }, "sha512-VHYAuBZ50KhSXou1ogGG5IBjG1ET3Mnr05rIwUtoUgmgFu7eg5iH9rlsLQ28KQI/Lkd6yBsdYUsE3wo7kyCkhA=="], + + "@dagrejs/dagre": ["@dagrejs/dagre@3.1.1", "", { "dependencies": { "@dagrejs/graphlib": "4.0.5" } }, "sha512-zroZB1dFOFiGgv4Xcrn1DckB1o4aOikPqD2NDQPV0WM//CXGcS6xiD0rNkqHmw6FEg4tabt4nxPLwgCWT+Vb2A=="], + + "@dagrejs/graphlib": ["@dagrejs/graphlib@4.0.5", "", {}, "sha512-7xrBTqIts3o+PMUZX97wSc+7TUbW+/rULzGNCTP6yooNVDXbzw4Wutg/H/xOutTB/c/k0YqOAavgPh4/Zk9PFA=="], + + "@types/bun": ["@types/bun@1.4.2", "", { "dependencies": { "bun-types": "1.4.2" } }, "sha512-GimotNn7+ZV0uVArItBbriZsR1oNf0+WTzPkdcFrzShI7k2norL0uzEaJT8T33dWr7O/c9ZDuAFQrctKCi72oQ=="], + + "@types/d3-color": ["@types/d3-color@3.1.3", "", {}, "sha512-iO90scth9WAbmgv7ogoq57O9YpKmFBbmoEoCHDB2xMBY0+/KVrqAaCDyCE16dUspeOvIxFFRI+0sEtqDqy2b4A=="], + + "@types/d3-drag": ["@types/d3-drag@3.0.7", "", { "dependencies": { "@types/d3-selection": "*" } }, "sha512-HE3jVKlzU9AaMazNufooRJ5ZpWmLIoc90A37WU2JMmeq28w1FQqCZswHZ3xR+SuxYftzHq6WU6KJHvqxKzTxxQ=="], + + "@types/d3-interpolate": ["@types/d3-interpolate@3.0.4", "", { "dependencies": { "@types/d3-color": "*" } }, "sha512-mgLPETlrpVV1YRJIglr4Ez47g7Yxjl1lj7YKsiMCb27VJH9W8NVM6Bb9d8kkpG/uAQS5AmbA48q2IAolKKo1MA=="], + + "@types/d3-selection": ["@types/d3-selection@3.0.12", "", {}, "sha512-Qe/KWYhEiIIxGs7HrAAjMfShxKldx19SJtr5zu53f3afPsdZNz7HHtdTLXo/kqeiWNXVycI24kSnfzBYkTzpgw=="], + + "@types/d3-transition": ["@types/d3-transition@3.0.9", "", { "dependencies": { "@types/d3-selection": "*" } }, "sha512-uZS5shfxzO3rGlu0cC3bjmMFKsXv+SmZZcgp0KD22ts4uGXp5EVYGzu/0YdwZeKmddhcAccYtREJKkPfXkZuCg=="], + + "@types/d3-zoom": ["@types/d3-zoom@3.0.9", "", { "dependencies": { "@types/d3-interpolate": "*", "@types/d3-selection": "*" } }, "sha512-0sE1406XBYJGiqD3AusTl9ZqC//2mIXix51tbom25gDCA8ri4xnSZg28CaSE8Srl6FClqABUKfsn0qghgdepMA=="], + + "@types/node": ["@types/node@24.19.1", "", { "dependencies": { "undici-types": ">=7.24.0 <7.24.7" } }, "sha512-aS3/DG0oM05K0RIXXP+hKjinGG5IgSSVGzswZxW3O0sS3pH4/fycXundUC9XsszgKCk4gHXylTEK6hyFxVxnoQ=="], + + "@xyflow/react": ["@xyflow/react@12.11.6", "", { "dependencies": { "@xyflow/system": "0.0.82", "classcat": "^5.0.3", "zustand": "^4.4.0" }, "peerDependencies": { "@types/react": ">=17", "@types/react-dom": ">=17", "react": ">=17", "react-dom": ">=17" }, "optionalPeers": ["@types/react", "@types/react-dom"] }, "sha512-9XsEJNHjatKYndszKTF/bsU7FOP9dJ6V/EQwzy3oMdtqgBuUq7BjKSwkEo+C7s4qHstHQfwwoHA3E8QfpPxZZQ=="], + + "@xyflow/system": ["@xyflow/system@0.0.82", "", { "dependencies": { "@types/d3-drag": "^3.0.7", "@types/d3-interpolate": "^3.0.4", "@types/d3-selection": "^3.0.10", "@types/d3-transition": "^3.0.8", "@types/d3-zoom": "^3.0.8", "d3-drag": "^3.0.0", "d3-interpolate": "^3.0.1", "d3-selection": "^3.0.0", "d3-zoom": "^3.0.0" } }, "sha512-4DKnL3CGtCGLRSmgDqaajRVgeksMXq/Yw4wPfdMfm7JvdIiWHGzVYOfFUztiATwdBXZqUU6HhehwUdbV9G23PQ=="], + + "bun-types": ["bun-types@1.4.2", "", { "dependencies": { "@types/node": "*" } }, "sha512-bxV1FgK7yBIzjRe5zBozIM4Bem11ZJcCXSrjWRG3YWLt8yFDePu4cLjpebO8OvPeIE9trbyPF4fuj3Cia4Fj3w=="], + + "classcat": ["classcat@5.0.5", "", {}, "sha512-JhZUT7JFcQy/EzW605k/ktHtncoo9vnyW/2GspNYwFlN1C/WmjuV/xtS04e9SOkL2sTdw0VAZ2UGCcQ9lR6p6w=="], + + "d3-color": ["d3-color@3.1.0", "", {}, "sha512-zg/chbXyeBtMQ1LbD/WSoW2DpC3I0mpmPdW+ynRTj/x2DAWYrIY7qeZIHidozwV24m4iavr15lNwIwLxRmOxhA=="], + + "d3-dispatch": ["d3-dispatch@3.0.1", "", {}, "sha512-rzUyPU/S7rwUflMyLc1ETDeBj0NRuHKKAcvukozwhshr6g6c5d8zh4c2gQjY2bZ0dXeGLWc1PF174P2tVvKhfg=="], + + "d3-drag": ["d3-drag@3.0.0", "", { "dependencies": { "d3-dispatch": "1 - 3", "d3-selection": "3" } }, "sha512-pWbUJLdETVA8lQNJecMxoXfH6x+mO2UQo8rSmZ+QqxcbyA3hfeprFgIT//HW2nlHChWeIIMwS2Fq+gEARkhTkg=="], + + "d3-ease": ["d3-ease@3.0.1", "", {}, "sha512-wR/XK3D3XcLIZwpbvQwQ5fK+8Ykds1ip7A2Txe0yxncXSdq1L9skcG7blcedkOX+ZcgxGAmLX1FrRGbADwzi0w=="], + + "d3-interpolate": ["d3-interpolate@3.0.1", "", { "dependencies": { "d3-color": "1 - 3" } }, "sha512-3bYs1rOD33uo8aqJfKP3JWPAibgw8Zm2+L9vBKEHJ2Rg+viTR7o5Mmv5mZcieN+FRYaAOWX5SJATX6k1PWz72g=="], + + "d3-selection": ["d3-selection@3.0.0", "", {}, "sha512-fmTRWbNMmsmWq6xJV8D19U/gw/bwrHfNXxrIN+HfZgnzqTHp9jOmKMhsTUjXOJnZOdZY9Q28y4yebKzqDKlxlQ=="], + + "d3-timer": ["d3-timer@3.0.1", "", {}, "sha512-ndfJ/JxxMd3nw31uyKoY2naivF+r29V+Lc0svZxe1JvvIRmi8hUsrMvdOwgS1o6uBHmiz91geQ0ylPP0aj1VUA=="], + + "d3-transition": ["d3-transition@3.0.1", "", { "dependencies": { "d3-color": "1 - 3", "d3-dispatch": "1 - 3", "d3-ease": "1 - 3", "d3-interpolate": "1 - 3", "d3-timer": "1 - 3" }, "peerDependencies": { "d3-selection": "2 - 3" } }, "sha512-ApKvfjsSR6tg06xrL434C0WydLr7JewBB3V+/39RMHsaXTOG0zmt/OAXeng5M5LBm0ojmxJrpomQVZ1aPvBL4w=="], + + "d3-zoom": ["d3-zoom@3.0.0", "", { "dependencies": { "d3-dispatch": "1 - 3", "d3-drag": "2 - 3", "d3-interpolate": "1 - 3", "d3-selection": "2 - 3", "d3-transition": "2 - 3" } }, "sha512-b8AmV3kfQaqWAuacbPuNbL6vahnOJflOhexLzMMNLga62+/nh0JzvJ0aO/5a5MVgUFGS7Hu1P9P03o3fJkDCyw=="], + + "prettier": ["prettier@3.9.9", "", { "bin": { "prettier": "bin/prettier.cjs" } }, "sha512-Z/CJHIkdujO/OtN7nXUii0Rf3VT5SRuhjBA82Xvu2XhBUgX3nhP67T0LHceBdQLex7OOFGTox+Q5Yg8Jk2Qivg=="], + + "react": ["react@19.3.0", "", {}, "sha512-E8LUcbtBWt20bbl2YoHfx4ZDBdxVTfOKtCZn9cDSJ4l6/nuoApcpIBcj47t2wZoVX8g2ZHuMHbiShgCR1T5Sog=="], + + "react-dom": ["react-dom@19.3.0", "", { "dependencies": { "scheduler": "^0.28.0" }, "peerDependencies": { "react": "^19.3.0" } }, "sha512-JDk8dgif51OjFoDE70+OT9ICyYr+69HlmihNwp1+Nsfbna3t5sIiCa9ZJktDmQ4/1b/rn26hIAR2uYXDMr5r0Q=="], + + "scheduler": ["scheduler@0.28.0", "", {}, "sha512-juorfCmIkIw8tT+p5BXSm6PJjQF/ycEYmKyzURCIt/RaZIhL+PulbQ9Yu2z1HdOJDdqDTlxA1+xKBmHXJsczAw=="], + + "typescript": ["typescript@5.9.3", "", { "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" } }, "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw=="], + + "undici-types": ["undici-types@7.24.6", "", {}, "sha512-WRNW+sJgj5OBN4/0JpHFqtqzhpbnV0GuB+OozA9gCL7a993SmU+1JBZCzLNxYsbMfIeDL+lTsphD5jN5N+n0zg=="], + + "use-sync-external-store": ["use-sync-external-store@1.7.0", "", { "peerDependencies": { "react": "^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0" } }, "sha512-6L+EeigHMQhdaIPNIFUKwfWJSwWFQ8gJbJ2DLOs5sDIegTwR9fRxvnM3uciHKjIZhFz+KAv2emhWMRvDmMcY8A=="], + + "zustand": ["zustand@4.5.7", "", { "dependencies": { "use-sync-external-store": "^1.2.2" }, "peerDependencies": { "@types/react": ">=16.8", "immer": ">=9.0.6", "react": ">=16.8" }, "optionalPeers": ["@types/react", "immer", "react"] }, "sha512-CHOUy7mu3lbD6o6LJLfllpjkzhHXSBlX8B9+qPddUsIfeF5S/UZ5q0kmCsnRqT1UHFQZchNFDDzMbQsuesHWlw=="], + + "bun-types/@types/node": ["@types/node@26.6.4", "", { "dependencies": { "undici-types": "~8.9.0" } }, "sha512-ldVPDCzj7fsaGZrLB0NuHuTvJcsNasysBAqMolr/cgxrLd1xbqxIr3XJiPnHHJUCxj5sNF1vnRj9aWnrVh5Jcg=="], + + "bun-types/@types/node/undici-types": ["undici-types@8.9.0", "", {}, "sha512-KTDyRTYX8sWmKXAikPHHSyc63CRPETMctyjKFupcC6OBLXT3xsN0e9aF7m+mIXutFWpUXuedtowG7iLOzp0kQg=="], + } +} diff --git a/typescript/orrery/package.json b/typescript/orrery/package.json new file mode 100644 index 00000000..6d36c308 --- /dev/null +++ b/typescript/orrery/package.json @@ -0,0 +1,46 @@ +{ + "name": "@policyengine/microcosm-provider-orrery", + "version": "0.1.0", + "description": "Validated immutable publication reads for Orrery applications", + "type": "module", + "license": "MIT", + "repository": { + "type": "git", + "url": "git+https://github.com/PolicyEngine/microcosm-emitter.git", + "directory": "typescript/orrery" + }, + "files": [ + "dist", + "README.md" + ], + "exports": { + "./contract": { + "types": "./dist/contract.d.ts", + "import": "./dist/contract.js" + }, + "./browser": { + "types": "./dist/browser.d.ts", + "import": "./dist/browser.js" + }, + "./server": { + "types": "./dist/server.d.ts", + "import": "./dist/server.js" + } + }, + "scripts": { + "build": "tsc -p tsconfig.build.json", + "typecheck": "tsc --noEmit", + "test": "bun test", + "format:check": "prettier --check src tests *.json README.md" + }, + "peerDependencies": { + "@axiom-foundation/orrery": "0.6.1" + }, + "devDependencies": { + "@axiom-foundation/orrery": "0.6.1", + "@types/bun": "^1.3.0", + "@types/node": "^24.0.0", + "prettier": "^3.6.0", + "typescript": "^5.9.0" + } +} diff --git a/typescript/orrery/src/browser.ts b/typescript/orrery/src/browser.ts new file mode 100644 index 00000000..a42e98d6 --- /dev/null +++ b/typescript/orrery/src/browser.ts @@ -0,0 +1,98 @@ +import { parseGraphDocument } from "@axiom-foundation/orrery"; +import { + inventory, + publication, + publicationId, + MAX_FILE_BYTES, +} from "./contract.js"; + +export type Fetcher = ( + input: string | URL | Request, + init?: RequestInit, +) => Promise; + +async function boundedBytes( + response: Response, + maximum: number, +): Promise { + if (!response.body) throw new Error("The stored graph is unavailable."); + const reader = response.body.getReader(); + const chunks: Uint8Array[] = []; + let size = 0; + try { + while (true) { + const { value, done } = await reader.read(); + if (done) break; + size += value.byteLength; + if (size > maximum) + throw new Error("Response exceeds its publication size limit."); + chunks.push(value); + } + } finally { + await reader.cancel(); + } + const result = new Uint8Array(size); + let offset = 0; + for (const chunk of chunks) { + result.set(chunk, offset); + offset += chunk.byteLength; + } + return result; +} +async function digest(bytes: Uint8Array): Promise { + return Array.from( + new Uint8Array( + await crypto.subtle.digest("SHA-256", bytes as Uint8Array), + ), + (value) => value.toString(16).padStart(2, "0"), + ).join(""); +} +export async function loadDocument( + id: string, + signal: AbortSignal, + fetcher: Fetcher = fetch, +) { + publicationId(id); + const response = await fetcher(`/api/runs/${encodeURIComponent(id)}/`, { + signal, + redirect: "error", + credentials: "omit", + }); + if (response.status === 404) + throw new Error("This graph publication was not found."); + if (!response.ok) + throw new Error("Graph publication metadata is unavailable."); + const bytes = await boundedBytes(response, 256 * 1024); + const saved = publication( + JSON.parse(new TextDecoder("utf-8", { fatal: true }).decode(bytes)), + id, + ); + if ( + (await digest( + new TextEncoder().encode(JSON.stringify(inventory(saved))), + )) !== saved.inventory_sha256 + ) + throw new Error("Inventory digest does not match its publication."); + const file = saved.files.find((file) => file.role === "graph")!; + const graph = await fetcher(file.url, { + signal, + redirect: "error", + credentials: "omit", + }); + if (!graph.ok) throw new Error("The stored graph is unavailable."); + const payload = await boundedBytes( + graph, + Math.min(file.bytes, MAX_FILE_BYTES), + ); + if (payload.length !== file.bytes) + throw new Error("Graph size does not match its publication."); + const checksum = await digest(payload); + if (checksum !== file.sha256) + throw new Error("Graph digest does not match its publication."); + return { + document: parseGraphDocument( + JSON.parse(new TextDecoder("utf-8", { fatal: true }).decode(payload)), + ), + digest: checksum, + }; +} diff --git a/typescript/orrery/src/contract.ts b/typescript/orrery/src/contract.ts new file mode 100644 index 00000000..2774419b --- /dev/null +++ b/typescript/orrery/src/contract.ts @@ -0,0 +1,130 @@ +export const MAX_FILE_BYTES = 64 * 1024 * 1024; +export const MAX_TOTAL_BYTES = 512 * 1024 * 1024; +export const MAX_FILES = 256; +export type FileRole = + | "graph" + | "schema" + | "index" + | "declaration" + | "manifest" + | "binding" + | "summaries" + | "upstream"; +export interface RunFile { + name: string; + role: FileRole; + bytes: number; + sha256: string; +} +export interface Inventory { + version: 1; + publication_id: string; + files: RunFile[]; +} +export interface Publication extends Inventory { + inventory_sha256: string; + published_at: string; + files: (RunFile & { url: string })[]; +} +export class ApiError extends Error { + constructor( + public status: number, + public code: string, + ) { + super(code); + } +} +export function publicationId(value: unknown): string { + if ( + typeof value !== "string" || + !/^[A-Za-z0-9][A-Za-z0-9_-]{0,127}$/.test(value) + ) + throw new ApiError(400, "invalid_publication_id"); + return value; +} +const roleNames: Record = { + graph: /^graph\.orrery\.json$/, + schema: /^graph\.schema\.json$/, + index: /^execution\.evidence\.json$/, + declaration: /^evidence-[a-zA-Z0-9_-]+-graph\.json$/, + manifest: /^evidence-[a-zA-Z0-9_-]+-manifest\.json$/, + binding: /^evidence-[a-zA-Z0-9_-]+-binding\.json$/, + summaries: /^evidence-[a-zA-Z0-9_-]+-summaries\.json$/, + upstream: /^upstream-[a-f0-9]{64}\.json$/, +}; +export function inventory(raw: unknown): Inventory { + if (!raw || typeof raw !== "object") + throw new ApiError(400, "invalid_inventory"); + const data = raw as Record; + const id = publicationId(data.publication_id); + if ( + data.version !== 1 || + !Array.isArray(data.files) || + data.files.length === 0 || + data.files.length > MAX_FILES + ) + throw new ApiError(400, "invalid_inventory"); + const names = new Set(); + let total = 0; + const files: RunFile[] = data.files.map((raw) => { + if (!raw || typeof raw !== "object") + throw new ApiError(400, "invalid_file"); + const f = raw as RunFile; + if ( + !Object.hasOwn(roleNames, f.role) || + typeof f.name !== "string" || + !roleNames[f.role].test(f.name) || + names.has(f.name) || + !Number.isSafeInteger(f.bytes) || + f.bytes <= 0 || + f.bytes > MAX_FILE_BYTES || + typeof f.sha256 !== "string" || + !/^[a-f0-9]{64}$/.test(f.sha256) + ) + throw new ApiError(400, "invalid_file"); + names.add(f.name); + total += f.bytes; + return { name: f.name, role: f.role, bytes: f.bytes, sha256: f.sha256 }; + }); + if ( + total > MAX_TOTAL_BYTES || + files.filter((f) => f.role === "graph").length !== 1 + ) + throw new ApiError(400, "invalid_inventory"); + return { + version: 1, + publication_id: id, + files: files.sort((a, b) => + a.name < b.name ? -1 : a.name > b.name ? 1 : 0, + ), + }; +} + +export function publication(raw: unknown, expectedId: string): Publication { + const value = inventory(raw); + const saved = raw as Publication; + if ( + value.publication_id !== expectedId || + !/^[a-f0-9]{64}$/.test(saved.inventory_sha256 ?? "") || + typeof saved.published_at !== "string" || + !Number.isFinite(Date.parse(saved.published_at)) + ) + throw new ApiError(400, "invalid_publication"); + const urls = new Map(saved.files.map((file) => [file.name, file.url])); + return { + ...value, + inventory_sha256: saved.inventory_sha256, + published_at: saved.published_at, + files: value.files.map((file) => { + const url = new URL(urls.get(file.name)!); + if ( + url.protocol !== "https:" || + !url.hostname.endsWith(".public.blob.vercel-storage.com") || + url.username || + url.password + ) + throw new ApiError(400, "invalid_storage_url"); + return { ...file, url: url.toString() }; + }), + }; +} diff --git a/typescript/orrery/src/server.ts b/typescript/orrery/src/server.ts new file mode 100644 index 00000000..3dff69cf --- /dev/null +++ b/typescript/orrery/src/server.ts @@ -0,0 +1,52 @@ +import { createHash } from "node:crypto"; +import { + ApiError, + inventory, + publication, + publicationId, + type Inventory, +} from "./contract.js"; + +export function sha256(bytes: string | Uint8Array): string { + return createHash("sha256").update(bytes).digest("hex"); +} +export function inventoryDigest(value: Inventory): string { + return sha256(JSON.stringify(inventory(value))); +} +export function publicationPath(id: string): string { + return `publications/${publicationId(id)}.json`; +} +export function objectPath(value: Inventory, name: string): string { + return `objects/${value.publication_id}/${inventoryDigest(value)}/${name}`; +} +export interface PublicationStorage { + read( + path: string, + maximum: number, + ): Promise<{ bytes: Uint8Array; url: string } | null>; +} +export function createPublicationReader(storage: PublicationStorage) { + async function read(id: string) { + const object = await storage.read(publicationPath(id), 256 * 1024); + if (!object) return null; + if (object.bytes.byteLength > 256 * 1024) + throw new ApiError(400, "metadata_too_large"); + const value = publication( + JSON.parse( + new TextDecoder("utf-8", { fatal: true }).decode(object.bytes), + ), + id, + ); + if (inventoryDigest(value) !== value.inventory_sha256) + throw new ApiError(400, "inventory_digest_mismatch"); + return value; + } + return { + read, + async evidence(id: string, name: string) { + return ( + (await read(id))?.files.find((file) => file.name === name)?.url ?? null + ); + }, + }; +} diff --git a/typescript/orrery/tests/contract.test.ts b/typescript/orrery/tests/contract.test.ts new file mode 100644 index 00000000..eb3f19ae --- /dev/null +++ b/typescript/orrery/tests/contract.test.ts @@ -0,0 +1,43 @@ +import { describe, expect, test } from "bun:test"; +import { inventory, MAX_FILE_BYTES } from "../src/contract.js"; +import { inventoryDigest, objectPath } from "../src/server.js"; + +const graph = { + name: "graph.orrery.json", + role: "graph", + bytes: 100, + sha256: "a".repeat(64), +}; +const valid = { version: 1, publication_id: "uk-run-123", files: [graph] }; +describe("publication inventory", () => { + test("canonical order and content-addressed paths", () => { + const evidence = { + ...graph, + name: "execution.evidence.json", + role: "index", + }; + const a = inventory({ ...valid, files: [graph, evidence] }); + const b = inventory({ ...valid, files: [evidence, graph] }); + expect(inventoryDigest(a)).toBe(inventoryDigest(b)); + expect(objectPath(a, graph.name)).toBe( + `objects/uk-run-123/${inventoryDigest(a)}/graph.orrery.json`, + ); + }); + for (const value of [ + { ...valid, version: 2 }, + { ...valid, publication_id: "../bad" }, + { ...valid, files: [graph, graph] }, + { ...valid, files: [{ ...graph, name: "../graph.orrery.json" }] }, + { ...valid, files: [{ ...graph, bytes: MAX_FILE_BYTES + 1 }] }, + { ...valid, files: [{ ...graph, bytes: -1 }] }, + { ...valid, files: [{ ...graph, sha256: "wrong" }] }, + { ...valid, files: [{ ...graph, name: "private.h5", role: "manifest" }] }, + { + ...valid, + files: [{ ...graph, name: "unreviewed.json", role: "manifest" }], + }, + { ...valid, files: [] }, + ]) + test(`rejects ${JSON.stringify(value).slice(0, 100)}`, () => + expect(() => inventory(value)).toThrow()); +}); diff --git a/typescript/orrery/tests/reader.test.ts b/typescript/orrery/tests/reader.test.ts new file mode 100644 index 00000000..5bc02b59 --- /dev/null +++ b/typescript/orrery/tests/reader.test.ts @@ -0,0 +1,160 @@ +import { expect, test } from "bun:test"; +import { loadDocument, type Fetcher } from "../src/browser.js"; +import { + createPublicationReader, + inventoryDigest, + sha256, +} from "../src/server.js"; +import { inventory, type Publication } from "../src/contract.js"; + +const document = { + schemaVersion: "graph-explorer/v1", + id: "fixture", + title: "Fixture", + nodes: [], + edges: [], +}; +function fixture(payload = JSON.stringify(document)): Publication { + const value = inventory({ + version: 1, + publication_id: "graph", + files: [ + { + name: "graph.orrery.json", + role: "graph", + bytes: new TextEncoder().encode(payload).length, + sha256: sha256(payload), + }, + ], + }); + return { + ...value, + inventory_sha256: inventoryDigest(value), + published_at: "2026-10-08T00:00:00Z", + files: value.files.map((file) => ({ + ...file, + url: "https://example.public.blob.vercel-storage.com/graph.json", + })), + }; +} +function fetcher(saved: unknown, payload = JSON.stringify(document)): Fetcher { + let calls = 0; + return async (_url, init) => { + expect(init?.credentials).toBe("omit"); + expect(init?.redirect).toBe("error"); + return ++calls === 1 ? Response.json(saved) : new Response(payload); + }; +} +const signal = () => new AbortController().signal; +test("browser verifies metadata, exact bytes and graph parser", async () => { + const result = await loadDocument("graph", signal(), fetcher(fixture())); + expect(result.document.id).toBe("fixture"); + expect(result.digest).toBe(sha256(JSON.stringify(document))); +}); +test("missing publications return an explicit error", async () => { + await expect( + loadDocument( + "graph", + signal(), + async () => new Response("", { status: 404 }), + ), + ).rejects.toThrow("not found"); +}); +for (const field of [ + "publication_id", + "inventory_sha256", + "published_at", +] as const) { + test("rejects invalid " + field, async () => { + const saved = { ...fixture(), [field]: "invalid" }; + await expect( + loadDocument("graph", signal(), fetcher(saved)), + ).rejects.toThrow(); + }); +} +for (const url of [ + "http://example.public.blob.vercel-storage.com/x", + "https://example.com/x", + "https://user:secret@example.public.blob.vercel-storage.com/x", +]) { + test("rejects credential-bearing or non-storage URL " + url, async () => { + const saved = fixture(); + saved.files[0].url = url; + await expect( + loadDocument("graph", signal(), fetcher(saved)), + ).rejects.toThrow(); + }); +} +test("rejects modified graph bytes", async () => { + await expect( + loadDocument( + "graph", + signal(), + fetcher(fixture(), "x".repeat(JSON.stringify(document).length)), + ), + ).rejects.toThrow("digest"); +}); +test("bounds graph streaming before parsing", async () => { + await expect( + loadDocument( + "graph", + signal(), + fetcher(fixture(), JSON.stringify(document) + "extra"), + ), + ).rejects.toThrow("size limit"); +}); +test("bounds metadata streaming", async () => { + await expect( + loadDocument( + "graph", + signal(), + async () => new Response("x".repeat(256 * 1024 + 1)), + ), + ).rejects.toThrow("size limit"); +}); +test("rejects a valid but incorrect inventory digest", async () => { + await expect( + loadDocument( + "graph", + signal(), + fetcher({ ...fixture(), inventory_sha256: "a".repeat(64) }), + ), + ).rejects.toThrow("Inventory digest"); +}); +test("server reads only completed publications and listed evidence", async () => { + const saved = fixture(); + const reads: string[] = []; + const reader = createPublicationReader({ + async read(path, maximum) { + reads.push(path); + expect(maximum).toBe(256 * 1024); + return path === "publications/graph.json" + ? { + bytes: new TextEncoder().encode(JSON.stringify(saved)), + url: "unused", + } + : null; + }, + }); + expect(await reader.read("missing")).toBeNull(); + expect((await reader.read("graph"))?.publication_id).toBe("graph"); + expect(await reader.evidence("graph", "private.h5")).toBeNull(); + expect(await reader.evidence("graph", "graph.orrery.json")).toBe( + saved.files[0].url, + ); + expect(reads.every((path) => path.startsWith("publications/"))).toBe(true); +}); +test("server rejects corrupt stored metadata", async () => { + const saved = { ...fixture(), inventory_sha256: "a".repeat(64) }; + const reader = createPublicationReader({ + async read() { + return { + bytes: new TextEncoder().encode(JSON.stringify(saved)), + url: "unused", + }; + }, + }); + await expect(reader.read("graph")).rejects.toThrow( + "inventory_digest_mismatch", + ); +}); diff --git a/typescript/orrery/tsconfig.build.json b/typescript/orrery/tsconfig.build.json new file mode 100644 index 00000000..0f6eea2d --- /dev/null +++ b/typescript/orrery/tsconfig.build.json @@ -0,0 +1,11 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "noEmit": false, + "declaration": true, + "outDir": "dist", + "rootDir": "src" + }, + "include": ["src"], + "exclude": ["tests"] +} diff --git a/typescript/orrery/tsconfig.json b/typescript/orrery/tsconfig.json new file mode 100644 index 00000000..8c1f9cdc --- /dev/null +++ b/typescript/orrery/tsconfig.json @@ -0,0 +1,13 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "NodeNext", + "moduleResolution": "NodeNext", + "lib": ["ES2023", "DOM"], + "strict": true, + "skipLibCheck": true, + "types": ["bun"], + "noEmit": true + }, + "include": ["src", "tests"] +} diff --git a/uv.lock b/uv.lock index cf68aa06..df221dee 100644 --- a/uv.lock +++ b/uv.lock @@ -604,6 +604,7 @@ version = "0.1.0" source = { editable = "." } dependencies = [ { name = "alembic" }, + { name = "httpx" }, { name = "huggingface-hub" }, { name = "psutil" }, { name = "sqlalchemy" }, @@ -621,6 +622,7 @@ dev = [ [package.metadata] requires-dist = [ { name = "alembic", specifier = ">=1.13.3,<2" }, + { name = "httpx", specifier = ">=0.27,<1" }, { name = "huggingface-hub", specifier = ">=0.20,<2" }, { name = "psutil", specifier = ">=6,<8" }, { name = "sqlalchemy", specifier = ">=2,<3" },