Delegate local telemetry and graph publication to microcosm-emitter (deferred)

README.md

@@ -51,8 +51,14 @@ immutable publication before Hugging Face artifact staging, and preserves
 durable retry jobs and receipt snapshots. See
 [graph publication](docs/orrery-publication.md) for configuration and retry.
 
-Supported US and UK build commands always start the local telemetry emitter
-service. It reports live progress to the hosted collector when the operator's
+Supported US and UK build commands start one local emitter service with separate
+telemetry and graph-publication workers. Microcosm supplies build metadata and
+graph files; the required `microcosm-emitter` package owns authentication,
+durable queues and delivery. Telemetry and graph publication are internal
+libraries in that single package, not separate installations.
+This draft migration requires a published emitter release before merge; see the
+[migration prerequisites](docs/orrery-publication.md#deferred-migration-prerequisites).
+The telemetry worker reports live progress to the hosted collector when the operator's
 existing Hugging Face login is accepted; otherwise it retains the events
 locally and the build continues. This live event path is independent of the
 staging run files described below.

changelog.d/1152-publication-types.changed.md

@@ -0,0 +1 @@
+Use the emitter package's explicit publication contracts and pin the deferred adapter to the typed implementation.

changelog.d/1152.changed.md

@@ -0,0 +1 @@
+Delegate local telemetry and durable Orrery graph delivery to the single microcosm-emitter package while retaining UK graph generation and artifact staging in Microcosm.

changelog.d/1153-emitter-concurrency.fixed.md

@@ -0,0 +1 @@
+Use the emitter revision that prevents graph file requests from delaying telemetry acknowledgements and prevents older graph status writes from replacing newer results.

changelog.d/1153-outcome-compatibility.fixed.md

@@ -0,0 +1 @@
+Preserve validation-refused and classified-failure telemetry when using the external emitter, and keep its graph worker available until the build's final cleanup.

changelog.d/1153-single-emitter.fixed.md

@@ -0,0 +1 @@
+Use one microcosm-emitter dependency and its socket-based graph APIs, preserve public graph files when the service is unavailable, and create one run-start event with the build identity.

conftest.py

@@ -10,7 +10,6 @@
 import importlib.util
 import sys
 from pathlib import Path
-from urllib.parse import urlsplit
 
 import pytest
 
@@ -74,10 +73,15 @@ def pytest_collection_modifyitems(items) -> None:
 def fake_telemetry_emitters(monkeypatch, tmp_path):
     """Default every workspace test to in-memory telemetry, never production."""
     from huggingface_hub import constants as hf_constants
+    from microcosm_emitter.host import launcher
 
     from microcosm.build import telemetry_emitter
-    from microcosm.build.telemetry_emitter_service import collector
-    from test_support.microcosm_build.telemetry import FakeTelemetryEmitter
+    from microcosm.build.uk_runtime import orrery_publication
+    from test_support.microcosm_build.telemetry import (
+        FakeTelemetryEmitter,
+        fake_publication_inventory,
+        fake_publish_graph,
+    )
 
     for name in ("HF_TOKEN", "HUGGING_FACE_HUB_TOKEN", "HUGGINGFACE_TOKEN"):
         monkeypatch.delenv(name, raising=False)
@@ -97,13 +101,24 @@ def start(_cls, **kwargs):
         run = telemetry_emitter.TelemetryRun(
             **{name: value for name, value in kwargs.items() if name in fields}
         )
-        emitter = FakeTelemetryEmitter(run)
+        emitter = FakeTelemetryEmitter(
+            run, close_service_on_finish=kwargs.get("close_service_on_finish", True)
+        )
         emitters.append(emitter)
         return emitter
 
     monkeypatch.setattr(
         telemetry_emitter.LocalTelemetryEmitter, "start", classmethod(start)
     )
+    monkeypatch.setattr(
+        orrery_publication, "publication_inventory", fake_publication_inventory
+    )
+    monkeypatch.setattr(orrery_publication, "publish_graph", fake_publish_graph)
+
+    def unexpected_start(*args, **options):
+        pytest.fail("A build test tried to start a real emitter service")
+
+    monkeypatch.setattr(launcher, "start_services", unexpected_start)
     # Reset any cached handle left by a previously loaded build entrypoint.
     for module in tuple(sys.modules.values()):
         if isinstance(
@@ -112,38 +127,6 @@ def start(_cls, **kwargs):
         ):
             monkeypatch.setattr(module, "_ACTIVE_EMITTER", None)
 
-    real_post = collector._http_post
-
-    def local_post(url, *args, **kwargs):
-        parsed = urlsplit(url)
-        collector._development_collector_url(f"{parsed.scheme}://{parsed.netloc}")
-        return real_post(url, *args, **kwargs)
-
-    monkeypatch.setattr(collector, "_http_post", local_post)
     yield emitters
     for emitter in emitters:
         emitter.close()
-
-
-@pytest.fixture
-def real_local_telemetry(monkeypatch, fake_telemetry_emitters):
-    """Permit actual startup only with an explicit loopback collector."""
-    from microcosm.build import telemetry_emitter
-    from microcosm.build.telemetry_emitter_service.collector import (
-        _development_collector_url,
-    )
-
-    def start(cls, **kwargs):
-        _development_collector_url(kwargs.get("development_collector_url") or "")
-        return _REAL_TELEMETRY_START(cls, **kwargs)
-
-    monkeypatch.setattr(
-        telemetry_emitter.LocalTelemetryEmitter, "start", classmethod(start)
-    )
-
-
-# Save the underlying method before per-test patches so imported aliases and
-# class references all share the same default fake and explicit local opt-in.
-from microcosm.build.telemetry_emitter import LocalTelemetryEmitter  # noqa: E402
-
-_REAL_TELEMETRY_START = LocalTelemetryEmitter.start.__func__

docs/agent-guide.md

@@ -15,7 +15,7 @@ the PEP 420 namespace `microcosm.<x>`: `frame`, `fit`, `calibrate`, `build`,
 ## Commands
 
 UK build completion exports and stages Orrery graph evidence. Publication uses
-an independent worker inside the existing local service and a durable queue;
+an independent worker inside the external `microcosm-emitter` service and a durable queue;
 see [configuration and retry](orrery-publication.md). No new producer secret or
 environment variable is required. Keep graph export/publication tests independent
 of the hosted runs app and collector; use synthetic files and fake HTTP.
@@ -97,11 +97,17 @@ uses the same pinned parser for groups, statuses, activities and artifacts.
 These CI tests perform no browser rendering.
 
 Workspace tests use an in-memory telemetry emitter by default. The shared
-fixture isolates Hugging Face credentials and cache paths and blocks collector
-HTTP requests outside loopback. Tests of actual emitter startup must request
-`real_local_telemetry` and supply an explicit loopback development collector;
-that opt-in retains credential isolation. Other build tests can inspect
-`fake_telemetry_emitters` without creating sockets or service processes.
+fixture isolates Hugging Face credentials and cache paths. Adapter tests replace
+emitter startup, inventory and publication APIs with fakes. Emitter process, persistence
+and HTTP-delivery tests belong in microcosm-emitter; do not copy them
+into Microcosm. Build tests inspect `fake_telemetry_emitters` without creating
+sockets or service processes.
+
+The UK full-build caller sets `close_service_on_finish=False`: reporting an outcome
+leaves the graph worker available, and that caller's existing `finally` cleanup
+closes the combined service. Other callers close automatically when they finish.
+Keep validation-refused outcomes and structured
+failure codes intact when changing the adapter.
 
 New commits to a PR cancel older unfinished CI runs for that same PR.
 Each main-push run has a unique concurrency group, so all main-push runs

docs/orrery-publication.md

@@ -6,13 +6,19 @@ public-safe file inventory in the attempt directory, attempt publication before
 Hugging Face artifact staging, and register graph/evidence files and an
 `orrery.publication.json` receipt snapshot in the artifact manifest.
 
-The existing local service has two independent workers: live telemetry and
-completed graph publication. Slow uploads cannot delay socket acknowledgement
-or telemetry delivery. This change does not introduce another generic runtime,
-rename the telemetry package, or add compatibility wrappers.
+Microcosm's metadata adapter starts one local emitter service with two independent
+workers: live telemetry and completed graph publication. The `microcosm-emitter` package
+owns sockets, authentication, persistence, retries and resource sampling.
+Microcosm retains graph generation, public-file selection and HF staging.
+Slow uploads cannot delay socket acknowledgement or telemetry delivery.
 
-Publication IDs do not depend on telemetry run IDs. Jobs use SQLAlchemy and
-Alembic in the existing local SQLite database. Pending jobs survive missing or
+UK full-build commands explicitly defer process shutdown when recording a completed,
+validation-refused or failed telemetry outcome. Their final cleanup closes the combined
+process after graph inventory and enqueue requests, so failure evidence uses the same
+graph socket. Other callers close the process automatically when reporting their outcome.
+
+Publication IDs do not depend on telemetry run IDs. Jobs use SQLAlchemy ORM and
+Alembic inside the emitter service's existing local SQLite database. Pending jobs survive missing or
 rejected credentials, process restarts and telemetry event pruning. Retry
 leases prevent a stale worker from replacing another worker's result.
 The exact source bytes must remain available until delivery completes.
@@ -29,7 +35,7 @@ aggregate summaries can be published publicly.
 | Setting | Source and consumer | Provisioning |
 | --- | --- | --- |
 | `HF_TOKEN` / `HUGGINGFACE_TOKEN` | Existing operator credential, exchanged by the service for a short-lived collector session | Existing environment or Hugging Face login; never committed |
-| Collector origin | Tracked `PRODUCTION_COLLECTOR_URL` in the existing service | No producer variable or destination override |
+| Collector origin | Tracked `PRODUCTION_COLLECTOR_URL` in the emitter's internal core library | No producer variable or destination override |
 | Runs origin | Tracked `DEFAULT_GRAPH_PUBLICATION_ORIGIN` | No new producer variable |
 | `XDG_CACHE_HOME` | Optional existing cache-location setting | Defaults to the user's cache directory |
 
@@ -56,15 +62,33 @@ an upload or stage to Hugging Face.
 ## Retry
 
 ```bash
-microcosm-publish-graph --publication-id PUBLICATION_ID \
+microcosm-emitter-publish-graph --publication-id PUBLICATION_ID \
   --directory /path/to/attempt/orrery-publication
 ```
 
 The preserved directory contains `orrery.upload.json` and the exact files.
 Retries update a separate `publication.status.json`, never the original
 HF receipt snapshot. Pending jobs also resume when a subsequent local service
-starts. A job whose preserved directory has been removed stays pending with
-error code `preserved_files_missing` until its files are restored to the same
-directory; it does not stop delivery of other jobs. Moving this
-service into the `microcosm-emitter` package is a separate, deferred consumer
-change; this implementation does not require that package.
+starts. The retry command comes from the emitter package; Microcosm does not
+retain a second delivery implementation or a compatibility wrapper.
+
+## Deferred migration prerequisites
+
+This adapter is implemented and tested in a draft PR, not merged or activated.
+It depends on PolicyEngine/microcosm-emitter#4. The draft uses one exact root
+Git source pin in `packages/microcosm-build/pyproject.toml` and `uv.lock` for
+that PR's single `microcosm-emitter` distribution; no registry release was
+available when this draft was updated.
+Before merge, merge emitter #4, verify registry-owner configuration, publish
+one `microcosm-emitter` release containing those changes, replace the draft source
+override with that released version, regenerate `uv.lock`, update its
+approved worker-identity digest and rerun tests and wheel checks. Merely building
+a Microcosm wheel does not make its unpublished dependencies installable.
+
+The emitter rejects unversioned databases without modification. Before switching
+an existing workstation, upgrade any old unversioned queue using the preceding
+Microcosm implementation. Versioned telemetry and graph jobs keep their existing
+IDs, eligibility, path and Alembic revision; do not delete a queue to migrate.
+No new runtime variable, credential, hosted service or infrastructure is required
+by this migration. The previously required hosted authorization/API deployments
+remain prerequisites for remote publication.

packages/microcosm-build/pyproject.toml

@@ -7,12 +7,10 @@ requires-python = ">=3.13"
 license = { text = "MIT" }
 authors = [{ name = "PolicyEngine" }]
 dependencies = [
-    "alembic>=1.13.3,<2",
     "numpy>=1.26",
     "pandas>=2",
     "scipy>=1.13",
     "huggingface-hub>=0.20",
-    "httpx>=0.27,<1",
     "microcosm-frame>=0.1,<0.2",
     "microcosm-graph>=0.1,<0.2",
     "microcosm-fit>=0.1,<0.2",
@@ -26,7 +24,7 @@ dependencies = [
     "pyyaml>=6",
     "jsonschema>=4.23,<5",
     "referencing>=0.35,<1",
-    "sqlalchemy>=2,<3",
+    "microcosm-emitter==0.1.0",
 ]
 
 [project.optional-dependencies]
@@ -51,7 +49,6 @@ us = [
 uk = ["policyengine-uk>=2.123.0", "microcosm-data>=0.1,<0.2", "h5py>=3", "tables>=3", "openpyxl>=3.1"]
 
 [project.scripts]
-microcosm-publish-graph = "microcosm.build.graph_publication_cli:main"
 microcosm-build-uk = "microcosm.build.uk_runtime.full_build_cli:main"
 microcosm-export-us-l0-refit-h5 = "microcosm.build.us_runtime.l0_refit_export:main"
 
@@ -84,3 +81,5 @@ microcosm-fit = { workspace = true }
 microcosm-calibrate = { workspace = true }
 microcosm-diagnostics = { workspace = true }
 microcosm-data = { workspace = true }
+# Draft-only root source pin. Replace with a published distribution before merge.
+microcosm-emitter = { git = "https://github.com/PolicyEngine/microcosm-emitter", rev = "12624a118db255405a0ce7397d53b3181deaf49a" }