diff --git a/src/analysis/post_run/logos_messaging.py b/src/analysis/post_run/logos_messaging.py index 009c4113..00279a9c 100644 --- a/src/analysis/post_run/logos_messaging.py +++ b/src/analysis/post_run/logos_messaging.py @@ -18,6 +18,7 @@ from src.analysis.mesh_analysis.readers.victoria_reader import VictoriaReader from src.analysis.metrics import scrape_utils from src.analysis.post_run.metrics import plot_run_metrics, scrape_run_metrics +from src.analysis.post_run.publish import publish_run_metrics from src.deployments.core.event_window_bridge import EventNotFound if TYPE_CHECKING: @@ -399,6 +400,7 @@ def scrape_and_plot(experiment: "LogosMessagingExperiment") -> None: label = f"{experiment.config.num_relay_nodes} nodes" dump = scrape_run_metrics(experiment.metadata, out / "metrics", name=label) plot_run_metrics({label: dump}, out / "plots", xlabel=experiment.namespace) + publish_run_metrics(experiment.metadata, out.name) def _safe(label: str, query, default): diff --git a/src/analysis/post_run/metrics.py b/src/analysis/post_run/metrics.py index 9d780d1b..7bfc637e 100644 --- a/src/analysis/post_run/metrics.py +++ b/src/analysis/post_run/metrics.py @@ -10,6 +10,7 @@ from typing import Dict, List, Optional from src.analysis.data.data_file_handler import DataPath +from src.analysis.metrics.config import ScrapeConfig from src.analysis.metrics.libp2p.scrape import Nimlibp2pScrapeBuilder from src.analysis.metrics.scrapper import Scrapper from src.analysis.plotting.config import DataGroup, PlotConfig @@ -47,14 +48,23 @@ """Templates: `name` is the figure's file stem, the run fills in groups and output path.""" +def run_scrape_config( + exp: dict, dump_location: Optional[Path] = None, name: Optional[str] = None +) -> ScrapeConfig: + """The standard metrics over one run's stable window.""" + builder = Nimlibp2pScrapeBuilder().with_exp(exp, extract_name=name is None) + if name is not None: + builder.name = name + if dump_location is not None: + builder.with_dump_location(str(dump_location)) + return builder.with_libp2p_metrics().build() + + def scrape_run_metrics( exp: dict, dump_location: Path, kube_config: Optional[str] = None, name: Optional[str] = None ) -> Path: """Dump the per-metric CSVs for one run's stable window.""" - builder = Nimlibp2pScrapeBuilder().with_exp(exp, extract_name=name is None) - if name is not None: - builder.name = name - config = builder.with_dump_location(str(dump_location)).with_libp2p_metrics().build() + config = run_scrape_config(exp, dump_location, name) logger.info(f"Scraping `{config.name}` metrics: {config.start} -> {config.end}") Scrapper(kube_config, config).query_and_dump_metrics() return dump_location diff --git a/src/analysis/post_run/nimlibp2p.py b/src/analysis/post_run/nimlibp2p.py index 43f10931..8c10266f 100644 --- a/src/analysis/post_run/nimlibp2p.py +++ b/src/analysis/post_run/nimlibp2p.py @@ -9,6 +9,7 @@ from src.analysis.plotting.latency_plotter import plot_dump_latency from src.analysis.post_run.delivery_cross_check import counter_deliveries, cross_check, report from src.analysis.post_run.metrics import plot_run_metrics, scrape_run_metrics +from src.analysis.post_run.publish import publish_run_metrics if TYPE_CHECKING: from src.deployments.experiments.libp2p.nimlibp2p import NimLibp2pExperiment @@ -133,3 +134,4 @@ def _scrape_and_plot(experiment: "NimLibp2pExperiment") -> None: experiment.output_folder / "plots", xlabel=f"{experiment.namespace}, {experiment.config.num_relay_nodes} nodes", ) + publish_run_metrics(exp, experiment.output_folder.name) diff --git a/src/analysis/post_run/publish.py b/src/analysis/post_run/publish.py new file mode 100644 index 00000000..a8f91937 --- /dev/null +++ b/src/analysis/post_run/publish.py @@ -0,0 +1,122 @@ +"""Publish a run's metrics to the DST dashboard, which keeps them past the lab's retention.""" + +import logging +import os +from datetime import datetime, timezone +from typing import List, Optional + +import requests + +from src.analysis.metrics.config import MetricToScrape, ScrapeConfig +from src.analysis.post_run.metrics import run_scrape_config + +logger = logging.getLogger(__name__) + +DASHBOARD_URL = "https://api.dashboard.lab.vac.dev" +TOKEN_ENV = "DST_DASHBOARD_TOKEN" +DATASOURCE = "victoria-metrics" +TIMEOUT_S = 600 + + +class PublishError(Exception): + pass + + +def _utc(dt: datetime) -> str: + return (dt.replace(tzinfo=timezone.utc) if dt.tzinfo is None else dt).isoformat() + + +def _dataset(metric: MetricToScrape, config: ScrapeConfig) -> dict: + labels = metric.extract_field.split("-") + labels += [label for label in ("container", "metrics_path") if getattr(metric, label)] + return { + "name": metric.folder_name.strip("/"), + "datasource": DATASOURCE, + "timeRange": {"start": _utc(config.start), "end": _utc(config.end)}, + "query": { + "expr": metric.query.replace("$__rate_interval", config.rate_interval), + "step": config.step, + }, + "schema": [{"name": "timestamp", "type": "datetime"}, {"name": "value", "type": "float"}] + + [{"name": label, "type": "string"} for label in labels], + } + + +def experiment_body(exp: dict, config: ScrapeConfig, run_name: str) -> dict: + """One dataset per scraped metric, over the same window and step as the local CSVs.""" + return { + "title": f"{run_name} {config.start:%Y-%m-%d %H:%M}", + "family": exp["experiment"]["name"], + "date": f"{config.start:%Y-%m-%d}", + "docker_image": exp.get("params", {}).get("image"), + "metadata": {"run": run_name, "experiment": exp["experiment"]["name"]} + | {key: exp[key] for key in ("params", "code", "stack", "results") if key in exp}, + "datasets": [_dataset(metric, config) for metric in config.metrics_to_scrape], + "panels": [], + "publish": False, + } + + +def _existing_id(title: str, url: str) -> str: + response = requests.get(f"{url}/experiments", timeout=TIMEOUT_S) + response.raise_for_status() + ids = [e["id"] for e in response.json() if e["title"] == title] + if not ids: + raise PublishError(f"`{title}` got a 409 but is not on the dashboard") + return ids[0] + + +def _empty_datasets(experiment_id: str, url: str) -> List[str]: + response = requests.get(f"{url}/experiments/{experiment_id}/datasets", timeout=TIMEOUT_S) + response.raise_for_status() + return [d["name"] for d in response.json()["datasets"] if not d.get("rowCount")] + + +def publish(body: dict, token: str, url: str = DASHBOARD_URL) -> str: + """Create the experiment, or find the one already there, and check every dataset has rows.""" + headers = {"Authorization": f"Bearer {token}"} + response = requests.post(f"{url}/experiments", json=body, headers=headers, timeout=TIMEOUT_S) + if response.status_code == 409: + experiment_id = _existing_id(body["title"], url) + else: + response.raise_for_status() + experiment_id = response.json()["id"] + + empty = _empty_datasets(experiment_id, url) + if empty: + requests.delete( + f"{url}/experiments/{experiment_id}", headers=headers, timeout=TIMEOUT_S + ).raise_for_status() + raise PublishError(f"`{body['title']}` got no rows for {empty}; removed it") + + logger.info(f"`{body['title']}` is on the dashboard as {experiment_id}") + return experiment_id + + +def unpublishable(exp: dict) -> Optional[str]: + """Why a run stays off the dashboard, or None if its 10ksim code was clean and pushed.""" + code = exp.get("code") + if not code: + return "no 10ksim commit was recorded" + if not code.get("clean"): + return f"10ksim had uncommitted changes on top of {code['commit'][:7]}" + if not code.get("pushed"): + return f"10ksim {code['commit'][:7]} was not pushed" + return None + + +def publish_run_metrics(exp: dict, run_name: str) -> Optional[str]: + """Publish what `scrape_run_metrics` scraped for this run. Logs failures, never raises.""" + reason = unpublishable(exp) + if reason: + logger.warning(f"`{run_name}` is not on the dashboard: {reason}") + return None + token = os.environ.get(TOKEN_ENV) + if not token: + logger.warning(f"{TOKEN_ENV} is not set; `{run_name}` is not on the dashboard") + return None + try: + return publish(experiment_body(exp, run_scrape_config(exp, name=run_name), run_name), token) + except Exception: + logger.exception(f"Publishing `{run_name}` to the dashboard failed") + return None diff --git a/src/analysis/post_run/tests/test_publish.py b/src/analysis/post_run/tests/test_publish.py new file mode 100644 index 00000000..18a25537 --- /dev/null +++ b/src/analysis/post_run/tests/test_publish.py @@ -0,0 +1,89 @@ +import pytest + +from src.analysis.post_run import publish as publish_module +from src.analysis.post_run.metrics import run_scrape_config +from src.analysis.post_run.publish import PublishError, experiment_body, publish + +EXP = { + "experiment": {"name": "nimlibp2p"}, + "params": {"muxer": "yamux", "image": "regression-node:v2.4.0"}, + "stack": {"namespace": "zerotesting"}, + "results": { + "complete": {"start": "2026-09-16T10:03:45", "end": "2026-09-16T10:26:32"}, + "stable": {"start": "2026-09-16T10:10:00", "end": "2026-09-16T10:16:30"}, + }, +} + + +class _Response: + def __init__(self, status_code, body=None): + self.status_code = status_code + self._body = body + + def json(self): + return self._body + + def raise_for_status(self): + if self.status_code >= 400: + raise RuntimeError(self.status_code) + + +def test_each_scraped_metric_is_a_dataset_over_the_stable_window(): + config = run_scrape_config(EXP, name="run") + body = experiment_body(EXP, config, "run") + + datasets = {d["name"]: d for d in body["datasets"]} + assert set(datasets) == {m.folder_name.strip("/") for m in config.metrics_to_scrape} + for dataset in datasets.values(): + assert dataset["timeRange"] == { + "start": "2026-09-16T10:10:00+00:00", + "end": "2026-09-16T10:16:30+00:00", + } + assert "$__rate_interval" not in dataset["query"]["expr"] + assert "[121s]" in datasets["libp2p-in"]["query"]["expr"] + # The scrape names CSV columns after these labels, so they must be kept. + open_streams = [field["name"] for field in datasets["libp2p-open-streams"]["schema"]] + assert {"instance", "type", "dir"} <= set(open_streams) + + +@pytest.mark.parametrize("created", [True, False]) +def test_a_dataset_without_rows_fails_the_publish_and_is_removed(monkeypatch, created): + deleted = [] + gets = { + "http://dash/experiments": [{"id": "e0", "title": "other"}, {"id": "e1", "title": "run"}], + "http://dash/experiments/e1/datasets": { + "datasets": [ + {"name": "libp2p-in", "rowCount": 7}, + {"name": "mesh-peers", "rowCount": 0}, + ] + }, + } + post = _Response(201, {"id": "e1"}) if created else _Response(409) + monkeypatch.setattr(publish_module.requests, "post", lambda *a, **k: post) + monkeypatch.setattr(publish_module.requests, "get", lambda url, **k: _Response(200, gets[url])) + monkeypatch.setattr( + publish_module.requests, "delete", lambda url, **k: deleted.append(url) or _Response(204) + ) + + with pytest.raises(PublishError, match="mesh-peers"): + publish({"title": "run"}, token="t", url="http://dash") + assert deleted == ["http://dash/experiments/e1"] + + +@pytest.mark.parametrize( + "code, published", + [ + (None, False), + ({"commit": "abc1234", "clean": False, "pushed": True}, False), + ({"commit": "abc1234", "clean": True, "pushed": False}, False), + ({"commit": "abc1234", "clean": True, "pushed": True}, True), + ], +) +def test_only_runs_from_clean_pushed_code_are_published(monkeypatch, code, published): + posted = [] + monkeypatch.setenv(publish_module.TOKEN_ENV, "t") + monkeypatch.setattr(publish_module, "publish", lambda body, token: posted.append(body)) + + publish_module.publish_run_metrics({**EXP, "code": code}, "run") + + assert bool(posted) is published diff --git a/src/analysis/publish_run.py b/src/analysis/publish_run.py new file mode 100644 index 00000000..eec091be --- /dev/null +++ b/src/analysis/publish_run.py @@ -0,0 +1,62 @@ +"""Publish finished runs' metrics to the DST dashboard from their folders.""" + +import argparse +import json +import logging +import os +import sys +from pathlib import Path + +from src.analysis.post_run.metrics import run_scrape_config +from src.analysis.post_run.publish import ( + DASHBOARD_URL, + TOKEN_ENV, + experiment_body, + publish, + unpublishable, +) +from src.analysis.utils.log_utils import init_logger + +logger = logging.getLogger(__name__) + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("runs", nargs="+", help="Run folders holding a metadata.json") + parser.add_argument("--dry-run", action="store_true", help="Print the experiments instead") + args = parser.parse_args() + init_logger(logging.getLogger(), verbosity=1) + token = os.environ.get(TOKEN_ENV) + if not token and not args.dry_run: + sys.exit(f"Set {TOKEN_ENV} to a token from {DASHBOARD_URL}/admin/token") + + failed = [] + for arg in args.runs: + run = Path(arg).resolve() + exp = json.loads((run / "metadata.json").read_text()) + # Same rule as the post-run scrape, so the dashboard holds what the CSVs hold. + if not exp.get("results", {}).get("stable", {}).get("duration"): + logger.error(f"`{run.name}` has no stable window; skipping it") + failed.append(run.name) + continue + reason = unpublishable(exp) + if reason: + logger.error(f"`{run.name}` is not published: {reason}") + failed.append(run.name) + continue + body = experiment_body(exp, run_scrape_config(exp, name=run.name), run.name) + if args.dry_run: + print(json.dumps(body, indent=2)) + continue + try: + publish(body, token) + except Exception: + logger.exception(f"Publishing `{run.name}` failed") + failed.append(run.name) + + if failed: + sys.exit(f"Not published: {failed}") + + +if __name__ == "__main__": + main() diff --git a/src/deployments/core/base_bridge.py b/src/deployments/core/base_bridge.py index 89cc9f2c..fa44b7ef 100644 --- a/src/deployments/core/base_bridge.py +++ b/src/deployments/core/base_bridge.py @@ -78,10 +78,9 @@ def get_metadata(self, events_log: Path) -> dict: except ValueError as e: logger.info(e) metadata["stack"]["name"] = self._get_name(metadata) - try: - metadata["params"] = all_metadata["params"] - except KeyError: - pass + for key in ("params", "code"): + if key in all_metadata: + metadata[key] = all_metadata[key] return dict(metadata) def _get_name(self, metadata: dict): diff --git a/src/deployments/core/code_version.py b/src/deployments/core/code_version.py new file mode 100644 index 00000000..aed4c595 --- /dev/null +++ b/src/deployments/core/code_version.py @@ -0,0 +1,33 @@ +"""The 10ksim commit a run deploys from.""" + +import logging +import subprocess +from pathlib import Path +from typing import Optional + +logger = logging.getLogger(__name__) + +REPO = Path(__file__).parent + + +def _git(repo: Path, *args: str) -> str: + result = subprocess.run( + ["git", "-C", str(repo), *args], check=True, capture_output=True, text=True + ) + return result.stdout.strip() + + +def code_version(repo: Path = REPO) -> Optional[dict]: + """Commit and branch of `repo`, and whether it was clean and pushed; None without git.""" + try: + commit = _git(repo, "rev-parse", "HEAD") + return { + "repo": _git(repo, "remote", "get-url", "origin"), + "commit": commit, + "branch": _git(repo, "rev-parse", "--abbrev-ref", "HEAD"), + "clean": not _git(repo, "status", "--porcelain", "--untracked-files=no"), + "pushed": bool(_git(repo, "branch", "--remotes", "--contains", commit)), + } + except (OSError, subprocess.CalledProcessError) as e: + logger.warning(f"Could not read the 10ksim commit: {e}") + return None diff --git a/src/deployments/core/tests/test_code_version.py b/src/deployments/core/tests/test_code_version.py new file mode 100644 index 00000000..66053813 --- /dev/null +++ b/src/deployments/core/tests/test_code_version.py @@ -0,0 +1,33 @@ +import subprocess + +from src.deployments.core.code_version import code_version + + +def _git(repo, *args): + subprocess.run( + ["git", "-C", str(repo), "-c", "user.name=t", "-c", "user.email=t@t"] + + ["-c", "commit.gpgsign=false", *args], + check=True, + capture_output=True, + ) + + +def test_records_whether_the_code_was_clean_and_pushed(tmp_path): + remote, repo = tmp_path / "remote.git", tmp_path / "repo" + _git(tmp_path, "init", "--bare", "-b", "main", str(remote)) + _git(tmp_path, "init", "-b", "main", str(repo)) + _git(repo, "remote", "add", "origin", str(remote)) + (repo / "a.py").write_text("x = 1\n") + _git(repo, "add", "a.py") + _git(repo, "commit", "-m", "a") + + assert code_version(repo)["pushed"] is False + + _git(repo, "push", "-u", "origin", "main") + (repo / "a.py").write_text("x = 2\n") + assert code_version(repo)["clean"] is False + + _git(repo, "checkout", "a.py") + (repo / "notes.txt").write_text("untracked\n") + code = code_version(repo) + assert (code["clean"], code["pushed"], code["branch"]) == (True, True, "main") diff --git a/src/deployments/experiments/base_experiment.py b/src/deployments/experiments/base_experiment.py index 992ac575..3e4c3ce5 100644 --- a/src/deployments/experiments/base_experiment.py +++ b/src/deployments/experiments/base_experiment.py @@ -36,6 +36,7 @@ from src.analysis.post_run_analysis import run_post_analysis from src.analysis.utils.log_utils import log_to_path from src.deployments.core.base_bridge import BaseBridge +from src.deployments.core.code_version import code_version from src.deployments.core.k8s_cleanup import ( get_cleanup, poll_namespace_has_objects, @@ -351,6 +352,7 @@ async def run(self, *, run_post_analysis: bool = True): self._deployed.clear() self._setup_log_paths() self._dump_initial_metadata() + self.log_metadata({"code": code_version()}) with log_to_path(self.out_log_path): with ExitStack() as self._stack: