Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions src/analysis/post_run/logos_messaging.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
from src.analysis.mesh_analysis.readers.victoria_reader import VictoriaReader
from src.analysis.metrics import scrape_utils
from src.analysis.post_run.metrics import plot_run_metrics, scrape_run_metrics
from src.analysis.post_run.publish import publish_run_metrics
from src.deployments.core.event_window_bridge import EventNotFound

if TYPE_CHECKING:
Expand Down Expand Up @@ -399,6 +400,7 @@ def scrape_and_plot(experiment: "LogosMessagingExperiment") -> None:
label = f"{experiment.config.num_relay_nodes} nodes"
dump = scrape_run_metrics(experiment.metadata, out / "metrics", name=label)
plot_run_metrics({label: dump}, out / "plots", xlabel=experiment.namespace)
publish_run_metrics(experiment.metadata, out.name)


def _safe(label: str, query, default):
Expand Down
18 changes: 14 additions & 4 deletions src/analysis/post_run/metrics.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
from typing import Dict, List, Optional

from src.analysis.data.data_file_handler import DataPath
from src.analysis.metrics.config import ScrapeConfig
from src.analysis.metrics.libp2p.scrape import Nimlibp2pScrapeBuilder
from src.analysis.metrics.scrapper import Scrapper
from src.analysis.plotting.config import DataGroup, PlotConfig
Expand Down Expand Up @@ -47,14 +48,23 @@
"""Templates: `name` is the figure's file stem, the run fills in groups and output path."""


def run_scrape_config(
exp: dict, dump_location: Optional[Path] = None, name: Optional[str] = None
) -> ScrapeConfig:
"""The standard metrics over one run's stable window."""
builder = Nimlibp2pScrapeBuilder().with_exp(exp, extract_name=name is None)
if name is not None:
builder.name = name
if dump_location is not None:
builder.with_dump_location(str(dump_location))
return builder.with_libp2p_metrics().build()


def scrape_run_metrics(
exp: dict, dump_location: Path, kube_config: Optional[str] = None, name: Optional[str] = None
) -> Path:
"""Dump the per-metric CSVs for one run's stable window."""
builder = Nimlibp2pScrapeBuilder().with_exp(exp, extract_name=name is None)
if name is not None:
builder.name = name
config = builder.with_dump_location(str(dump_location)).with_libp2p_metrics().build()
config = run_scrape_config(exp, dump_location, name)
logger.info(f"Scraping `{config.name}` metrics: {config.start} -> {config.end}")
Scrapper(kube_config, config).query_and_dump_metrics()
return dump_location
Expand Down
2 changes: 2 additions & 0 deletions src/analysis/post_run/nimlibp2p.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
from src.analysis.plotting.latency_plotter import plot_dump_latency
from src.analysis.post_run.delivery_cross_check import counter_deliveries, cross_check, report
from src.analysis.post_run.metrics import plot_run_metrics, scrape_run_metrics
from src.analysis.post_run.publish import publish_run_metrics

if TYPE_CHECKING:
from src.deployments.experiments.libp2p.nimlibp2p import NimLibp2pExperiment
Expand Down Expand Up @@ -133,3 +134,4 @@ def _scrape_and_plot(experiment: "NimLibp2pExperiment") -> None:
experiment.output_folder / "plots",
xlabel=f"{experiment.namespace}, {experiment.config.num_relay_nodes} nodes",
)
publish_run_metrics(exp, experiment.output_folder.name)
122 changes: 122 additions & 0 deletions src/analysis/post_run/publish.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,122 @@
"""Publish a run's metrics to the DST dashboard, which keeps them past the lab's retention."""

import logging
import os
from datetime import datetime, timezone
from typing import List, Optional

import requests

from src.analysis.metrics.config import MetricToScrape, ScrapeConfig
from src.analysis.post_run.metrics import run_scrape_config

logger = logging.getLogger(__name__)

DASHBOARD_URL = "https://api.dashboard.lab.vac.dev"
TOKEN_ENV = "DST_DASHBOARD_TOKEN"
DATASOURCE = "victoria-metrics"
TIMEOUT_S = 600


class PublishError(Exception):
pass


def _utc(dt: datetime) -> str:
return (dt.replace(tzinfo=timezone.utc) if dt.tzinfo is None else dt).isoformat()


def _dataset(metric: MetricToScrape, config: ScrapeConfig) -> dict:
labels = metric.extract_field.split("-")
labels += [label for label in ("container", "metrics_path") if getattr(metric, label)]
return {
"name": metric.folder_name.strip("/"),
"datasource": DATASOURCE,
"timeRange": {"start": _utc(config.start), "end": _utc(config.end)},
"query": {
"expr": metric.query.replace("$__rate_interval", config.rate_interval),
"step": config.step,
},
"schema": [{"name": "timestamp", "type": "datetime"}, {"name": "value", "type": "float"}]
+ [{"name": label, "type": "string"} for label in labels],
}


def experiment_body(exp: dict, config: ScrapeConfig, run_name: str) -> dict:
"""One dataset per scraped metric, over the same window and step as the local CSVs."""
return {
"title": f"{run_name} {config.start:%Y-%m-%d %H:%M}",
"family": exp["experiment"]["name"],
"date": f"{config.start:%Y-%m-%d}",
"docker_image": exp.get("params", {}).get("image"),
"metadata": {"run": run_name, "experiment": exp["experiment"]["name"]}
| {key: exp[key] for key in ("params", "code", "stack", "results") if key in exp},
"datasets": [_dataset(metric, config) for metric in config.metrics_to_scrape],
"panels": [],
"publish": False,
}


def _existing_id(title: str, url: str) -> str:
response = requests.get(f"{url}/experiments", timeout=TIMEOUT_S)
response.raise_for_status()
ids = [e["id"] for e in response.json() if e["title"] == title]
if not ids:
raise PublishError(f"`{title}` got a 409 but is not on the dashboard")
return ids[0]


def _empty_datasets(experiment_id: str, url: str) -> List[str]:
response = requests.get(f"{url}/experiments/{experiment_id}/datasets", timeout=TIMEOUT_S)
response.raise_for_status()
return [d["name"] for d in response.json()["datasets"] if not d.get("rowCount")]


def publish(body: dict, token: str, url: str = DASHBOARD_URL) -> str:
"""Create the experiment, or find the one already there, and check every dataset has rows."""
headers = {"Authorization": f"Bearer {token}"}
response = requests.post(f"{url}/experiments", json=body, headers=headers, timeout=TIMEOUT_S)
if response.status_code == 409:
experiment_id = _existing_id(body["title"], url)
else:
response.raise_for_status()
experiment_id = response.json()["id"]

empty = _empty_datasets(experiment_id, url)
if empty:
requests.delete(
f"{url}/experiments/{experiment_id}", headers=headers, timeout=TIMEOUT_S
).raise_for_status()
raise PublishError(f"`{body['title']}` got no rows for {empty}; removed it")

logger.info(f"`{body['title']}` is on the dashboard as {experiment_id}")
return experiment_id


def unpublishable(exp: dict) -> Optional[str]:
"""Why a run stays off the dashboard, or None if its 10ksim code was clean and pushed."""
code = exp.get("code")
if not code:
return "no 10ksim commit was recorded"
if not code.get("clean"):
return f"10ksim had uncommitted changes on top of {code['commit'][:7]}"
if not code.get("pushed"):
return f"10ksim {code['commit'][:7]} was not pushed"
return None


def publish_run_metrics(exp: dict, run_name: str) -> Optional[str]:
"""Publish what `scrape_run_metrics` scraped for this run. Logs failures, never raises."""
reason = unpublishable(exp)
if reason:
logger.warning(f"`{run_name}` is not on the dashboard: {reason}")
return None
token = os.environ.get(TOKEN_ENV)
if not token:
logger.warning(f"{TOKEN_ENV} is not set; `{run_name}` is not on the dashboard")
return None
try:
return publish(experiment_body(exp, run_scrape_config(exp, name=run_name), run_name), token)
except Exception:
logger.exception(f"Publishing `{run_name}` to the dashboard failed")
return None
89 changes: 89 additions & 0 deletions src/analysis/post_run/tests/test_publish.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,89 @@
import pytest

from src.analysis.post_run import publish as publish_module
from src.analysis.post_run.metrics import run_scrape_config
from src.analysis.post_run.publish import PublishError, experiment_body, publish

EXP = {
"experiment": {"name": "nimlibp2p"},
"params": {"muxer": "yamux", "image": "regression-node:v2.4.0"},
"stack": {"namespace": "zerotesting"},
"results": {
"complete": {"start": "2026-09-16T10:03:45", "end": "2026-09-16T10:26:32"},
"stable": {"start": "2026-09-16T10:10:00", "end": "2026-09-16T10:16:30"},
},
}


class _Response:
def __init__(self, status_code, body=None):
self.status_code = status_code
self._body = body

def json(self):
return self._body

def raise_for_status(self):
if self.status_code >= 400:
raise RuntimeError(self.status_code)


def test_each_scraped_metric_is_a_dataset_over_the_stable_window():
config = run_scrape_config(EXP, name="run")
body = experiment_body(EXP, config, "run")

datasets = {d["name"]: d for d in body["datasets"]}
assert set(datasets) == {m.folder_name.strip("/") for m in config.metrics_to_scrape}
for dataset in datasets.values():
assert dataset["timeRange"] == {
"start": "2026-09-16T10:10:00+00:00",
"end": "2026-09-16T10:16:30+00:00",
}
assert "$__rate_interval" not in dataset["query"]["expr"]
assert "[121s]" in datasets["libp2p-in"]["query"]["expr"]
# The scrape names CSV columns after these labels, so they must be kept.
open_streams = [field["name"] for field in datasets["libp2p-open-streams"]["schema"]]
assert {"instance", "type", "dir"} <= set(open_streams)


@pytest.mark.parametrize("created", [True, False])
def test_a_dataset_without_rows_fails_the_publish_and_is_removed(monkeypatch, created):
deleted = []
gets = {
"http://dash/experiments": [{"id": "e0", "title": "other"}, {"id": "e1", "title": "run"}],
"http://dash/experiments/e1/datasets": {
"datasets": [
{"name": "libp2p-in", "rowCount": 7},
{"name": "mesh-peers", "rowCount": 0},
]
},
}
post = _Response(201, {"id": "e1"}) if created else _Response(409)
monkeypatch.setattr(publish_module.requests, "post", lambda *a, **k: post)
monkeypatch.setattr(publish_module.requests, "get", lambda url, **k: _Response(200, gets[url]))
monkeypatch.setattr(
publish_module.requests, "delete", lambda url, **k: deleted.append(url) or _Response(204)
)

with pytest.raises(PublishError, match="mesh-peers"):
publish({"title": "run"}, token="t", url="http://dash")
assert deleted == ["http://dash/experiments/e1"]


@pytest.mark.parametrize(
"code, published",
[
(None, False),
({"commit": "abc1234", "clean": False, "pushed": True}, False),
({"commit": "abc1234", "clean": True, "pushed": False}, False),
({"commit": "abc1234", "clean": True, "pushed": True}, True),
],
)
def test_only_runs_from_clean_pushed_code_are_published(monkeypatch, code, published):
posted = []
monkeypatch.setenv(publish_module.TOKEN_ENV, "t")
monkeypatch.setattr(publish_module, "publish", lambda body, token: posted.append(body))

publish_module.publish_run_metrics({**EXP, "code": code}, "run")

assert bool(posted) is published
62 changes: 62 additions & 0 deletions src/analysis/publish_run.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,62 @@
"""Publish finished runs' metrics to the DST dashboard from their folders."""

import argparse
import json
import logging
import os
import sys
from pathlib import Path

from src.analysis.post_run.metrics import run_scrape_config
from src.analysis.post_run.publish import (
DASHBOARD_URL,
TOKEN_ENV,
experiment_body,
publish,
unpublishable,
)
from src.analysis.utils.log_utils import init_logger

logger = logging.getLogger(__name__)


def main() -> None:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("runs", nargs="+", help="Run folders holding a metadata.json")
parser.add_argument("--dry-run", action="store_true", help="Print the experiments instead")
args = parser.parse_args()
init_logger(logging.getLogger(), verbosity=1)
token = os.environ.get(TOKEN_ENV)
if not token and not args.dry_run:
sys.exit(f"Set {TOKEN_ENV} to a token from {DASHBOARD_URL}/admin/token")

failed = []
for arg in args.runs:
run = Path(arg).resolve()
exp = json.loads((run / "metadata.json").read_text())
# Same rule as the post-run scrape, so the dashboard holds what the CSVs hold.
if not exp.get("results", {}).get("stable", {}).get("duration"):
logger.error(f"`{run.name}` has no stable window; skipping it")
failed.append(run.name)
continue
reason = unpublishable(exp)
if reason:
logger.error(f"`{run.name}` is not published: {reason}")
failed.append(run.name)
continue
body = experiment_body(exp, run_scrape_config(exp, name=run.name), run.name)
if args.dry_run:
print(json.dumps(body, indent=2))
continue
try:
publish(body, token)
except Exception:
logger.exception(f"Publishing `{run.name}` failed")
failed.append(run.name)

if failed:
sys.exit(f"Not published: {failed}")


if __name__ == "__main__":
main()
7 changes: 3 additions & 4 deletions src/deployments/core/base_bridge.py
Original file line number Diff line number Diff line change
Expand Up @@ -78,10 +78,9 @@ def get_metadata(self, events_log: Path) -> dict:
except ValueError as e:
logger.info(e)
metadata["stack"]["name"] = self._get_name(metadata)
try:
metadata["params"] = all_metadata["params"]
except KeyError:
pass
for key in ("params", "code"):
if key in all_metadata:
metadata[key] = all_metadata[key]
return dict(metadata)

def _get_name(self, metadata: dict):
Expand Down
33 changes: 33 additions & 0 deletions src/deployments/core/code_version.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
"""The 10ksim commit a run deploys from."""

import logging
import subprocess
from pathlib import Path
from typing import Optional

logger = logging.getLogger(__name__)

REPO = Path(__file__).parent


def _git(repo: Path, *args: str) -> str:
result = subprocess.run(
["git", "-C", str(repo), *args], check=True, capture_output=True, text=True
)
return result.stdout.strip()


def code_version(repo: Path = REPO) -> Optional[dict]:
"""Commit and branch of `repo`, and whether it was clean and pushed; None without git."""
try:
commit = _git(repo, "rev-parse", "HEAD")
return {
"repo": _git(repo, "remote", "get-url", "origin"),
"commit": commit,
"branch": _git(repo, "rev-parse", "--abbrev-ref", "HEAD"),
"clean": not _git(repo, "status", "--porcelain", "--untracked-files=no"),
"pushed": bool(_git(repo, "branch", "--remotes", "--contains", commit)),
}
except (OSError, subprocess.CalledProcessError) as e:
logger.warning(f"Could not read the 10ksim commit: {e}")
return None
Loading
Loading