From 3220ef03578ceaadd40bc8a441ef6f29c31cc8a9 Mon Sep 17 00:00:00 2001 From: radiken Date: Tue, 17 Nov 2026 11:54:05 +0000 Subject: [PATCH] Publish a run's logs, whole-run node and pod metrics and specs to the dashboard Besides the 13 stable-window metrics, each run stores every metric the nodes' own scrape job reported on any statefulset's first pod, plus container CPU, throttling, memory and network and the pods' host and start time, per pod at 30 s over the whole run. nim-libp2p runs also store their sends and deliveries (the dashboard parses only that tracer), checked against the run's summary CSVs; metadata adds the events and deployed specs. A metric that every pod reports with the same labels and value for the whole run is recorded in metadata.unchanged with the rule, not stored; fetch.run_dataset returns it as one row per pod. A publish deletes only an experiment it created and could not check. On a 409 the stored run is checked and an error raised if it is short, but it is never deleted: it may still be processing, or hold data VM no longer has. The loader reads datasets one at a time instead of all at once. Co-Authored-By: Claude Opus 5.5 --- src/analysis/post_run/fetch.py | 31 +- src/analysis/post_run/publish.py | 335 ++++++++++++++++++-- src/analysis/post_run/tests/test_fetch.py | 7 +- src/analysis/post_run/tests/test_publish.py | 196 ++++++++++-- src/analysis/publish_run.py | 23 +- 5 files changed, 508 insertions(+), 84 deletions(-) diff --git a/src/analysis/post_run/fetch.py b/src/analysis/post_run/fetch.py index 70f41d30..e25cfd8c 100644 --- a/src/analysis/post_run/fetch.py +++ b/src/analysis/post_run/fetch.py @@ -15,7 +15,9 @@ DASHBOARD_URL, SCRAPE_LABEL, TIMEOUT_S, + UNCHANGED, metric_dataset, + run_pods, run_title, ) @@ -69,15 +71,33 @@ def _query_response(rows: List[dict]) -> dict: return {"data": {"result": [{"metric": m, "values": v} for m, v in series.values()]}} +def run_dataset(experiment: dict, name: str, url: str = DASHBOARD_URL) -> List[dict]: + """A dataset's rows; a metric under UNCHANGED_RULE is one row per pod at the end of the run.""" + metadata = experiment["metadata"] + unchanged = metadata.get(UNCHANGED, {}).get("metrics", {}) + if name in unchanged: + end, same = metadata["results"]["complete"]["end"], unchanged[name] + return [ + {"timestamp": end, "value": same["value"], "pod": pod, **same["labels"]} + for pod in run_pods(metadata["stack"]) + ] + if name not in {d["name"] for d in experiment.get("datasets", [])}: + raise LookupError(f"`{experiment['title']}` has no `{name}` dataset") + response = requests.get( + f"{url}/experiments/{experiment['id']}/datasets/{name}/data", timeout=TIMEOUT_S + ) + if response.status_code == 404: + raise LookupError(f"`{experiment['title']}` has no stored `{name}` data") + response.raise_for_status() + return response.json()["data"] + + def fetch_run_metrics( run_name: str, dump: Path, url: str = DASHBOARD_URL, title: Optional[str] = None ) -> Path: """Write the run's stored datasets to `dump` with the scraper's own CSV writer.""" experiment = find_experiment(run_name, url, title) queries = {d["name"]: d["query"] for d in experiment.get("datasets", [])} - response = requests.get(f"{url}/experiments/{experiment['id']}/datasets", timeout=TIMEOUT_S) - response.raise_for_status() - stored = {d["name"]: d["data"] for d in response.json()["datasets"]} partial = dump.with_name(f"{dump.name}.partial") shutil.rmtree(partial, ignore_errors=True) @@ -91,9 +111,10 @@ def fetch_run_metrics( query, current = queries[name], metric_dataset(metric, config)["query"] if (query.get("expr"), query.get("step")) != (current["expr"], current["step"]): raise ValueError(f"`{run_name}` stored {name} as {query}, not as today's {current}") - if not stored.get(name): + rows = run_dataset(experiment, name, url) + if not rows: raise LookupError(f"`{run_name}` has no stored rows for {name}") - handler = DataRequestHandler(_query_response(stored[name])) + handler = DataRequestHandler(_query_response(rows)) handler.create_dataframe_from_request( metric.extract_field, metric.container, metric.metrics_path ) diff --git a/src/analysis/post_run/publish.py b/src/analysis/post_run/publish.py index 6b5f1c96..9b940db9 100644 --- a/src/analysis/post_run/publish.py +++ b/src/analysis/post_run/publish.py @@ -1,10 +1,13 @@ -"""Publish a run's metrics to the DST dashboard, which keeps them past the lab's retention.""" +"""Publish a run's data to the DST dashboard, which keeps it past the lab's retention.""" +import csv +import json import logging import os +from concurrent.futures import ThreadPoolExecutor from datetime import datetime, timezone from pathlib import Path -from typing import List, Optional +from typing import Dict, List, Optional, Tuple import requests @@ -16,9 +19,55 @@ DASHBOARD_URL = "https://api.dashboard.lab.vac.dev" TOKEN_ENV = "DST_DASHBOARD_TOKEN" DATASOURCE = "victoria-metrics" +LOGS_DATASOURCE = "victoria-logs" TIMEOUT_S = 600 # The name the run's scraped CSVs share, kept so a fetch writes the same files. SCRAPE_LABEL = "scrape_label" +SCRAPE_INTERVAL = "30s" +# Labels every series carries; the others tell a pod's series apart. +TARGET_LABELS = { + "__name__", + "cluster", + "container", + "endpoint", + "instance", + "job", + "metrics_path", + "namespace", + "node", + "pod", + "pod-node", + "prometheus", + "service", + "uid", +} +CONTAINER_METRICS = [ + "container_cpu_usage_seconds_total", + "container_cpu_cfs_periods_total", + "container_cpu_cfs_throttled_periods_total", + "container_memory_working_set_bytes", + "container_memory_rss", +] +POD_METRICS = ["container_network_receive_bytes_total", "container_network_transmit_bytes_total"] +KUBE_METRICS = {"kube_pod_info": ["node"], "kube_pod_start_time": []} +# The dashboard parses only the nimlibp2p tracer's log lines. +LOG_EXPERIMENTS = "nimlibp2p" +LOG_FIELDS = { + "sent": {"timestamp": "datetime", "msgId": "integer", "pod_name": "string"}, + "received": { + "timestamp": "datetime", + "msgId": "integer", + "sentAt": "datetime", + "delayMs": "float", + "pod_name": "string", + }, +} +SUMMARY = Path("analysis_data/summary") +UNCHANGED = "unchanged" +UNCHANGED_RULE = ( + "A whole-run metric that every pod of the run reports with the same labels and value for the " + "whole run is not stored as a dataset; `metrics` gives those labels and that value." +) class PublishError(Exception): @@ -29,6 +78,10 @@ def _utc(dt: datetime) -> str: return (dt.replace(tzinfo=timezone.utc) if dt.tzinfo is None else dt).isoformat() +def _schema(fields: Dict[str, str]) -> List[dict]: + return [{"name": name, "type": kind} for name, kind in fields.items()] + + def metric_dataset(metric: MetricToScrape, config: ScrapeConfig) -> dict: labels = metric.extract_field.split("-") labels += [label for label in ("container", "metrics_path") if getattr(metric, label)] @@ -40,8 +93,9 @@ def metric_dataset(metric: MetricToScrape, config: ScrapeConfig) -> dict: "expr": metric.query.replace("$__rate_interval", config.rate_interval), "step": config.step, }, - "schema": [{"name": "timestamp", "type": "datetime"}, {"name": "value", "type": "float"}] - + [{"name": label, "type": "string"} for label in labels], + "schema": _schema( + {"timestamp": "datetime", "value": "float"} | dict.fromkeys(labels, "string") + ), } @@ -55,62 +109,281 @@ def scrape_label(run: Path) -> Optional[str]: return labels.pop() if len(labels) == 1 else None +def run_pods(stack: dict) -> List[str]: + return [ + f"{name}-{i}" + for name, count in zip(stack["stateful_sets"], stack["nodes_per_statefulset"]) + for i in range(count) + ] + + +def _window(exp: dict, name: str) -> Dict[str, str]: + window = exp["results"][name] + return {key: _utc(datetime.fromisoformat(window[key])) for key in ("start", "end")} + + +def _pods(exp: dict, ordinal: str = "[0-9]+") -> str: + stack = exp["stack"] + sets = "|".join(stack["stateful_sets"]) + return f'namespace="{stack["namespace"]}", pod=~"({sets})-{ordinal}"' + + +def _raw(name: str, selector: str, labels: List[str], window: dict) -> dict: + by = ", ".join(["pod", *labels]) + return { + "name": name, + "datasource": DATASOURCE, + "timeRange": window, + "query": {"expr": f"max by ({by}) ({name}{{{selector}}})", "step": SCRAPE_INTERVAL}, + "schema": _schema( + {"timestamp": "datetime", "value": "float", "pod": "string"} + | dict.fromkeys(labels, "string") + ), + } + + +def _instant(expr: str, at: float, vm_url: str) -> List[dict]: + for attempt in range(2): + response = requests.get( + f"{vm_url.rstrip('/')}/query", params={"query": expr, "time": at}, timeout=TIMEOUT_S + ) + if response.ok: + break + # Cold data can run past VM's query limit once, then answer. + logger.warning(f"VM answered {response.status_code} to `{expr}`: {response.text[:200]}") + response.raise_for_status() + return response.json()["data"]["result"] + + +def node_metrics(exp: dict, vm_url: str) -> Dict[str, List[str]]: + """The metrics the nodes' own scrape job reported, from each statefulset's first pod.""" + end = datetime.fromisoformat(exp["results"]["stable"]["end"]).replace(tzinfo=timezone.utc) + first = _pods(exp, "0") + found = _instant(f"last_over_time({{{first}}}[300s])", end.timestamp(), vm_url) + series = [s["metric"] for s in found] + jobs = {s.get("job") for s in series if s["__name__"].startswith("libp2p_")} + labels: Dict[str, set] = {} + for s in series: + name = s["__name__"] + if ( + s.get("job") in jobs + and not name.endswith("_created") + and name != "up" + and not name.startswith("scrape_") + ): + labels.setdefault(name, set()).update( + l for l in set(s) - TARGET_LABELS if l.isidentifier() + ) + if not labels: + raise PublishError(f"No node metrics on {first} in the 5 min before the stable end") + return {name: sorted(labels[name]) for name in sorted(labels)} + + +def _unchanged( + selector: str, labels: List[str], window: dict, pods: List[str], vm_url: str +) -> Optional[dict]: + """The labels and value every pod reports for `selector` over `window`, else None.""" + start, end = (datetime.fromisoformat(window[key]).timestamp() for key in ("start", "end")) + # A range query can take samples from a scrape before `start` and a step after `end`. + at, span = end + 30, f"{int(end - start) + 90}s" + over = lambda agg: f"{agg}_over_time({selector}[{span}])" + low, high = (_instant(f"{agg}({over(agg)})", at, vm_url) for agg in ("min", "max")) + if not low or float(low[0]["value"][1]) != float(high[0]["value"][1]): + return None + by = ", ".join(["pod", *labels]) + sets = [s["metric"] for s in _instant(f"count by ({by}) ({over('last')})", at, vm_url)] + reported = {s.pop("pod", None) for s in sets} + if reported != set(pods) or len({tuple(sorted(s.items())) for s in sets}) != 1: + return None + return {"value": float(low[0]["value"][1]), "labels": sets[0]} + + +def run_datasets( + exp: dict, metrics: Dict[str, List[str]], vm_url: str +) -> Tuple[List[dict], Dict[str, dict]]: + """Whole-run node and pod datasets, and the metrics UNCHANGED_RULE leaves out.""" + window, pods, names = _window(exp, "complete"), _pods(exp), run_pods(exp["stack"]) + cadvisor = f'{pods}, metrics_path="/metrics/cadvisor"' + container = f'{cadvisor}, container="{exp["stack"]["container_name"]}"' + series = ( + [(name, pods, labels) for name, labels in (metrics | KUBE_METRICS).items()] + + [(name, container, []) for name in CONTAINER_METRICS] + + [(name, cadvisor, []) for name in POD_METRICS] + ) + + def check(entry: tuple) -> Optional[dict]: + name, selector, labels = entry + return _unchanged(f"{name}{{{selector}}}", labels, window, names, vm_url) + + with ThreadPoolExecutor(4) as pool: + found = list(pool.map(check, series)) + datasets, unchanged = [], {} + for (name, selector, labels), same in zip(series, found): + if same is None: + datasets.append(_raw(name, selector, labels, window)) + else: + unchanged[name] = same + return datasets, unchanged + + +def log_datasets(exp: dict) -> List[dict]: + """A nim-libp2p run's sends and deliveries, the rows of its summary/{sent,received}.csv.""" + if not exp["experiment"]["name"].startswith(LOG_EXPERIMENTS): + return [] + query = {"namespace": exp["stack"]["namespace"], "tracer": "nimlibp2p"} + return [ + { + "name": pattern, + "datasource": LOGS_DATASOURCE, + "timeRange": _window(exp, "complete"), + "query": query | {"pattern": pattern}, + "schema": _schema(fields), + } + for pattern, fields in LOG_FIELDS.items() + ] + + +def run_files(run: Path) -> dict: + """The run's events, without experiment dumps and per-message sends, and its deployed specs.""" + events = run / "events.log" + lines = events.read_text().splitlines() if events.exists() else [] + kept = [ + e + for e in map(json.loads, filter(None, lines)) + if e.get("event") not in ("metadata", "publish", "send") + ] + specs = { + path.name: path.read_text() for path in sorted((run / "deployment_yamls").glob("*.yaml")) + } + return {"events": kept, "deployment": specs} + + def experiment_body( - exp: dict, config: ScrapeConfig, run_name: str, label: Optional[str] = None + exp: dict, + config: ScrapeConfig, + run_name: str, + label: Optional[str] = None, + extra: List[dict] = (), + metadata: Optional[dict] = None, ) -> dict: - """One dataset per scraped metric, over the same window and step as the local CSVs.""" + """The scraped metrics over the stable window, as the local CSVs, then `extra` datasets.""" + stack = exp.get("stack", {}) return { "title": run_title(run_name, config), "family": exp["experiment"]["name"], "date": f"{config.start:%Y-%m-%d}", "docker_image": exp.get("params", {}).get("image"), + # The dashboard's log fetch reads statefulsets, nodes, namespace and container_name. "metadata": {"run": run_name, "experiment": exp["experiment"]["name"]} | {key: exp[key] for key in ("params", "code", "stack", "results") if key in exp} - | ({SCRAPE_LABEL: label} if label else {}), - "datasets": [metric_dataset(metric, config) for metric in config.metrics_to_scrape], + | ({SCRAPE_LABEL: label} if label else {}) + | { + "statefulsets": stack.get("stateful_sets"), + "nodes": stack.get("nodes_per_statefulset"), + "namespace": stack.get("namespace"), + "container_name": stack.get("container_name"), + } + | (metadata or {}), + "datasets": [metric_dataset(metric, config) for metric in config.metrics_to_scrape] + + list(extra), "panels": [], "publish": False, } -def _existing_id(title: str, url: str) -> str: +def run_body(exp: dict, run: Path) -> dict: + """Everything published for one run folder.""" + config = run_scrape_config(exp, name=run.name) + datasets, unchanged = run_datasets(exp, node_metrics(exp, config.url), config.url) + metadata = run_files(run) | {UNCHANGED: {"rule": UNCHANGED_RULE, "metrics": unchanged}} + extra = datasets + log_datasets(exp) + return experiment_body(exp, config, run.name, scrape_label(run), extra, metadata) + + +def expected_rows(body: dict, run: Path) -> Dict[str, int]: + """Row counts the log datasets must match: the run's own summary CSVs.""" + counts = {} + for dataset in body["datasets"]: + if dataset["datasource"] != LOGS_DATASOURCE: + continue + path = run / SUMMARY / f"{dataset['name']}.csv" + if not path.exists(): + logger.warning(f"`{run.name}` has no {path.name}; only checking it got rows") + continue + with open(path, newline="") as f: + counts[dataset["name"]] = sum(1 for _ in csv.reader(f)) - 1 + return counts + + +def _existing(title: str, url: str) -> dict: response = requests.get(f"{url}/experiments", timeout=TIMEOUT_S) response.raise_for_status() - ids = [e["id"] for e in response.json() if e["title"] == title] - if not ids: + found = [e for e in response.json() if e["title"] == title] + if not found: raise PublishError(f"`{title}` got a 409 but is not on the dashboard") - return ids[0] + return found[0] -def _empty_datasets(experiment_id: str, url: str) -> List[str]: - response = requests.get(f"{url}/experiments/{experiment_id}/datasets", timeout=TIMEOUT_S) - response.raise_for_status() - return [d["name"] for d in response.json()["datasets"] if not d.get("rowCount")] +def _bad_datasets( + experiment_id: str, names: List[str], url: str, expected: Dict[str, int] +) -> List[str]: + """Datasets stored empty, or with other row counts than the run's CSVs.""" + bad = [] + for name in names: + response = requests.get( + f"{url}/experiments/{experiment_id}/datasets/{name}/data", timeout=TIMEOUT_S + ) + if response.status_code == 404: + bad.append(f"{name} (not stored)") + continue + response.raise_for_status() + rows = len(response.json()["data"]) + if not rows or rows != expected.get(name, rows): + bad.append(f"{name} ({rows} rows, expected {expected.get(name, 'some')})") + return bad -def publish(body: dict, token: str, url: str = DASHBOARD_URL) -> str: - """Create the experiment, or find the one already there, and check every dataset has rows.""" - headers = {"Authorization": f"Bearer {token}"} +def publish( + body: dict, token: str, url: str = DASHBOARD_URL, expected: Dict[str, int] = None +) -> str: + """Create the experiment and check every dataset's rows, or check the one already there.""" + headers, title, expected = {"Authorization": f"Bearer {token}"}, body["title"], expected or {} response = requests.post(f"{url}/experiments", json=body, headers=headers, timeout=TIMEOUT_S) if response.status_code == 409: - experiment_id = _existing_id(body["title"], url) - else: - response.raise_for_status() - experiment_id = response.json()["id"] + existing = _existing(title, url) + names = [d["name"] for d in existing["datasets"]] + missing = [d["name"] for d in body["datasets"] if d["name"] not in names] + if missing: + logger.warning(f"`{title}` is already on the dashboard without {missing}; not added") + # Never deleted here: it may still be processing, or hold data VM no longer has. + bad = _bad_datasets(existing["id"], names, url, expected) + if bad: + raise PublishError(f"`{title}` is already on the dashboard with {bad}") + logger.info(f"`{title}` is already on the dashboard as {existing['id']}") + return existing["id"] + if not response.ok: + raise PublishError(f"`{title}` got {response.status_code}: {response.text[:500]}") - empty = _empty_datasets(experiment_id, url) - if empty: + experiment_id = response.json()["id"] + try: + bad = _bad_datasets(experiment_id, [d["name"] for d in body["datasets"]], url, expected) + if bad: + raise PublishError(f"`{title}` got {bad}; removed it") + except Exception: requests.delete( f"{url}/experiments/{experiment_id}", headers=headers, timeout=TIMEOUT_S ).raise_for_status() - raise PublishError(f"`{body['title']}` got no rows for {empty}; removed it") + raise - logger.info(f"`{body['title']}` is on the dashboard as {experiment_id}") + logger.info(f"`{title}` is on the dashboard as {experiment_id}") return experiment_id def unpublishable(exp: dict) -> Optional[str]: - """Why a run stays off the dashboard, or None if its 10ksim code was clean and pushed.""" + """Why a run stays off the dashboard, or None if it can go there.""" + # The bridge leaves out `duration` when the window is inverted. + if not exp.get("results", {}).get("stable", {}).get("duration"): + return "it has no stable window" code = exp.get("code") if not code: return "no 10ksim commit was recorded" @@ -122,7 +395,7 @@ def unpublishable(exp: dict) -> Optional[str]: def publish_run_metrics(exp: dict, run: Path) -> Optional[str]: - """Publish what `scrape_run_metrics` scraped for this run. Logs failures, never raises.""" + """Publish this run's metrics, logs and specs. Logs failures, never raises.""" reason = unpublishable(exp) if reason: logger.warning(f"`{run.name}` is not on the dashboard: {reason}") @@ -132,8 +405,8 @@ def publish_run_metrics(exp: dict, run: Path) -> Optional[str]: logger.warning(f"{TOKEN_ENV} is not set; `{run.name}` is not on the dashboard") return None try: - config = run_scrape_config(exp, name=run.name) - return publish(experiment_body(exp, config, run.name, scrape_label(run)), token) + body = run_body(exp, run) + return publish(body, token, expected=expected_rows(body, run)) except Exception: logger.exception(f"Publishing `{run.name}` to the dashboard failed") return None diff --git a/src/analysis/post_run/tests/test_fetch.py b/src/analysis/post_run/tests/test_fetch.py index cff23bfc..7c443b51 100644 --- a/src/analysis/post_run/tests/test_fetch.py +++ b/src/analysis/post_run/tests/test_fetch.py @@ -33,6 +33,8 @@ class _Response: + status_code = 200 + def __init__(self, body): self._body = body @@ -76,9 +78,10 @@ def _published(monkeypatch, run, query=None, empty=()): def get(url, **kwargs): if url.endswith("/experiments"): return _Response(experiments) - assert url.endswith("/experiments/e1/datasets"), url rows = {STREAMS: _stored_rows(VM_RESPONSE)} | dict.fromkeys(empty, []) - return _Response({"datasets": [{"name": n, "data": r} for n, r in rows.items()]}) + name = url.removeprefix("http://dash/experiments/e1/datasets/").removesuffix("/data") + assert name in rows, url + return _Response({"data": rows[name]}) monkeypatch.setattr(fetch.requests, "get", get) diff --git a/src/analysis/post_run/tests/test_publish.py b/src/analysis/post_run/tests/test_publish.py index 3d9b1612..0a0be354 100644 --- a/src/analysis/post_run/tests/test_publish.py +++ b/src/analysis/post_run/tests/test_publish.py @@ -1,7 +1,9 @@ +import re from pathlib import Path import pytest +from src.analysis.post_run import fetch from src.analysis.post_run import publish as publish_module from src.analysis.post_run.metrics import run_scrape_config from src.analysis.post_run.publish import PublishError, experiment_body, publish @@ -12,24 +14,71 @@ "stack": {"namespace": "zerotesting"}, "results": { "complete": {"start": "2026-09-16T10:03:45", "end": "2026-09-16T10:26:32"}, - "stable": {"start": "2026-09-16T10:10:00", "end": "2026-09-16T10:16:30"}, + "stable": { + "start": "2026-09-16T10:10:00", + "end": "2026-09-16T10:16:30", + "duration": "0h 6m 30s", + }, }, } +BODY = {"title": "run", "datasets": [{"name": "mesh-peers"}, {"name": "received"}]} class _Response: def __init__(self, status_code, body=None): self.status_code = status_code + self.ok = status_code < 400 + self.text = "" self._body = body def json(self): return self._body def raise_for_status(self): - if self.status_code >= 400: + if not self.ok: raise RuntimeError(self.status_code) +def _dashboard(monkeypatch, post, existing, rows): + """A dashboard answering a POST with `post`, holding `existing`, its datasets with `rows`.""" + deleted = [] + + def get(url, **kwargs): + if url == "http://dash/experiments": + return _Response(200, [{"id": "e0", "title": "other"}, existing]) + name = url.removeprefix("http://dash/experiments/e1/datasets/").removesuffix("/data") + if isinstance(rows.get(name), Exception): + raise rows[name] + if name not in rows: + return _Response(404) + return _Response(200, {"data": [{}] * rows[name]}) + + monkeypatch.setattr(publish_module.requests, "post", lambda *a, **k: post) + monkeypatch.setattr(publish_module.requests, "get", get) + monkeypatch.setattr( + publish_module.requests, "delete", lambda url, **k: deleted.append(url) or _Response(204) + ) + return deleted + + +def _vm(monkeypatch, discovered, metrics): + """VM with `discovered` series on the first pods; `metrics`: name -> (min, max, series).""" + + def get(url, params=None, **kwargs): + assert url.endswith("/query"), f"contacted {url}" + query = params["query"] + if query.startswith("last_over_time({"): + return _Response(200, {"data": {"result": [{"metric": s} for s in discovered]}}) + name = re.search(r"_over_time\((\w+)\{", query)[1] + low, high, series = metrics.get(name, (0.0, 5.0, [])) + if query.startswith("count by"): + return _Response(200, {"data": {"result": [{"metric": dict(s)} for s in series]}}) + value = low if query.startswith("min") else high + return _Response(200, {"data": {"result": [{"value": [0, str(value)]}]}}) + + monkeypatch.setattr(publish_module.requests, "get", get) + + def test_each_scraped_metric_is_a_dataset_over_the_stable_window(): config = run_scrape_config(EXP, name="run") body = experiment_body(EXP, config, "run") @@ -48,44 +97,129 @@ def test_each_scraped_metric_is_a_dataset_over_the_stable_window(): assert {"instance", "type", "dir"} <= set(open_streams) -@pytest.mark.parametrize("created", [True, False]) -def test_a_dataset_without_rows_fails_the_publish_and_is_removed(monkeypatch, created): - deleted = [] - gets = { - "http://dash/experiments": [{"id": "e0", "title": "other"}, {"id": "e1", "title": "run"}], - "http://dash/experiments/e1/datasets": { - "datasets": [ - {"name": "libp2p-in", "rowCount": 7}, - {"name": "mesh-peers", "rowCount": 0}, - ] - }, - } - post = _Response(201, {"id": "e1"}) if created else _Response(409) - monkeypatch.setattr(publish_module.requests, "post", lambda *a, **k: post) - monkeypatch.setattr(publish_module.requests, "get", lambda url, **k: _Response(200, gets[url])) - monkeypatch.setattr( - publish_module.requests, "delete", lambda url, **k: deleted.append(url) or _Response(204) - ) - - with pytest.raises(PublishError, match="mesh-peers"): - publish({"title": "run"}, token="t", url="http://dash") +@pytest.mark.parametrize( + "rows, bad", + [ + ({"mesh-peers": 0, "received": 5}, "mesh-peers"), + ({"mesh-peers": 3, "received": 4}, "received"), + ({"mesh-peers": 3, "received": TimeoutError("read timed out")}, "read timed out"), + ({"mesh-peers": 3, "received": 5}, None), + ], +) +def test_a_created_experiment_that_fails_its_check_is_removed(monkeypatch, rows, bad): + deleted = _dashboard(monkeypatch, _Response(201, {"id": "e1"}), {"id": "e1"}, rows) + + if bad is None: + assert publish(BODY, token="t", url="http://dash", expected={"received": 5}) == "e1" + assert deleted == [] + return + with pytest.raises((PublishError, TimeoutError), match=bad): + publish(BODY, token="t", url="http://dash", expected={"received": 5}) assert deleted == ["http://dash/experiments/e1"] @pytest.mark.parametrize( - "code, published", + "datasets, rows, bad", + [ + (["mesh-peers", "received"], {"mesh-peers": 3, "received": 4}, "received"), + (["mesh-peers", "received"], {"mesh-peers": 3}, "not stored"), + (["mesh-peers"], {"mesh-peers": 3}, None), + ], +) +def test_an_experiment_already_there_is_checked_but_never_removed(monkeypatch, datasets, rows, bad): + existing = {"id": "e1", "title": "run", "datasets": [{"name": n} for n in datasets]} + deleted = _dashboard(monkeypatch, _Response(409), existing, rows) + + if bad is None: + assert publish(BODY, token="t", url="http://dash", expected={"received": 5}) == "e1" + else: + with pytest.raises(PublishError, match=bad): + publish(BODY, token="t", url="http://dash", expected={"received": 5}) + assert deleted == [] + + +def test_node_metrics_are_the_node_jobs_own_series_on_every_statefulset(monkeypatch): + discovered = [ + {"__name__": "libp2p_peers", "job": "nodes", "pod": "relay-0", "instance": "i"}, + {"__name__": "libp2p_network_bytes_total", "job": "nodes", "direction": "in"}, + {"__name__": "libp2p_network_bytes_total", "job": "nodes", "direction": "out"}, + {"__name__": "logos_archive_messages", "job": "nodes", "pod": "store-0"}, + {"__name__": "libp2p_pubsub_peers_created", "job": "nodes"}, + {"__name__": "scrape_duration_seconds", "job": "nodes"}, + {"__name__": "up", "job": "nodes"}, + {"__name__": "container_cpu_usage_seconds_total", "job": "kubelet", "cpu": "total"}, + ] + stack = {"namespace": "z", "stateful_sets": ["relay", "store"], "nodes_per_statefulset": [9, 2]} + exp = {**EXP, "stack": stack} + + _vm(monkeypatch, discovered, {}) + assert publish_module.node_metrics(exp, "http://vm") == { + "libp2p_network_bytes_total": ["direction"], + "libp2p_peers": [], + "logos_archive_messages": [], + } + _vm(monkeypatch, [], {}) + with pytest.raises(PublishError, match="No node metrics"): + publish_module.node_metrics(exp, "http://vm") + + +def test_a_metric_that_never_changes_is_published_as_its_value_and_read_back_as_it( + monkeypatch, tmp_path +): + pods = ["pod-0", "pod-1", "pod-2"] + every = lambda labels: [{"pod": pod} | labels for pod in pods] + # name: (min, max, series over the run), and the labels discovery finds on it + vm = { + "libp2p_peers": (3.0, 9.0, every({}), {}), + "libp2p_pubsub_topics": (1.0, 1.0, every({}), {}), + "logos_delivery_version": (1.0, 1.0, every({"version": "v1"}), {"version": "v1"}), + "libp2p_gossipsub_received_iwants": ( + 0.0, + 0.0, + every({"kind": "a"}) + every({"kind": "b"}), + {"kind": "a"}, + ), + "libp2p_gossipsub_low_peers_topics": (0.0, 0.0, every({})[:2], {}), + } + discovered = [{"__name__": n, "job": "nodes"} | v[3] for n, v in vm.items()] + _vm(monkeypatch, discovered, {n: v[:3] for n, v in vm.items()}) + stack = {"namespace": "z", "stateful_sets": ["pod"], "nodes_per_statefulset": [3]} + exp = {**EXP, "stack": stack | {"container_name": "pod-0"}} + body = publish_module.run_body(exp, tmp_path / "run") + + names = {d["name"] for d in body["datasets"]} + assert {"libp2p_peers", "libp2p_gossipsub_received_iwants"} <= names + assert "libp2p_gossipsub_low_peers_topics" in names + assert body["metadata"]["unchanged"]["metrics"] == { + "libp2p_pubsub_topics": {"value": 1.0, "labels": {}}, + "logos_delivery_version": {"value": 1.0, "labels": {"version": "v1"}}, + } + end = EXP["results"]["complete"]["end"] + assert fetch.run_dataset({"id": "e1"} | body, "logos_delivery_version") == [ + {"timestamp": end, "value": 1.0, "pod": pod, "version": "v1"} for pod in pods + ] + + +@pytest.mark.parametrize( + "stable, code, published", [ - (None, False), - ({"commit": "abc1234", "clean": False, "pushed": True}, False), - ({"commit": "abc1234", "clean": True, "pushed": False}, False), - ({"commit": "abc1234", "clean": True, "pushed": True}, True), + (True, None, False), + (True, {"commit": "abc1234", "clean": False, "pushed": True}, False), + (True, {"commit": "abc1234", "clean": True, "pushed": False}, False), + (True, {"commit": "abc1234", "clean": True, "pushed": True}, True), + (False, {"commit": "abc1234", "clean": True, "pushed": True}, False), ], ) -def test_only_runs_from_clean_pushed_code_are_published(monkeypatch, code, published): +def test_only_runs_with_a_stable_window_from_clean_pushed_code_are_published( + monkeypatch, stable, code, published +): posted = [] monkeypatch.setenv(publish_module.TOKEN_ENV, "t") - monkeypatch.setattr(publish_module, "publish", lambda body, token: posted.append(body)) + monkeypatch.setattr(publish_module, "run_body", lambda exp, run: {"datasets": []}) + monkeypatch.setattr(publish_module, "publish", lambda body, token, **k: posted.append(body)) + # The bridge leaves out `duration` when the stable window is inverted. + results = EXP["results"] if stable else {"stable": {"start": "10:02:25", "end": "10:01:51"}} - publish_module.publish_run_metrics({**EXP, "code": code}, Path("run")) + publish_module.publish_run_metrics({**EXP, "results": results, "code": code}, Path("run")) assert bool(posted) is published diff --git a/src/analysis/publish_run.py b/src/analysis/publish_run.py index 3d30b607..17d082a5 100644 --- a/src/analysis/publish_run.py +++ b/src/analysis/publish_run.py @@ -1,4 +1,4 @@ -"""Publish finished runs' metrics to the DST dashboard from their folders.""" +"""Publish finished runs' metrics, logs and specs to the DST dashboard from their folders.""" import argparse import json @@ -7,13 +7,12 @@ import sys from pathlib import Path -from src.analysis.post_run.metrics import run_scrape_config from src.analysis.post_run.publish import ( DASHBOARD_URL, TOKEN_ENV, - experiment_body, + expected_rows, publish, - scrape_label, + run_body, unpublishable, ) from src.analysis.utils.log_utils import init_logger @@ -35,23 +34,17 @@ def main() -> None: for arg in args.runs: run = Path(arg).resolve() exp = json.loads((run / "metadata.json").read_text()) - # Same rule as the post-run scrape, so the dashboard holds what the CSVs hold. - if not exp.get("results", {}).get("stable", {}).get("duration"): - logger.error(f"`{run.name}` has no stable window; skipping it") - failed.append(run.name) - continue reason = unpublishable(exp) if reason: logger.error(f"`{run.name}` is not published: {reason}") failed.append(run.name) continue - config = run_scrape_config(exp, name=run.name) - body = experiment_body(exp, config, run.name, scrape_label(run)) - if args.dry_run: - print(json.dumps(body, indent=2)) - continue try: - publish(body, token) + body = run_body(exp, run) + if args.dry_run: + print(json.dumps(body, indent=2)) + continue + publish(body, token, expected=expected_rows(body, run)) except Exception: logger.exception(f"Publishing `{run.name}` failed") failed.append(run.name)