diff --git a/src/analysis/post_run/fetch.py b/src/analysis/post_run/fetch.py index 70f41d30..e25cfd8c 100644 --- a/src/analysis/post_run/fetch.py +++ b/src/analysis/post_run/fetch.py @@ -15,7 +15,9 @@ DASHBOARD_URL, SCRAPE_LABEL, TIMEOUT_S, + UNCHANGED, metric_dataset, + run_pods, run_title, ) @@ -69,15 +71,33 @@ def _query_response(rows: List[dict]) -> dict: return {"data": {"result": [{"metric": m, "values": v} for m, v in series.values()]}} +def run_dataset(experiment: dict, name: str, url: str = DASHBOARD_URL) -> List[dict]: + """A dataset's rows; a metric under UNCHANGED_RULE is one row per pod at the end of the run.""" + metadata = experiment["metadata"] + unchanged = metadata.get(UNCHANGED, {}).get("metrics", {}) + if name in unchanged: + end, same = metadata["results"]["complete"]["end"], unchanged[name] + return [ + {"timestamp": end, "value": same["value"], "pod": pod, **same["labels"]} + for pod in run_pods(metadata["stack"]) + ] + if name not in {d["name"] for d in experiment.get("datasets", [])}: + raise LookupError(f"`{experiment['title']}` has no `{name}` dataset") + response = requests.get( + f"{url}/experiments/{experiment['id']}/datasets/{name}/data", timeout=TIMEOUT_S + ) + if response.status_code == 404: + raise LookupError(f"`{experiment['title']}` has no stored `{name}` data") + response.raise_for_status() + return response.json()["data"] + + def fetch_run_metrics( run_name: str, dump: Path, url: str = DASHBOARD_URL, title: Optional[str] = None ) -> Path: """Write the run's stored datasets to `dump` with the scraper's own CSV writer.""" experiment = find_experiment(run_name, url, title) queries = {d["name"]: d["query"] for d in experiment.get("datasets", [])} - response = requests.get(f"{url}/experiments/{experiment['id']}/datasets", timeout=TIMEOUT_S) - response.raise_for_status() - stored = {d["name"]: d["data"] for d in response.json()["datasets"]} partial = dump.with_name(f"{dump.name}.partial") shutil.rmtree(partial, ignore_errors=True) @@ -91,9 +111,10 @@ def fetch_run_metrics( query, current = queries[name], metric_dataset(metric, config)["query"] if (query.get("expr"), query.get("step")) != (current["expr"], current["step"]): raise ValueError(f"`{run_name}` stored {name} as {query}, not as today's {current}") - if not stored.get(name): + rows = run_dataset(experiment, name, url) + if not rows: raise LookupError(f"`{run_name}` has no stored rows for {name}") - handler = DataRequestHandler(_query_response(stored[name])) + handler = DataRequestHandler(_query_response(rows)) handler.create_dataframe_from_request( metric.extract_field, metric.container, metric.metrics_path ) diff --git a/src/analysis/post_run/publish.py b/src/analysis/post_run/publish.py index 6b5f1c96..9b940db9 100644 --- a/src/analysis/post_run/publish.py +++ b/src/analysis/post_run/publish.py @@ -1,10 +1,13 @@ -"""Publish a run's metrics to the DST dashboard, which keeps them past the lab's retention.""" +"""Publish a run's data to the DST dashboard, which keeps it past the lab's retention.""" +import csv +import json import logging import os +from concurrent.futures import ThreadPoolExecutor from datetime import datetime, timezone from pathlib import Path -from typing import List, Optional +from typing import Dict, List, Optional, Tuple import requests @@ -16,9 +19,55 @@ DASHBOARD_URL = "https://api.dashboard.lab.vac.dev" TOKEN_ENV = "DST_DASHBOARD_TOKEN" DATASOURCE = "victoria-metrics" +LOGS_DATASOURCE = "victoria-logs" TIMEOUT_S = 600 # The name the run's scraped CSVs share, kept so a fetch writes the same files. SCRAPE_LABEL = "scrape_label" +SCRAPE_INTERVAL = "30s" +# Labels every series carries; the others tell a pod's series apart. +TARGET_LABELS = { + "__name__", + "cluster", + "container", + "endpoint", + "instance", + "job", + "metrics_path", + "namespace", + "node", + "pod", + "pod-node", + "prometheus", + "service", + "uid", +} +CONTAINER_METRICS = [ + "container_cpu_usage_seconds_total", + "container_cpu_cfs_periods_total", + "container_cpu_cfs_throttled_periods_total", + "container_memory_working_set_bytes", + "container_memory_rss", +] +POD_METRICS = ["container_network_receive_bytes_total", "container_network_transmit_bytes_total"] +KUBE_METRICS = {"kube_pod_info": ["node"], "kube_pod_start_time": []} +# The dashboard parses only the nimlibp2p tracer's log lines. +LOG_EXPERIMENTS = "nimlibp2p" +LOG_FIELDS = { + "sent": {"timestamp": "datetime", "msgId": "integer", "pod_name": "string"}, + "received": { + "timestamp": "datetime", + "msgId": "integer", + "sentAt": "datetime", + "delayMs": "float", + "pod_name": "string", + }, +} +SUMMARY = Path("analysis_data/summary") +UNCHANGED = "unchanged" +UNCHANGED_RULE = ( + "A whole-run metric that every pod of the run reports with the same labels and value for the " + "whole run is not stored as a dataset; `metrics` gives those labels and that value." +) class PublishError(Exception): @@ -29,6 +78,10 @@ def _utc(dt: datetime) -> str: return (dt.replace(tzinfo=timezone.utc) if dt.tzinfo is None else dt).isoformat() +def _schema(fields: Dict[str, str]) -> List[dict]: + return [{"name": name, "type": kind} for name, kind in fields.items()] + + def metric_dataset(metric: MetricToScrape, config: ScrapeConfig) -> dict: labels = metric.extract_field.split("-") labels += [label for label in ("container", "metrics_path") if getattr(metric, label)] @@ -40,8 +93,9 @@ def metric_dataset(metric: MetricToScrape, config: ScrapeConfig) -> dict: "expr": metric.query.replace("$__rate_interval", config.rate_interval), "step": config.step, }, - "schema": [{"name": "timestamp", "type": "datetime"}, {"name": "value", "type": "float"}] - + [{"name": label, "type": "string"} for label in labels], + "schema": _schema( + {"timestamp": "datetime", "value": "float"} | dict.fromkeys(labels, "string") + ), } @@ -55,62 +109,281 @@ def scrape_label(run: Path) -> Optional[str]: return labels.pop() if len(labels) == 1 else None +def run_pods(stack: dict) -> List[str]: + return [ + f"{name}-{i}" + for name, count in zip(stack["stateful_sets"], stack["nodes_per_statefulset"]) + for i in range(count) + ] + + +def _window(exp: dict, name: str) -> Dict[str, str]: + window = exp["results"][name] + return {key: _utc(datetime.fromisoformat(window[key])) for key in ("start", "end")} + + +def _pods(exp: dict, ordinal: str = "[0-9]+") -> str: + stack = exp["stack"] + sets = "|".join(stack["stateful_sets"]) + return f'namespace="{stack["namespace"]}", pod=~"({sets})-{ordinal}"' + + +def _raw(name: str, selector: str, labels: List[str], window: dict) -> dict: + by = ", ".join(["pod", *labels]) + return { + "name": name, + "datasource": DATASOURCE, + "timeRange": window, + "query": {"expr": f"max by ({by}) ({name}{{{selector}}})", "step": SCRAPE_INTERVAL}, + "schema": _schema( + {"timestamp": "datetime", "value": "float", "pod": "string"} + | dict.fromkeys(labels, "string") + ), + } + + +def _instant(expr: str, at: float, vm_url: str) -> List[dict]: + for attempt in range(2): + response = requests.get( + f"{vm_url.rstrip('/')}/query", params={"query": expr, "time": at}, timeout=TIMEOUT_S + ) + if response.ok: + break + # Cold data can run past VM's query limit once, then answer. + logger.warning(f"VM answered {response.status_code} to `{expr}`: {response.text[:200]}") + response.raise_for_status() + return response.json()["data"]["result"] + + +def node_metrics(exp: dict, vm_url: str) -> Dict[str, List[str]]: + """The metrics the nodes' own scrape job reported, from each statefulset's first pod.""" + end = datetime.fromisoformat(exp["results"]["stable"]["end"]).replace(tzinfo=timezone.utc) + first = _pods(exp, "0") + found = _instant(f"last_over_time({{{first}}}[300s])", end.timestamp(), vm_url) + series = [s["metric"] for s in found] + jobs = {s.get("job") for s in series if s["__name__"].startswith("libp2p_")} + labels: Dict[str, set] = {} + for s in series: + name = s["__name__"] + if ( + s.get("job") in jobs + and not name.endswith("_created") + and name != "up" + and not name.startswith("scrape_") + ): + labels.setdefault(name, set()).update( + l for l in set(s) - TARGET_LABELS if l.isidentifier() + ) + if not labels: + raise PublishError(f"No node metrics on {first} in the 5 min before the stable end") + return {name: sorted(labels[name]) for name in sorted(labels)} + + +def _unchanged( + selector: str, labels: List[str], window: dict, pods: List[str], vm_url: str +) -> Optional[dict]: + """The labels and value every pod reports for `selector` over `window`, else None.""" + start, end = (datetime.fromisoformat(window[key]).timestamp() for key in ("start", "end")) + # A range query can take samples from a scrape before `start` and a step after `end`. + at, span = end + 30, f"{int(end - start) + 90}s" + over = lambda agg: f"{agg}_over_time({selector}[{span}])" + low, high = (_instant(f"{agg}({over(agg)})", at, vm_url) for agg in ("min", "max")) + if not low or float(low[0]["value"][1]) != float(high[0]["value"][1]): + return None + by = ", ".join(["pod", *labels]) + sets = [s["metric"] for s in _instant(f"count by ({by}) ({over('last')})", at, vm_url)] + reported = {s.pop("pod", None) for s in sets} + if reported != set(pods) or len({tuple(sorted(s.items())) for s in sets}) != 1: + return None + return {"value": float(low[0]["value"][1]), "labels": sets[0]} + + +def run_datasets( + exp: dict, metrics: Dict[str, List[str]], vm_url: str +) -> Tuple[List[dict], Dict[str, dict]]: + """Whole-run node and pod datasets, and the metrics UNCHANGED_RULE leaves out.""" + window, pods, names = _window(exp, "complete"), _pods(exp), run_pods(exp["stack"]) + cadvisor = f'{pods}, metrics_path="/metrics/cadvisor"' + container = f'{cadvisor}, container="{exp["stack"]["container_name"]}"' + series = ( + [(name, pods, labels) for name, labels in (metrics | KUBE_METRICS).items()] + + [(name, container, []) for name in CONTAINER_METRICS] + + [(name, cadvisor, []) for name in POD_METRICS] + ) + + def check(entry: tuple) -> Optional[dict]: + name, selector, labels = entry + return _unchanged(f"{name}{{{selector}}}", labels, window, names, vm_url) + + with ThreadPoolExecutor(4) as pool: + found = list(pool.map(check, series)) + datasets, unchanged = [], {} + for (name, selector, labels), same in zip(series, found): + if same is None: + datasets.append(_raw(name, selector, labels, window)) + else: + unchanged[name] = same + return datasets, unchanged + + +def log_datasets(exp: dict) -> List[dict]: + """A nim-libp2p run's sends and deliveries, the rows of its summary/{sent,received}.csv.""" + if not exp["experiment"]["name"].startswith(LOG_EXPERIMENTS): + return [] + query = {"namespace": exp["stack"]["namespace"], "tracer": "nimlibp2p"} + return [ + { + "name": pattern, + "datasource": LOGS_DATASOURCE, + "timeRange": _window(exp, "complete"), + "query": query | {"pattern": pattern}, + "schema": _schema(fields), + } + for pattern, fields in LOG_FIELDS.items() + ] + + +def run_files(run: Path) -> dict: + """The run's events, without experiment dumps and per-message sends, and its deployed specs.""" + events = run / "events.log" + lines = events.read_text().splitlines() if events.exists() else [] + kept = [ + e + for e in map(json.loads, filter(None, lines)) + if e.get("event") not in ("metadata", "publish", "send") + ] + specs = { + path.name: path.read_text() for path in sorted((run / "deployment_yamls").glob("*.yaml")) + } + return {"events": kept, "deployment": specs} + + def experiment_body( - exp: dict, config: ScrapeConfig, run_name: str, label: Optional[str] = None + exp: dict, + config: ScrapeConfig, + run_name: str, + label: Optional[str] = None, + extra: List[dict] = (), + metadata: Optional[dict] = None, ) -> dict: - """One dataset per scraped metric, over the same window and step as the local CSVs.""" + """The scraped metrics over the stable window, as the local CSVs, then `extra` datasets.""" + stack = exp.get("stack", {}) return { "title": run_title(run_name, config), "family": exp["experiment"]["name"], "date": f"{config.start:%Y-%m-%d}", "docker_image": exp.get("params", {}).get("image"), + # The dashboard's log fetch reads statefulsets, nodes, namespace and container_name. "metadata": {"run": run_name, "experiment": exp["experiment"]["name"]} | {key: exp[key] for key in ("params", "code", "stack", "results") if key in exp} - | ({SCRAPE_LABEL: label} if label else {}), - "datasets": [metric_dataset(metric, config) for metric in config.metrics_to_scrape], + | ({SCRAPE_LABEL: label} if label else {}) + | { + "statefulsets": stack.get("stateful_sets"), + "nodes": stack.get("nodes_per_statefulset"), + "namespace": stack.get("namespace"), + "container_name": stack.get("container_name"), + } + | (metadata or {}), + "datasets": [metric_dataset(metric, config) for metric in config.metrics_to_scrape] + + list(extra), "panels": [], "publish": False, } -def _existing_id(title: str, url: str) -> str: +def run_body(exp: dict, run: Path) -> dict: + """Everything published for one run folder.""" + config = run_scrape_config(exp, name=run.name) + datasets, unchanged = run_datasets(exp, node_metrics(exp, config.url), config.url) + metadata = run_files(run) | {UNCHANGED: {"rule": UNCHANGED_RULE, "metrics": unchanged}} + extra = datasets + log_datasets(exp) + return experiment_body(exp, config, run.name, scrape_label(run), extra, metadata) + + +def expected_rows(body: dict, run: Path) -> Dict[str, int]: + """Row counts the log datasets must match: the run's own summary CSVs.""" + counts = {} + for dataset in body["datasets"]: + if dataset["datasource"] != LOGS_DATASOURCE: + continue + path = run / SUMMARY / f"{dataset['name']}.csv" + if not path.exists(): + logger.warning(f"`{run.name}` has no {path.name}; only checking it got rows") + continue + with open(path, newline="") as f: + counts[dataset["name"]] = sum(1 for _ in csv.reader(f)) - 1 + return counts + + +def _existing(title: str, url: str) -> dict: response = requests.get(f"{url}/experiments", timeout=TIMEOUT_S) response.raise_for_status() - ids = [e["id"] for e in response.json() if e["title"] == title] - if not ids: + found = [e for e in response.json() if e["title"] == title] + if not found: raise PublishError(f"`{title}` got a 409 but is not on the dashboard") - return ids[0] + return found[0] -def _empty_datasets(experiment_id: str, url: str) -> List[str]: - response = requests.get(f"{url}/experiments/{experiment_id}/datasets", timeout=TIMEOUT_S) - response.raise_for_status() - return [d["name"] for d in response.json()["datasets"] if not d.get("rowCount")] +def _bad_datasets( + experiment_id: str, names: List[str], url: str, expected: Dict[str, int] +) -> List[str]: + """Datasets stored empty, or with other row counts than the run's CSVs.""" + bad = [] + for name in names: + response = requests.get( + f"{url}/experiments/{experiment_id}/datasets/{name}/data", timeout=TIMEOUT_S + ) + if response.status_code == 404: + bad.append(f"{name} (not stored)") + continue + response.raise_for_status() + rows = len(response.json()["data"]) + if not rows or rows != expected.get(name, rows): + bad.append(f"{name} ({rows} rows, expected {expected.get(name, 'some')})") + return bad -def publish(body: dict, token: str, url: str = DASHBOARD_URL) -> str: - """Create the experiment, or find the one already there, and check every dataset has rows.""" - headers = {"Authorization": f"Bearer {token}"} +def publish( + body: dict, token: str, url: str = DASHBOARD_URL, expected: Dict[str, int] = None +) -> str: + """Create the experiment and check every dataset's rows, or check the one already there.""" + headers, title, expected = {"Authorization": f"Bearer {token}"}, body["title"], expected or {} response = requests.post(f"{url}/experiments", json=body, headers=headers, timeout=TIMEOUT_S) if response.status_code == 409: - experiment_id = _existing_id(body["title"], url) - else: - response.raise_for_status() - experiment_id = response.json()["id"] + existing = _existing(title, url) + names = [d["name"] for d in existing["datasets"]] + missing = [d["name"] for d in body["datasets"] if d["name"] not in names] + if missing: + logger.warning(f"`{title}` is already on the dashboard without {missing}; not added") + # Never deleted here: it may still be processing, or hold data VM no longer has. + bad = _bad_datasets(existing["id"], names, url, expected) + if bad: + raise PublishError(f"`{title}` is already on the dashboard with {bad}") + logger.info(f"`{title}` is already on the dashboard as {existing['id']}") + return existing["id"] + if not response.ok: + raise PublishError(f"`{title}` got {response.status_code}: {response.text[:500]}") - empty = _empty_datasets(experiment_id, url) - if empty: + experiment_id = response.json()["id"] + try: + bad = _bad_datasets(experiment_id, [d["name"] for d in body["datasets"]], url, expected) + if bad: + raise PublishError(f"`{title}` got {bad}; removed it") + except Exception: requests.delete( f"{url}/experiments/{experiment_id}", headers=headers, timeout=TIMEOUT_S ).raise_for_status() - raise PublishError(f"`{body['title']}` got no rows for {empty}; removed it") + raise - logger.info(f"`{body['title']}` is on the dashboard as {experiment_id}") + logger.info(f"`{title}` is on the dashboard as {experiment_id}") return experiment_id def unpublishable(exp: dict) -> Optional[str]: - """Why a run stays off the dashboard, or None if its 10ksim code was clean and pushed.""" + """Why a run stays off the dashboard, or None if it can go there.""" + # The bridge leaves out `duration` when the window is inverted. + if not exp.get("results", {}).get("stable", {}).get("duration"): + return "it has no stable window" code = exp.get("code") if not code: return "no 10ksim commit was recorded" @@ -122,7 +395,7 @@ def unpublishable(exp: dict) -> Optional[str]: def publish_run_metrics(exp: dict, run: Path) -> Optional[str]: - """Publish what `scrape_run_metrics` scraped for this run. Logs failures, never raises.""" + """Publish this run's metrics, logs and specs. Logs failures, never raises.""" reason = unpublishable(exp) if reason: logger.warning(f"`{run.name}` is not on the dashboard: {reason}") @@ -132,8 +405,8 @@ def publish_run_metrics(exp: dict, run: Path) -> Optional[str]: logger.warning(f"{TOKEN_ENV} is not set; `{run.name}` is not on the dashboard") return None try: - config = run_scrape_config(exp, name=run.name) - return publish(experiment_body(exp, config, run.name, scrape_label(run)), token) + body = run_body(exp, run) + return publish(body, token, expected=expected_rows(body, run)) except Exception: logger.exception(f"Publishing `{run.name}` to the dashboard failed") return None diff --git a/src/analysis/post_run/tests/test_fetch.py b/src/analysis/post_run/tests/test_fetch.py index cff23bfc..7c443b51 100644 --- a/src/analysis/post_run/tests/test_fetch.py +++ b/src/analysis/post_run/tests/test_fetch.py @@ -33,6 +33,8 @@ class _Response: + status_code = 200 + def __init__(self, body): self._body = body @@ -76,9 +78,10 @@ def _published(monkeypatch, run, query=None, empty=()): def get(url, **kwargs): if url.endswith("/experiments"): return _Response(experiments) - assert url.endswith("/experiments/e1/datasets"), url rows = {STREAMS: _stored_rows(VM_RESPONSE)} | dict.fromkeys(empty, []) - return _Response({"datasets": [{"name": n, "data": r} for n, r in rows.items()]}) + name = url.removeprefix("http://dash/experiments/e1/datasets/").removesuffix("/data") + assert name in rows, url + return _Response({"data": rows[name]}) monkeypatch.setattr(fetch.requests, "get", get) diff --git a/src/analysis/post_run/tests/test_publish.py b/src/analysis/post_run/tests/test_publish.py index 3d9b1612..0a0be354 100644 --- a/src/analysis/post_run/tests/test_publish.py +++ b/src/analysis/post_run/tests/test_publish.py @@ -1,7 +1,9 @@ +import re from pathlib import Path import pytest +from src.analysis.post_run import fetch from src.analysis.post_run import publish as publish_module from src.analysis.post_run.metrics import run_scrape_config from src.analysis.post_run.publish import PublishError, experiment_body, publish @@ -12,24 +14,71 @@ "stack": {"namespace": "zerotesting"}, "results": { "complete": {"start": "2026-09-16T10:03:45", "end": "2026-09-16T10:26:32"}, - "stable": {"start": "2026-09-16T10:10:00", "end": "2026-09-16T10:16:30"}, + "stable": { + "start": "2026-09-16T10:10:00", + "end": "2026-09-16T10:16:30", + "duration": "0h 6m 30s", + }, }, } +BODY = {"title": "run", "datasets": [{"name": "mesh-peers"}, {"name": "received"}]} class _Response: def __init__(self, status_code, body=None): self.status_code = status_code + self.ok = status_code < 400 + self.text = "" self._body = body def json(self): return self._body def raise_for_status(self): - if self.status_code >= 400: + if not self.ok: raise RuntimeError(self.status_code) +def _dashboard(monkeypatch, post, existing, rows): + """A dashboard answering a POST with `post`, holding `existing`, its datasets with `rows`.""" + deleted = [] + + def get(url, **kwargs): + if url == "http://dash/experiments": + return _Response(200, [{"id": "e0", "title": "other"}, existing]) + name = url.removeprefix("http://dash/experiments/e1/datasets/").removesuffix("/data") + if isinstance(rows.get(name), Exception): + raise rows[name] + if name not in rows: + return _Response(404) + return _Response(200, {"data": [{}] * rows[name]}) + + monkeypatch.setattr(publish_module.requests, "post", lambda *a, **k: post) + monkeypatch.setattr(publish_module.requests, "get", get) + monkeypatch.setattr( + publish_module.requests, "delete", lambda url, **k: deleted.append(url) or _Response(204) + ) + return deleted + + +def _vm(monkeypatch, discovered, metrics): + """VM with `discovered` series on the first pods; `metrics`: name -> (min, max, series).""" + + def get(url, params=None, **kwargs): + assert url.endswith("/query"), f"contacted {url}" + query = params["query"] + if query.startswith("last_over_time({"): + return _Response(200, {"data": {"result": [{"metric": s} for s in discovered]}}) + name = re.search(r"_over_time\((\w+)\{", query)[1] + low, high, series = metrics.get(name, (0.0, 5.0, [])) + if query.startswith("count by"): + return _Response(200, {"data": {"result": [{"metric": dict(s)} for s in series]}}) + value = low if query.startswith("min") else high + return _Response(200, {"data": {"result": [{"value": [0, str(value)]}]}}) + + monkeypatch.setattr(publish_module.requests, "get", get) + + def test_each_scraped_metric_is_a_dataset_over_the_stable_window(): config = run_scrape_config(EXP, name="run") body = experiment_body(EXP, config, "run") @@ -48,44 +97,129 @@ def test_each_scraped_metric_is_a_dataset_over_the_stable_window(): assert {"instance", "type", "dir"} <= set(open_streams) -@pytest.mark.parametrize("created", [True, False]) -def test_a_dataset_without_rows_fails_the_publish_and_is_removed(monkeypatch, created): - deleted = [] - gets = { - "http://dash/experiments": [{"id": "e0", "title": "other"}, {"id": "e1", "title": "run"}], - "http://dash/experiments/e1/datasets": { - "datasets": [ - {"name": "libp2p-in", "rowCount": 7}, - {"name": "mesh-peers", "rowCount": 0}, - ] - }, - } - post = _Response(201, {"id": "e1"}) if created else _Response(409) - monkeypatch.setattr(publish_module.requests, "post", lambda *a, **k: post) - monkeypatch.setattr(publish_module.requests, "get", lambda url, **k: _Response(200, gets[url])) - monkeypatch.setattr( - publish_module.requests, "delete", lambda url, **k: deleted.append(url) or _Response(204) - ) - - with pytest.raises(PublishError, match="mesh-peers"): - publish({"title": "run"}, token="t", url="http://dash") +@pytest.mark.parametrize( + "rows, bad", + [ + ({"mesh-peers": 0, "received": 5}, "mesh-peers"), + ({"mesh-peers": 3, "received": 4}, "received"), + ({"mesh-peers": 3, "received": TimeoutError("read timed out")}, "read timed out"), + ({"mesh-peers": 3, "received": 5}, None), + ], +) +def test_a_created_experiment_that_fails_its_check_is_removed(monkeypatch, rows, bad): + deleted = _dashboard(monkeypatch, _Response(201, {"id": "e1"}), {"id": "e1"}, rows) + + if bad is None: + assert publish(BODY, token="t", url="http://dash", expected={"received": 5}) == "e1" + assert deleted == [] + return + with pytest.raises((PublishError, TimeoutError), match=bad): + publish(BODY, token="t", url="http://dash", expected={"received": 5}) assert deleted == ["http://dash/experiments/e1"] @pytest.mark.parametrize( - "code, published", + "datasets, rows, bad", + [ + (["mesh-peers", "received"], {"mesh-peers": 3, "received": 4}, "received"), + (["mesh-peers", "received"], {"mesh-peers": 3}, "not stored"), + (["mesh-peers"], {"mesh-peers": 3}, None), + ], +) +def test_an_experiment_already_there_is_checked_but_never_removed(monkeypatch, datasets, rows, bad): + existing = {"id": "e1", "title": "run", "datasets": [{"name": n} for n in datasets]} + deleted = _dashboard(monkeypatch, _Response(409), existing, rows) + + if bad is None: + assert publish(BODY, token="t", url="http://dash", expected={"received": 5}) == "e1" + else: + with pytest.raises(PublishError, match=bad): + publish(BODY, token="t", url="http://dash", expected={"received": 5}) + assert deleted == [] + + +def test_node_metrics_are_the_node_jobs_own_series_on_every_statefulset(monkeypatch): + discovered = [ + {"__name__": "libp2p_peers", "job": "nodes", "pod": "relay-0", "instance": "i"}, + {"__name__": "libp2p_network_bytes_total", "job": "nodes", "direction": "in"}, + {"__name__": "libp2p_network_bytes_total", "job": "nodes", "direction": "out"}, + {"__name__": "logos_archive_messages", "job": "nodes", "pod": "store-0"}, + {"__name__": "libp2p_pubsub_peers_created", "job": "nodes"}, + {"__name__": "scrape_duration_seconds", "job": "nodes"}, + {"__name__": "up", "job": "nodes"}, + {"__name__": "container_cpu_usage_seconds_total", "job": "kubelet", "cpu": "total"}, + ] + stack = {"namespace": "z", "stateful_sets": ["relay", "store"], "nodes_per_statefulset": [9, 2]} + exp = {**EXP, "stack": stack} + + _vm(monkeypatch, discovered, {}) + assert publish_module.node_metrics(exp, "http://vm") == { + "libp2p_network_bytes_total": ["direction"], + "libp2p_peers": [], + "logos_archive_messages": [], + } + _vm(monkeypatch, [], {}) + with pytest.raises(PublishError, match="No node metrics"): + publish_module.node_metrics(exp, "http://vm") + + +def test_a_metric_that_never_changes_is_published_as_its_value_and_read_back_as_it( + monkeypatch, tmp_path +): + pods = ["pod-0", "pod-1", "pod-2"] + every = lambda labels: [{"pod": pod} | labels for pod in pods] + # name: (min, max, series over the run), and the labels discovery finds on it + vm = { + "libp2p_peers": (3.0, 9.0, every({}), {}), + "libp2p_pubsub_topics": (1.0, 1.0, every({}), {}), + "logos_delivery_version": (1.0, 1.0, every({"version": "v1"}), {"version": "v1"}), + "libp2p_gossipsub_received_iwants": ( + 0.0, + 0.0, + every({"kind": "a"}) + every({"kind": "b"}), + {"kind": "a"}, + ), + "libp2p_gossipsub_low_peers_topics": (0.0, 0.0, every({})[:2], {}), + } + discovered = [{"__name__": n, "job": "nodes"} | v[3] for n, v in vm.items()] + _vm(monkeypatch, discovered, {n: v[:3] for n, v in vm.items()}) + stack = {"namespace": "z", "stateful_sets": ["pod"], "nodes_per_statefulset": [3]} + exp = {**EXP, "stack": stack | {"container_name": "pod-0"}} + body = publish_module.run_body(exp, tmp_path / "run") + + names = {d["name"] for d in body["datasets"]} + assert {"libp2p_peers", "libp2p_gossipsub_received_iwants"} <= names + assert "libp2p_gossipsub_low_peers_topics" in names + assert body["metadata"]["unchanged"]["metrics"] == { + "libp2p_pubsub_topics": {"value": 1.0, "labels": {}}, + "logos_delivery_version": {"value": 1.0, "labels": {"version": "v1"}}, + } + end = EXP["results"]["complete"]["end"] + assert fetch.run_dataset({"id": "e1"} | body, "logos_delivery_version") == [ + {"timestamp": end, "value": 1.0, "pod": pod, "version": "v1"} for pod in pods + ] + + +@pytest.mark.parametrize( + "stable, code, published", [ - (None, False), - ({"commit": "abc1234", "clean": False, "pushed": True}, False), - ({"commit": "abc1234", "clean": True, "pushed": False}, False), - ({"commit": "abc1234", "clean": True, "pushed": True}, True), + (True, None, False), + (True, {"commit": "abc1234", "clean": False, "pushed": True}, False), + (True, {"commit": "abc1234", "clean": True, "pushed": False}, False), + (True, {"commit": "abc1234", "clean": True, "pushed": True}, True), + (False, {"commit": "abc1234", "clean": True, "pushed": True}, False), ], ) -def test_only_runs_from_clean_pushed_code_are_published(monkeypatch, code, published): +def test_only_runs_with_a_stable_window_from_clean_pushed_code_are_published( + monkeypatch, stable, code, published +): posted = [] monkeypatch.setenv(publish_module.TOKEN_ENV, "t") - monkeypatch.setattr(publish_module, "publish", lambda body, token: posted.append(body)) + monkeypatch.setattr(publish_module, "run_body", lambda exp, run: {"datasets": []}) + monkeypatch.setattr(publish_module, "publish", lambda body, token, **k: posted.append(body)) + # The bridge leaves out `duration` when the stable window is inverted. + results = EXP["results"] if stable else {"stable": {"start": "10:02:25", "end": "10:01:51"}} - publish_module.publish_run_metrics({**EXP, "code": code}, Path("run")) + publish_module.publish_run_metrics({**EXP, "results": results, "code": code}, Path("run")) assert bool(posted) is published diff --git a/src/analysis/publish_run.py b/src/analysis/publish_run.py index 3d30b607..17d082a5 100644 --- a/src/analysis/publish_run.py +++ b/src/analysis/publish_run.py @@ -1,4 +1,4 @@ -"""Publish finished runs' metrics to the DST dashboard from their folders.""" +"""Publish finished runs' metrics, logs and specs to the DST dashboard from their folders.""" import argparse import json @@ -7,13 +7,12 @@ import sys from pathlib import Path -from src.analysis.post_run.metrics import run_scrape_config from src.analysis.post_run.publish import ( DASHBOARD_URL, TOKEN_ENV, - experiment_body, + expected_rows, publish, - scrape_label, + run_body, unpublishable, ) from src.analysis.utils.log_utils import init_logger @@ -35,23 +34,17 @@ def main() -> None: for arg in args.runs: run = Path(arg).resolve() exp = json.loads((run / "metadata.json").read_text()) - # Same rule as the post-run scrape, so the dashboard holds what the CSVs hold. - if not exp.get("results", {}).get("stable", {}).get("duration"): - logger.error(f"`{run.name}` has no stable window; skipping it") - failed.append(run.name) - continue reason = unpublishable(exp) if reason: logger.error(f"`{run.name}` is not published: {reason}") failed.append(run.name) continue - config = run_scrape_config(exp, name=run.name) - body = experiment_body(exp, config, run.name, scrape_label(run)) - if args.dry_run: - print(json.dumps(body, indent=2)) - continue try: - publish(body, token) + body = run_body(exp, run) + if args.dry_run: + print(json.dumps(body, indent=2)) + continue + publish(body, token, expected=expected_rows(body, run)) except Exception: logger.exception(f"Publishing `{run.name}` failed") failed.append(run.name)